tap/fuzz: transport socket extension for traffic capture. (envoyproxy…

…#3244)

* tap/fuzz: transport socket extension for traffic capture.

This PR introduces a transport socket extension that wraps a given transport socket, interposes on its
plain text traffic and records it into a proto trace file on the filesystem. This can be used for a
number of purposes:
1. As a corpus for fuzzing the data plane.
2. Converted to PCAP using a soon-to-be-written utility, allowing existing tools such as Wireshark
   to be used to decode L4/L7 protocol history in the trace. Essentially this lets us take advantage
   of the PCAP ecosystem.

Relates to envoyproxy#1413 and envoyproxy#508.

Risk Level: Low (opt-in).
Testing: New SSL integration tests, demonstrating plain text intercept.

Signed-off-by: Harvey Tuch <[email protected]>

.circleci/config.yml

@@ -1,6 +1,6 @@
 references:
   envoy-build-image: &envoy-build-image
-    envoyproxy/envoy-build:220e5cb537b5185c953de1aac7d0613f8cf155ac
+    envoyproxy/envoy-build:7f7f5666c72e00ac7c1909b4fc9a2121d772c859
 
 version: 2
 jobs:

api/docs/BUILD

@@ -48,6 +48,8 @@ proto_library(
         "//envoy/config/metrics/v2:stats",
         "//envoy/config/ratelimit/v2:rls",
         "//envoy/config/trace/v2:trace",
+        "//envoy/config/transport_socket/capture/v2alpha:capture",
+        "//envoy/extensions/common/tap/v2alpha:capture",
         "//envoy/service/discovery/v2:ads",
         "//envoy/service/load_stats/v2:lrs",
         "//envoy/service/metrics/v2:metrics_service",

api/envoy/api/v2/BUILD

@@ -11,6 +11,7 @@ package_group(
         "//envoy/admin/...",
         "//envoy/api/v2",
         "//envoy/config/...",
+        "//envoy/extensions/...",
         "//envoy/service/...",
     ],
 )

api/envoy/config/transport_socket/capture/v2alpha/BUILD

@@ -0,0 +1,11 @@
+load("//bazel:api_build_system.bzl", "api_proto_library")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library(
+    name = "capture",
+    srcs = ["capture.proto"],
+    deps = [
+        "//envoy/api/v2/core:base",
+    ],
+)

api/envoy/config/transport_socket/capture/v2alpha/capture.proto

@@ -0,0 +1,45 @@
+syntax = "proto3";
+
+package envoy.config.transport_socket.capture.v2alpha;
+option go_package = "v2";
+
+// [#protodoc-title: Capture]
+
+import "envoy/api/v2/core/base.proto";
+
+// File sink.
+//
+// .. warning::
+//
+//   The current file sink implementation buffers the entire trace in memory
+//   prior to writing. This will OOM for long lived sockets and/or where there
+//   is a large amount of traffic on the socket.
+message FileSink {
+  // Path prefix. The output file will be of the form <path_prefix>_<id>.pb, where <id> is an
+  // identifier distinguishing the recorded trace for individual socket instances (the Envoy
+  // connection ID).
+  string path_prefix = 1;
+
+  // File format.
+  enum Format {
+    // Binary proto format as per :ref:`Trace
+    // <envoy_api_msg_extensions.common.tap.v2alpha.Trace>`.
+    PROTO_BINARY = 0;
+    // Text proto format as per :ref:`Trace
+    // <envoy_api_msg_extensions.common.tap.v2alpha.Trace>`.
+    PROTO_TEXT = 1;
+  }
+  Format format = 2;
+}
+
+// Configuration for capture transport socket. This wraps another transport socket, providing the
+// ability to interpose and record in plain text any traffic that is surfaced to Envoy.
+message Capture {
+  oneof sink_selector {
+    // Trace is to be written to a file sink.
+    FileSink file_sink = 1;
+  }
+
+  // The underlying transport socket being wrapped.
+  api.v2.core.TransportSocket transport_socket = 2;
+}

api/envoy/extensions/common/tap/v2alpha/BUILD

@@ -0,0 +1,9 @@
+load("//bazel:api_build_system.bzl", "api_proto_library")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library(
+    name = "capture",
+    srcs = ["capture.proto"],
+    deps = ["//envoy/api/v2/core:address"],
+)

api/envoy/extensions/common/tap/v2alpha/capture.proto

@@ -0,0 +1,57 @@
+syntax = "proto3";
+
+// [#protodoc-title: Common TAP]
+// Trace capture format for the capture transport socket extension. This dumps plain text read/write
+// sequences on a socket.
+
+package envoy.extensions.common.tap.v2alpha;
+option go_package = "v2";
+
+import "envoy/api/v2/core/address.proto";
+
+import "google/protobuf/timestamp.proto";
+
+// Connection properties.
+message Connection {
+  // Global unique connection ID for Envoy session. Matches connection IDs used
+  // in Envoy logs.
+  uint64 id = 1;
+  // Local address.
+  envoy.api.v2.core.Address local_address = 2;
+  // Remote address.
+  envoy.api.v2.core.Address remote_address = 3;
+}
+
+// Event in a capture trace.
+message Event {
+  // Timestamp for event.
+  google.protobuf.Timestamp timestamp = 1;
+  // Data read by Envoy from the transport socket.
+  message Read {
+    // Binary data read.
+    bytes data = 1;
+    // TODO(htuch): Half-close for reads.
+  }
+  // Data written by Envoy to the transport socket.
+  message Write {
+    // Binary data written.
+    bytes data = 1;
+    // Stream was half closed after this write.
+    bool end_stream = 2;
+  }
+  // Read or write with content as bytes string.
+  oneof event_selector {
+    Read read = 2;
+    Write write = 3;
+  }
+}
+
+// Sequence of read/write events that constitute a captured trace on a socket.
+// Multiple Trace messages might be emitted for a given connection ID, with the
+// sink (e.g. file set, network) responsible for later reassembly.
+message Trace {
+  // Connection properties.
+  Connection connection = 1;
+  // Sequence of observed events.
+  repeated Event events = 2;
+}

api/tools/BUILD

@@ -1,5 +1,29 @@
 licenses(["notice"])  # Apache 2
 
+py_binary(
+    name = "capture2pcap",
+    srcs = ["capture2pcap.py"],
+    licenses = ["notice"],  # Apache 2
+    visibility = ["//visibility:public"],
+    deps = [
+        "//envoy/extensions/common/tap/v2alpha:capture_py",
+    ],
+)
+
+py_test(
+    name = "capture2pcap_test",
+    srcs = ["capture2pcap_test.py"],
+    data = [
+        "data/capture2pcap_h2_ipv4.pb_text",
+        "data/capture2pcap_h2_ipv4.txt",
+    ],
+    # Don't run this by default, since we don't want to force local dependency on Wireshark/tshark,
+    # will explicitly invoke in CI.
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+    deps = [":capture2pcap"],
+)
+
 py_binary(
     name = "generate_listeners",
     srcs = ["generate_listeners.py"],

api/tools/capture2pcap.py

@@ -0,0 +1,91 @@
+"""Tool to convert Envoy capture trace format to PCAP.
+
+Uses od and text2pcap (part of Wireshark) utilities to translate the Envoy
+capture trace proto format to a PCAP file suitable for consuming in Wireshark
+and other tools in the PCAP ecosystem. The TCP stream in the output PCAP is
+synthesized based on the known IP/port/timestamps that Envoy produces in its
+capture files; it is not a literal wire capture.
+
+Usage:
+
+bazel run @envoy_api//tools:capture2pcap <capture .pb/.pb_text> <pcap path>
+
+Known issues:
+- IPv6 PCAP generation has malformed TCP packets. This appears to be a text2pcap
+issue.
+
+TODO(htuch):
+- Figure out IPv6 PCAP issue above, or file a bug once the root cause is clear.
+"""
+
+import datetime
+import socket
+import StringIO
+import subprocess as sp
+import sys
+import time
+
+from google.protobuf import text_format
+
+from envoy.extensions.common.tap.v2alpha import capture_pb2
+
+
+def DumpEvent(direction, timestamp, data):
+  dump = StringIO.StringIO()
+  dump.write('%s\n' % direction)
+  # Adjust to local timezone
+  adjusted_dt = timestamp.ToDatetime() - datetime.timedelta(
+      seconds=time.altzone)
+  dump.write('%s\n' % adjusted_dt)
+  od = sp.Popen(
+      ['od', '-Ax', '-tx1', '-v'],
+      stdout=sp.PIPE,
+      stdin=sp.PIPE,
+      stderr=sp.PIPE)
+  packet_dump = od.communicate(data)[0]
+  dump.write(packet_dump)
+  return dump.getvalue()
+
+
+def Capture2Pcap(capture_path, pcap_path):
+  trace = capture_pb2.Trace()
+  if capture_path.endswith('.pb_text'):
+    with open(capture_path, 'r') as f:
+      text_format.Merge(f.read(), trace)
+  else:
+    with open(capture_path, 'r') as f:
+      trace.ParseFromString(f.read())
+
+  local_address = trace.connection.local_address.socket_address.address
+  local_port = trace.connection.local_address.socket_address.port_value
+  remote_address = trace.connection.remote_address.socket_address.address
+  remote_port = trace.connection.remote_address.socket_address.port_value
+
+  dumps = []
+  for event in trace.events:
+    if event.HasField('read'):
+      dumps.append(DumpEvent('I', event.timestamp, event.read.data))
+    elif event.HasField('write'):
+      dumps.append(DumpEvent('O', event.timestamp, event.write.data))
+
+  ipv6 = False
+  try:
+    socket.inet_pton(socket.AF_INET6, local_address)
+    ipv6 = True
+  except socket.error:
+    pass
+
+  text2pcap_args = [
+      'text2pcap', '-D', '-t', '%Y-%m-%d %H:%M:%S.', '-6' if ipv6 else '-4',
+      '%s,%s' % (remote_address, local_address), '-T',
+      '%d,%d' % (remote_port, local_port), '-', pcap_path
+  ]
+  text2pcap = sp.Popen(text2pcap_args, stdout=sp.PIPE, stdin=sp.PIPE)
+  text2pcap.communicate('\n'.join(dumps))
+
+
+if __name__ == '__main__':
+  if len(sys.argv) != 3:
+    print 'Usage: %s <capture .pb/.pb_text> <pcap path>' % sys.argv[0]
+    sys.exit(1)
+  Capture2Pcap(sys.argv[1], sys.argv[2])

api/tools/capture2pcap_test.py

@@ -0,0 +1,27 @@
+"""Tests for capture2pcap."""
+
+import os
+import subprocess as sp
+import sys
+
+import capture2pcap
+
+# Validate that the captured trace when run through capture2cap | tshark matches
+# a golden output file for the tshark dump. Since we run capture2pcap in a
+# subshell with a limited environment, the inferred time zone should be UTC.
+if __name__ == '__main__':
+  srcdir = os.path.join(os.getenv('TEST_SRCDIR'), 'envoy_api')
+  capture_path = os.path.join(srcdir, 'tools/data/capture2pcap_h2_ipv4.pb_text')
+  expected_path = os.path.join(srcdir, 'tools/data/capture2pcap_h2_ipv4.txt')
+  pcap_path = os.path.join(os.getenv('TEST_TMPDIR'), 'generated.pcap')
+
+  capture2pcap.Capture2Pcap(capture_path, pcap_path)
+  actual_output = sp.check_output(
+      ['tshark', '-r', pcap_path, '-d', 'tcp.port==10000,http2', '-P'])
+  with open(expected_path, 'r') as f:
+    expected_output = f.read()
+  if actual_output != expected_output:
+    print 'Mismatch'
+    print 'Expected: %s' % expected_output
+    print 'Actual: %s' % actual_output
+    sys.exit(1)

api/tools/data/capture2pcap_h2_ipv4.txt

@@ -0,0 +1,6 @@
+    1   0.000000    127.0.0.1 → 127.0.0.1    HTTP2 157 Magic, SETTINGS, WINDOW_UPDATE, HEADERS
+    2   0.013713    127.0.0.1 → 127.0.0.1    HTTP2 91 SETTINGS, SETTINGS, WINDOW_UPDATE
+    3   0.013820    127.0.0.1 → 127.0.0.1    HTTP2 63 SETTINGS
+    4   0.128649    127.0.0.1 → 127.0.0.1    HTTP2 5586 HEADERS
+    5   0.130006    127.0.0.1 → 127.0.0.1    HTTP2 7573 DATA
+    6   0.131044    127.0.0.1 → 127.0.0.1    HTTP2 3152 DATA, DATA

ci/build_setup.sh

@@ -116,7 +116,7 @@ mkdir -p "${ENVOY_CI_DIR}"/bazel
 ln -sf "${ENVOY_SRCDIR}"/bazel/get_workspace_status "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/bazel/
 ln -sf "${ENVOY_SRCDIR}"/bazel/get_workspace_status "${ENVOY_CI_DIR}"/bazel/
 
-export BUILDIFIER_BIN="/usr/lib/go/bin/buildifier"
+export BUILDIFIER_BIN="/usr/local/bin/buildifier"
 
 function cleanup() {
   # Remove build artifacts. This doesn't mess with incremental builds as these

ci/do_ci.sh

@@ -117,7 +117,8 @@ elif [[ "$1" == "bazel.api" ]]; then
   echo "Building API..."
   bazel --batch build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//envoy/...
   echo "Testing API..."
-  bazel --batch test ${BAZEL_TEST_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/...
+  bazel --batch test ${BAZEL_TEST_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \
+    @envoy_api//tools:capture2pcap_test
   exit 0
 elif [[ "$1" == "bazel.coverage" ]]; then
   setup_gcc_toolchain

docs/build.sh

@@ -91,6 +91,8 @@ PROTO_RST="
   /envoy/config/filter/network/redis_proxy/v2/redis_proxy/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto.rst
   /envoy/config/filter/network/tcp_proxy/v2/tcp_proxy/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto.rst
   /envoy/config/health_checker/redis/v2/redis/envoy/config/health_checker/redis/v2/redis.proto.rst
+  /envoy/config/transport_socket/capture/v2alpha/capture/envoy/config/transport_socket/capture/v2alpha/capture.proto.rst
+  /envoy/extensions/common/tap/v2alpha/capture/envoy/extensions/common/tap/v2alpha/capture.proto.rst
   /envoy/type/percent/envoy/type/percent.proto.rst
   /envoy/type/range/envoy/type/range.proto.rst
 "

docs/root/api-v2/api.rst

@@ -13,5 +13,6 @@ v2 API reference
   http_routes/http_routes
   config/filter/filter
   config/health_checker/health_checker
+  config/transport_socket/transport_socket
   common_messages/common_messages
   types/types

docs/root/api-v2/common_messages/common_messages.rst

@@ -13,3 +13,4 @@ Common messages
   ../api/v2/core/grpc_service.proto
   ../api/v2/auth/cert.proto
   ../api/v2/ratelimit/ratelimit.proto
+  ../extensions/common/tap/v2alpha/capture.proto

docs/root/api-v2/config/transport_socket/transport_socket.rst

@@ -0,0 +1,8 @@
+Transport sockets
+=================
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+
+  */v2alpha/*

docs/root/intro/version_history.rst

@@ -57,6 +57,8 @@ Version history
 * logger: added the ability to optionally set the log format via the :option:`--log-format` option.
 * logger: all :ref:`logging levels <operations_admin_interface_logging>` can be configured
   at run-time: trace debug info warning error critical.
+* sockets: added :ref:`capture transport socket extension <operations_traffic_capture>` to support
+  recording plain text traffic and PCAP generation.
 * sockets: added `IP_FREEBIND` socket option support for :ref:`listeners
   <envoy_api_field_Listener.freebind>` and upstream connections via
   :ref:`cluster manager wide

docs/root/operations/operations.rst

@@ -12,3 +12,4 @@ Operations and administration
   stats_overview
   runtime
   fs_flags
+  traffic_capture