From 1ee01ec60106c4da3853a155785bfcbbcfa5172e Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Tue, 21 Jan 2025 04:51:54 +0100 Subject: [PATCH] service-mesh: blackhole traffic destined for the TPROXY port Traffic to the TPROXY port (15006/15007) led to a traffic storm as envoy used the original destination to forward the traffic to, therefore forwarding it again to the TPROXY port where envoy listens. This commit introduces a Blackhole cluster where we send traffic to, that arrives on the TPROXY listeners and which original destination port is the TPROXY. --- service-mesh/config.go | 149 +++++++++++++++++++++++++++++++++++++++-- service-mesh/go.mod | 2 +- 2 files changed, 146 insertions(+), 5 deletions(-) diff --git a/service-mesh/config.go b/service-mesh/config.go index a1d8a4a5e9..84e630222e 100644 --- a/service-mesh/config.go +++ b/service-mesh/config.go @@ -9,7 +9,10 @@ import ( "net/netip" "strconv" "strings" + "time" + envoyXDSCoreV3 "github.com/cncf/xds/go/xds/core/v3" + envoyXDSMatcherV3 "github.com/cncf/xds/go/xds/type/matcher/v3" envoyConfigBootstrapV3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" envoyConfigClusterV3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoyCoreV3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -17,14 +20,21 @@ import ( envoyConfigListenerV3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoyOrigDstV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/original_dst/v3" envoyConfigTCPProxyV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" + envoyConfigNetworkV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/common_inputs/network/v3" envoyTLSV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" ) var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") +const ( + blackHoleClusterName = "BlackHoleCluster" + ingressClusterName = "ingressCluster" +) + // ProxyConfig represents the configuration for the proxy. type ProxyConfig struct { egress []egressConfigEntry @@ -179,7 +189,7 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { } ingressCluster := &envoyConfigClusterV3.Cluster{ - Name: "ingress", + Name: ingressClusterName, ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{Type: envoyConfigClusterV3.Cluster_ORIGINAL_DST}, DnsLookupFamily: envoyConfigClusterV3.Cluster_V4_ONLY, LbPolicy: envoyConfigClusterV3.Cluster_CLUSTER_PROVIDED, @@ -192,6 +202,10 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { config.StaticResources.Listeners = listeners config.StaticResources.Clusters = clusters + if err := addBlackHoleToConfig(config); err != nil { + return nil, err + } + if err := config.ValidateAll(); err != nil { return nil, err } @@ -231,9 +245,10 @@ func listener(entry egressConfigEntry) (*envoyConfigListenerV3.Listener, error) }, FilterChains: []*envoyConfigListenerV3.FilterChain{ { + Name: entry.name, Filters: []*envoyConfigListenerV3.Filter{ { - Name: "envoy.filters.network.tcp_proxy", + Name: "ingress", ConfigType: &envoyConfigListenerV3.Filter_TypedConfig{ TypedConfig: proxyAny, }, @@ -288,7 +303,7 @@ func cluster(entry egressConfigEntry) (*envoyConfigClusterV3.Cluster, error) { func ingressListener(name string, listenPort uint16, requireClientCertificate bool) (*envoyConfigListenerV3.Listener, error) { ingressListener, err := listener(egressConfigEntry{ name: name, - clusterName: "ingress", + clusterName: ingressClusterName, listenAddr: netip.MustParseAddr("0.0.0.0"), listenPort: listenPort, }) @@ -303,7 +318,7 @@ func ingressListener(name string, listenPort uint16, requireClientCertificate bo } ingressListener.ListenerFilters = []*envoyConfigListenerV3.ListenerFilter{ { - Name: "envoy.filters.listener.original_dst", + Name: "tcpListener", ConfigType: &envoyConfigListenerV3.ListenerFilter_TypedConfig{TypedConfig: originalDstAny}, }, } @@ -397,3 +412,129 @@ func downstreamTLSTransportSocket(requireClientCertificate bool) (*envoyCoreV3.T }, }, nil } + +func addBlackHoleToConfig(config *envoyConfigBootstrapV3.Bootstrap) error { + // Add blackHoleCluster + config.StaticResources.Clusters = append(config.StaticResources.Clusters, blackHoleCluster()) + + // Add BlackHole matching to all listeners + for _, listener := range config.StaticResources.Listeners { + if listener.FilterChainMatcher != nil { + return fmt.Errorf("listener %s already has a filterChainMatcher", listener.Name) + } + listenPort := listener.Address.GetSocketAddress().GetPortValue() + if listenPort == 0 { + return fmt.Errorf("listener %s listens on port 0", listener.Name) + } + if listenPort != 15006 && listenPort != 15007 { + // listener is none of the ingress listeners + continue + } + if len(listener.FilterChains) != 1 { + return fmt.Errorf("listener %s doesn't has exactly one existing listener", listener.Name) + } + var err error + listener.FilterChainMatcher, err = filterChainMatcher(int(listenPort), listener.FilterChains[0].GetName()) + if err != nil { + return fmt.Errorf("could not add filterChainMatcher to listener %s: %w", listener.Name, err) + } + + bhFilter, err := blackHoleFilter() + if err != nil { + return err + } + listener.FilterChains = append(listener.FilterChains, bhFilter) + } + return nil +} + +// Blackhole traffic that arrives on the original destination listerners, +// which original port is the envoy itself, i.e. traffic that was not redirected +// to the envoy via the TROXY iptables rule. Such traffic would lead to a +// traffic storm since envoy would connect to the original destination +// i.e. itself again. +func blackHoleCluster() *envoyConfigClusterV3.Cluster { + return &envoyConfigClusterV3.Cluster{ + Name: blackHoleClusterName, + ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{ + Type: envoyConfigClusterV3.Cluster_STATIC, + }, + ConnectTimeout: durationpb.New(10 * time.Second), + } +} + +func blackHoleFilter() (*envoyConfigListenerV3.FilterChain, error) { + blackHole := &envoyConfigTCPProxyV3.TcpProxy{ + StatPrefix: blackHoleClusterName, + ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{ + Cluster: blackHoleClusterName, + }, + } + + blackHoleAny, err := anypb.New(blackHole) + if err != nil { + return nil, err + } + + return &envoyConfigListenerV3.FilterChain{ + Name: "BlackHoleFilter", + Filters: []*envoyConfigListenerV3.Filter{ + { + Name: "BlackHoleFilter", + ConfigType: &envoyConfigListenerV3.Filter_TypedConfig{ + TypedConfig: blackHoleAny, + }, + }, + }, + }, nil +} + +func filterChainMatcher(port int, defaultFilter string) (*envoyXDSMatcherV3.Matcher, error) { + ingressStringValueAny, err := anypb.New(wrapperspb.String(defaultFilter)) + if err != nil { + return nil, err + } + + blackHoleStringValueAny, err := anypb.New(wrapperspb.String("BlackHoleFilter")) + if err != nil { + return nil, err + } + + destPortStringValueAny, err := anypb.New(&envoyConfigNetworkV3.DestinationPortInput{}) + if err != nil { + return nil, err + } + + return &envoyXDSMatcherV3.Matcher{ + MatcherType: &envoyXDSMatcherV3.Matcher_MatcherTree_{ + MatcherTree: &envoyXDSMatcherV3.Matcher_MatcherTree{ + Input: &envoyXDSCoreV3.TypedExtensionConfig{ + Name: "port", + TypedConfig: destPortStringValueAny, + }, + TreeType: &envoyXDSMatcherV3.Matcher_MatcherTree_ExactMatchMap{ + ExactMatchMap: &envoyXDSMatcherV3.Matcher_MatcherTree_MatchMap{ + Map: map[string]*envoyXDSMatcherV3.Matcher_OnMatch{ + strconv.Itoa(port): { + OnMatch: &envoyXDSMatcherV3.Matcher_OnMatch_Action{ + Action: &envoyXDSCoreV3.TypedExtensionConfig{ + Name: "forwardToBlackHoleFilter", + TypedConfig: blackHoleStringValueAny, + }, + }, + }, + }, + }, + }, + }, + }, + OnNoMatch: &envoyXDSMatcherV3.Matcher_OnMatch{ + OnMatch: &envoyXDSMatcherV3.Matcher_OnMatch_Action{ + Action: &envoyXDSCoreV3.TypedExtensionConfig{ + Name: "forwardToIngress", + TypedConfig: ingressStringValueAny, + }, + }, + }, + }, nil +} diff --git a/service-mesh/go.mod b/service-mesh/go.mod index 2fa070cf70..c849a6d1b9 100644 --- a/service-mesh/go.mod +++ b/service-mesh/go.mod @@ -3,6 +3,7 @@ module github.com/edgelesssys/contrast/service-mesh go 1.23.0 require ( + github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 github.com/coreos/go-iptables v0.8.0 github.com/envoyproxy/go-control-plane/envoy v1.32.2 google.golang.org/protobuf v1.36.1 @@ -10,7 +11,6 @@ require ( require ( cel.dev/expr v0.16.0 // indirect - github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect