diff --git a/pkg/xds/generator/egress/internal_services_generator.go b/pkg/xds/generator/egress/internal_services_generator.go index 1aea10752021..ab7c77348306 100644 --- a/pkg/xds/generator/egress/internal_services_generator.go +++ b/pkg/xds/generator/egress/internal_services_generator.go @@ -6,7 +6,6 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_xds "github.com/kumahq/kuma/pkg/core/xds" - util_maps "github.com/kumahq/kuma/pkg/util/maps" xds_context "github.com/kumahq/kuma/pkg/xds/context" envoy_listeners "github.com/kumahq/kuma/pkg/xds/envoy/listeners" "github.com/kumahq/kuma/pkg/xds/envoy/tags" @@ -27,7 +26,6 @@ func (g *InternalServicesGenerator) Generate( meshName := meshResources.Mesh.GetMeta().GetName() servicesMap := g.buildServices(meshResources.EndpointMap, meshResources.Mesh.ZoneEgressEnabled(), xdsCtx.ControlPlane.Zone) - services := util_maps.SortedKeys(servicesMap) availableServices := g.distinctAvailableServices(proxy.ZoneEgressProxy.ZoneIngresses, meshName, servicesMap) @@ -36,9 +34,9 @@ func (g *InternalServicesGenerator) Generate( xds_context.Resources{MeshLocalResources: meshResources.Resources}, ) - zoneproxy.AddFilterChains(availableServices, proxy.APIVersion, listenerBuilder, destinations, meshResources.EndpointMap) + services := zoneproxy.AddFilterChains(availableServices, proxy.APIVersion, listenerBuilder, destinations, meshResources.EndpointMap) - cds, err := zoneproxy.GenerateCDS(services, destinations, proxy.APIVersion, meshName, OriginEgress) + cds, err := zoneproxy.GenerateCDS(destinations, services, proxy.APIVersion, meshName, OriginEgress) if err != nil { return nil, err } diff --git a/pkg/xds/generator/egress/testdata/04.mixed-services-custom-trafficroute.golden.yaml b/pkg/xds/generator/egress/testdata/04.mixed-services-custom-trafficroute.golden.yaml index cefc40c23737..5dacfb8533d6 100644 --- a/pkg/xds/generator/egress/testdata/04.mixed-services-custom-trafficroute.golden.yaml +++ b/pkg/xds/generator/egress/testdata/04.mixed-services-custom-trafficroute.golden.yaml @@ -85,68 +85,6 @@ resources: '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions explicitHttpConfig: http2ProtocolOptions: {} -- name: mesh-1:service-1-zone-2 - resource: - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - altStatName: mesh-1_service-1-zone-2 - connectTimeout: 10s - edsClusterConfig: - edsConfig: - ads: {} - resourceApiVersion: V3 - name: mesh-1:service-1-zone-2 - type: EDS -- name: mesh-1:service-2-zone-2 - resource: - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster - altStatName: mesh-1_service-2-zone-2 - connectTimeout: 10s - edsClusterConfig: - edsConfig: - ads: {} - resourceApiVersion: V3 - name: mesh-1:service-2-zone-2 - type: EDS -- name: mesh-1:service-1-zone-2 - resource: - '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment - clusterName: mesh-1:service-1-zone-2 - endpoints: - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: 10.0.0.254 - portValue: 10001 - loadBalancingWeight: 3 - metadata: - filterMetadata: - envoy.lb: - kuma.io/protocol: http - mesh: mesh-1 - envoy.transport_socket_match: - kuma.io/protocol: http - mesh: mesh-1 -- name: mesh-1:service-2-zone-2 - resource: - '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment - clusterName: mesh-1:service-2-zone-2 - endpoints: - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: 10.0.0.254 - portValue: 10001 - loadBalancingWeight: 30 - metadata: - filterMetadata: - envoy.lb: - kuma.io/protocol: http2 - mesh: mesh-1 - envoy.transport_socket_match: - kuma.io/protocol: http2 - mesh: mesh-1 - name: inbound:192.168.0.1:10002 resource: '@type': type.googleapis.com/envoy.config.listener.v3.Listener diff --git a/pkg/xds/generator/ingress_generator.go b/pkg/xds/generator/ingress_generator.go index 499b041f3118..0afac0e8e5d0 100644 --- a/pkg/xds/generator/ingress_generator.go +++ b/pkg/xds/generator/ingress_generator.go @@ -43,14 +43,16 @@ func (i IngressGenerator) Generate( for _, mr := range proxy.ZoneIngressProxy.MeshResourceList { meshName := mr.Mesh.GetMeta().GetName() - services := maps.Keys(mr.EndpointMap) - sort.Strings(services) + serviceList := maps.Keys(mr.EndpointMap) + sort.Strings(serviceList) dest := zoneproxy.BuildMeshDestinations( availableSvcsByMesh[meshName], xds_context.Resources{MeshLocalResources: mr.Resources}, ) - cdsResources, err := zoneproxy.GenerateCDS(services, dest, proxy.APIVersion, meshName, OriginIngress) + services := zoneproxy.AddFilterChains(availableSvcsByMesh[meshName], proxy.APIVersion, listenerBuilder, dest, mr.EndpointMap) + + cdsResources, err := zoneproxy.GenerateCDS(dest, services, proxy.APIVersion, meshName, OriginIngress) if err != nil { return nil, err } @@ -61,8 +63,6 @@ func (i IngressGenerator) Generate( return nil, err } resources.Add(edsResources...) - - zoneproxy.AddFilterChains(availableSvcsByMesh[meshName], proxy.APIVersion, listenerBuilder, dest, mr.EndpointMap) } listener, err := listenerBuilder.Build() diff --git a/pkg/xds/generator/ingress_generator_test.go b/pkg/xds/generator/ingress_generator_test.go index 88db0c7008a6..9a0535802d01 100644 --- a/pkg/xds/generator/ingress_generator_test.go +++ b/pkg/xds/generator/ingress_generator_test.go @@ -715,6 +715,101 @@ var _ = Describe("IngressGenerator", func() { }, }, }), + Entry("with MeshHTTPRoute and subsets", testCase{ + ingress: ` + networking: + address: 10.0.0.1 + port: 10001 + availableServices: + - mesh: mesh1 + tags: + kuma.io/service: backend + version: v1 + region: eu + kuma.io/zone: zone + - mesh: mesh1 + tags: + kuma.io/service: backend + version: v2 + region: us + kuma.io/zone: zone +`, + expected: "with-meshhttproute-subset.envoy.golden.yaml", + meshResourceList: []*core_xds.MeshIngressResources{ + { + Mesh: builders.Mesh().WithName("mesh1").Build(), + EndpointMap: map[core_xds.ServiceName][]core_xds.Endpoint{ + "backend": { + { + Target: "192.168.0.1", + Port: 2521, + Tags: map[string]string{ + "kuma.io/service": "backend", + "version": "v1", + "region": "eu", + "mesh": "mesh1", + "kuma.io/zone": "zone", + }, + Weight: 1, + }, + { + Target: "192.168.0.2", + Port: 2521, + Tags: map[string]string{ + "kuma.io/service": "backend", + "version": "v2", + "region": "us", + "mesh": "mesh1", + "kuma.io/zone": "zone", + }, + Weight: 1, + }, + }, + }, + Resources: map[core_model.ResourceType]core_model.ResourceList{ + v1alpha1.MeshHTTPRouteType: &v1alpha1.MeshHTTPRouteResourceList{ + Items: []*v1alpha1.MeshHTTPRouteResource{ + { + Spec: &v1alpha1.MeshHTTPRoute{ + TargetRef: common_api.TargetRef{ + Kind: common_api.MeshService, + Name: "frontend", + }, + To: []v1alpha1.To{{ + TargetRef: common_api.TargetRef{ + Kind: common_api.MeshService, + Name: "backend", + }, + Rules: []v1alpha1.Rule{{ + Matches: []v1alpha1.Match{{ + Path: &v1alpha1.PathMatch{ + Type: v1alpha1.PathPrefix, + Value: "/v1", + }, + }}, + Default: v1alpha1.RuleConf{ + BackendRefs: &[]common_api.BackendRef{{ + TargetRef: common_api.TargetRef{ + Kind: common_api.MeshServiceSubset, + Name: "backend", + Tags: map[string]string{ + "version": "v1", + "region": "eu", + "kuma.io/zone": "zone", + }, + }, + }}, + }, + }}, + }}, + }, + }, + }, + }, + }, + }, + }, + }), Entry("with VirtualOutbound", testCase{ ingress: ` networking: diff --git a/pkg/xds/generator/testdata/ingress/with-meshhttproute-subset.envoy.golden.yaml b/pkg/xds/generator/testdata/ingress/with-meshhttproute-subset.envoy.golden.yaml new file mode 100644 index 000000000000..037b86ce06b1 --- /dev/null +++ b/pkg/xds/generator/testdata/ingress/with-meshhttproute-subset.envoy.golden.yaml @@ -0,0 +1,101 @@ +resources: +- name: mesh1:backend + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: mesh1_backend + connectTimeout: 10s + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + lbSubsetConfig: + fallbackPolicy: ANY_ENDPOINT + subsetSelectors: + - fallbackPolicy: NO_FALLBACK + keys: + - region + - version + name: mesh1:backend + type: EDS +- name: mesh1:backend + resource: + '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment + clusterName: mesh1:backend + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.0.1 + portValue: 2521 + loadBalancingWeight: 1 + metadata: + filterMetadata: + envoy.lb: + kuma.io/zone: zone + mesh: mesh1 + region: eu + version: v1 + envoy.transport_socket_match: + kuma.io/zone: zone + mesh: mesh1 + region: eu + version: v1 + - endpoint: + address: + socketAddress: + address: 192.168.0.2 + portValue: 2521 + loadBalancingWeight: 1 + metadata: + filterMetadata: + envoy.lb: + kuma.io/zone: zone + mesh: mesh1 + region: us + version: v2 + envoy.transport_socket_match: + kuma.io/zone: zone + mesh: mesh1 + region: us + version: v2 +- name: inbound:10.0.0.1:10001 + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 10.0.0.1 + portValue: 10001 + enableReusePort: false + filterChains: + - filterChainMatch: + serverNames: + - backend{kuma.io/zone=zone,mesh=mesh1,region=eu,version=v1} + transportProtocol: tls + filters: + - name: envoy.filters.network.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: mesh1:backend + metadataMatch: + filterMetadata: + envoy.lb: + region: eu + version: v1 + statPrefix: mesh1_backend + - filterChainMatch: + serverNames: + - backend{mesh=mesh1} + transportProtocol: tls + filters: + - name: envoy.filters.network.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: mesh1:backend + statPrefix: mesh1_backend + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + name: inbound:10.0.0.1:10001 + trafficDirection: INBOUND diff --git a/pkg/xds/generator/zoneproxy/generator.go b/pkg/xds/generator/zoneproxy/generator.go index d024026ccd02..0bc16fa1cc1e 100644 --- a/pkg/xds/generator/zoneproxy/generator.go +++ b/pkg/xds/generator/zoneproxy/generator.go @@ -13,21 +13,29 @@ import ( ) func GenerateCDS( - services []string, destinationsPerService map[string][]envoy_tags.Tags, + services envoy_common.Services, apiVersion core_xds.APIVersion, meshName string, origin string, ) ([]*core_xds.Resource, error) { + matchAllDestinations := destinationsPerService[mesh_proto.MatchAllTag] + var resources []*core_xds.Resource - for _, service := range services { - clusterName := envoy_names.GetMeshClusterName(meshName, service) + for _, service := range services.Sorted() { + clusters := services[service] + + var tagsSlice envoy_tags.TagsSlice + for _, cluster := range clusters.Clusters() { + tagsSlice = append(tagsSlice, cluster.Tags()) + } + tagSlice := append(tagsSlice, matchAllDestinations...) - tagSlice := envoy_tags.TagsSlice(append(destinationsPerService[service], destinationsPerService[mesh_proto.MatchAllTag]...)) tagKeySlice := tagSlice.ToTagKeysSlice().Transform( envoy_tags.Without(mesh_proto.ServiceTag), ) + clusterName := envoy_names.GetMeshClusterName(meshName, service) edsCluster, err := envoy_clusters.NewClusterBuilder(apiVersion, clusterName). Configure(envoy_clusters.EdsCluster()). Configure(envoy_clusters.LbSubset(tagKeySlice)). @@ -47,7 +55,7 @@ func GenerateCDS( } func GenerateEDS( - services []string, + services envoy_common.Services, endpointMap core_xds.EndpointMap, apiVersion core_xds.APIVersion, meshName string, @@ -55,10 +63,10 @@ func GenerateEDS( ) ([]*core_xds.Resource, error) { var resources []*core_xds.Resource - for _, service := range services { + for _, service := range services.Sorted() { endpoints := endpointMap[service] - clusterName := envoy_names.GetMeshClusterName(meshName, service) + clusterName := envoy_names.GetMeshClusterName(meshName, service) cla, err := envoy_endpoints.CreateClusterLoadAssignment(clusterName, endpoints, apiVersion) if err != nil { return nil, err @@ -88,7 +96,9 @@ func AddFilterChains( listenerBuilder *envoy_listeners.ListenerBuilder, destinationsPerService map[string][]envoy_tags.Tags, endpointMap core_xds.EndpointMap, -) { +) envoy_common.Services { + servicesAcc := envoy_common.NewServicesAccumulator(nil) + sniUsed := map[string]struct{}{} for _, service := range availableServices { serviceName := service.Tags[mesh_proto.ServiceTag] @@ -138,8 +148,10 @@ func AddFilterChains( cluster := envoy_common.NewCluster( envoy_common.WithName(clusterName), + envoy_common.WithService(serviceName), envoy_common.WithTags(relevantTags), ) + cluster.SetMesh(service.Mesh) filterChain := envoy_listeners.FilterChain( envoy_listeners.NewFilterChainBuilder(apiVersion, envoy_common.AnonymousResource).Configure( @@ -153,6 +165,10 @@ func AddFilterChains( ) listenerBuilder.Configure(filterChain) + + servicesAcc.Add(cluster) } } + + return servicesAcc.Services() } diff --git a/test/e2e_env/multizone/meshhttproute/test.go b/test/e2e_env/multizone/meshhttproute/test.go index 56e4ecbcbdee..39c74e2ce25e 100644 --- a/test/e2e_env/multizone/meshhttproute/test.go +++ b/test/e2e_env/multizone/meshhttproute/test.go @@ -34,14 +34,14 @@ func test(meshName string, meshBuilder *builders.MeshBuilder) { err := NewClusterSetup(). Install(DemoClientUniversal(AppModeDemoClient, meshName, WithTransparentProxy(true))). Install(TestServerUniversal("dp-echo-1", meshName, - WithArgs([]string{"echo", "--instance", "zone1"}), + WithArgs([]string{"echo", "--instance", "zone1-v1"}), WithServiceVersion("v1"), )). Setup(multizone.UniZone1) Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). - Install(TestServerUniversal("dp-echo-2", meshName, + Install(TestServerUniversal("dp-echo-2-v1", meshName, WithArgs([]string{"echo", "--instance", "zone2-v1"}), WithServiceVersion("v1"), )). @@ -49,13 +49,21 @@ func test(meshName string, meshBuilder *builders.MeshBuilder) { Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). - Install(TestServerUniversal("dp-echo-3", meshName, + Install(TestServerUniversal("dp-echo-2-v2", meshName, WithArgs([]string{"echo", "--instance", "zone2-v2"}), WithServiceVersion("v2"), )). Setup(multizone.UniZone2) Expect(err).ToNot(HaveOccurred()) + err = NewClusterSetup(). + Install(TestServerUniversal("dp-echo-2-v3", meshName, + WithArgs([]string{"echo", "--instance", "zone2-v3"}), + WithServiceVersion("v3"), + )). + Setup(multizone.UniZone2) + Expect(err).ToNot(HaveOccurred()) + err = NewClusterSetup(). Install(TestServerUniversal("dp-echo-4", meshName, WithArgs([]string{"echo", "--instance", "alias-zone2"}), @@ -142,11 +150,13 @@ spec: name: test-server weight: 1 tags: + kuma.io/zone: kuma-5 version: v1 - kind: MeshServiceSubset name: test-server weight: 1 tags: + kuma.io/zone: kuma-5 version: v2 `, meshName))(multizone.Global)).To(Succeed()) @@ -155,8 +165,9 @@ spec: g.Expect(err).ToNot(HaveOccurred()) g.Expect(response).To( And( - HaveKey(MatchRegexp(`^zone2-v1.*`)), + HaveKey(MatchRegexp(`^.*-v1.*`)), HaveKey(MatchRegexp(`^zone2-v2.*`)), + Not(HaveKey(MatchRegexp(`^zone2-v3.*`))), ), ) }, "30s", "500ms").Should(Succeed())