From 5fb579dafaf233b405c171b7dae630619d088203 Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Thu, 8 Feb 2024 12:56:58 +0100 Subject: [PATCH 1/2] test(e2e): upgrade KIC (#9157) Signed-off-by: Jakub Dyszkiewicz --- test/e2e_env/kubernetes/gateway/delegated.go | 8 +- .../gateway/delegated_meshproxypatch.go | 67 +++++++++++ test/e2e_env/kubernetes/kic/kong_ingress.go | 105 +++++++++++++----- .../kubernetes/kubernetes_suite_test.go | 4 +- test/framework/deployments/kic/deployment.go | 12 +- test/framework/deployments/kic/kubernetes.go | 57 ++++++++-- 6 files changed, 209 insertions(+), 44 deletions(-) create mode 100644 test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go diff --git a/test/e2e_env/kubernetes/gateway/delegated.go b/test/e2e_env/kubernetes/gateway/delegated.go index 23a33b509be9..e3830bd2947a 100644 --- a/test/e2e_env/kubernetes/gateway/delegated.go +++ b/test/e2e_env/kubernetes/gateway/delegated.go @@ -43,10 +43,14 @@ func Delegated() { testserver.WithName("test-server"), )). Install(kic.KongIngressController( + kic.WithName("delegated"), kic.WithNamespace(config.namespace), kic.WithMesh(config.mesh), )). - Install(kic.KongIngressService(kic.WithNamespace(config.namespace))). + Install(kic.KongIngressService( + kic.WithName("delegated"), + kic.WithNamespace(config.namespace), + )). Install(YamlK8s(fmt.Sprintf(` apiVersion: networking.k8s.io/v1 kind: Ingress @@ -54,7 +58,7 @@ metadata: namespace: %s name: %s-ingress annotations: - kubernetes.io/ingress.class: kong + kubernetes.io/ingress.class: delegated spec: rules: - http: diff --git a/test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go b/test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go new file mode 100644 index 000000000000..c738647a5158 --- /dev/null +++ b/test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go @@ -0,0 +1,67 @@ +package gateway + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/kumahq/kuma/test/framework" + "github.com/kumahq/kuma/test/framework/client" + "github.com/kumahq/kuma/test/framework/envs/kubernetes" + "github.com/kumahq/kuma/test/server/types" +) + +func MeshProxyPatch(config *delegatedE2EConfig) func() { + GinkgoHelper() + + return func() { + It("should add a header using Lua filter", func() { + // given + meshProxyPatch := fmt.Sprintf(` +apiVersion: kuma.io/v1alpha1 +kind: MeshProxyPatch +metadata: + name: backend-lua-filter + namespace: %s + labels: + kuma.io/mesh: %s +spec: + targetRef: + kind: Mesh + default: + appendModifications: + - httpFilter: + operation: AddBefore + match: + name: envoy.filters.http.router + origin: outbound + value: | + name: envoy.filters.http.lua + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + function envoy_on_request(request_handle) + request_handle:headers():add("X-Header", "test") + end +`, Config.KumaNamespace, config.mesh) + + // when + err := kubernetes.Cluster.Install(YamlK8s(meshProxyPatch)) + + // then + Expect(err).ToNot(HaveOccurred()) + Eventually(func() ([]types.EchoResponse, error) { + return client.CollectResponses( + kubernetes.Cluster, + "demo-client", + fmt.Sprintf("http://%s/test-server", config.kicIP), + client.FromKubernetesPod(config.namespaceOutsideMesh, "demo-client"), + ) + }, "30s", "1s").Should(ContainElement(HaveField( + `Received.Headers`, + HaveKeyWithValue("X-Header", ContainElement("test")), + ))) + }) + } +} diff --git a/test/e2e_env/kubernetes/kic/kong_ingress.go b/test/e2e_env/kubernetes/kic/kong_ingress.go index 4287651862fc..c3ef56d0f912 100644 --- a/test/e2e_env/kubernetes/kic/kong_ingress.go +++ b/test/e2e_env/kubernetes/kic/kong_ingress.go @@ -43,9 +43,13 @@ func KICKubernetes() { Install(democlient.Install(democlient.WithNamespace(namespaceOutsideMesh))). // this will not be in the mesh Install(kic.KongIngressController( kic.WithNamespace(namespace), + kic.WithName("kic"), kic.WithMesh(mesh), )). - Install(kic.KongIngressService(kic.WithNamespace(namespace))). + Install(kic.KongIngressService( + kic.WithNamespace(namespace), + kic.WithName("kic"), + )). Install(testserver.Install( testserver.WithNamespace(namespace), testserver.WithMesh(mesh), @@ -67,24 +71,48 @@ func KICKubernetes() { It("should route to service using Kube DNS", func() { ingress := ` -apiVersion: networking.k8s.io/v1 -kind: Ingress +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: kic + annotations: + konghq.com/gatewayclass-unmanaged: 'true' +spec: + controllerName: konghq.com/kic-gateway-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway metadata: + name: kong + namespace: kic +spec: + gatewayClassName: kic + listeners: + - name: proxy + port: 80 + protocol: HTTP +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: echo namespace: kic - name: kube-dns-ingress annotations: - kubernetes.io/ingress.class: kong + konghq.com/strip-path: 'true' spec: + parentRefs: + - name: kong + namespace: kic rules: - - http: - paths: - - path: /test-server - pathType: Prefix - backend: - service: - name: test-server - port: - number: 80 + - matches: + - path: + type: PathPrefix + value: /test-server + backendRefs: + - name: test-server + kind: Service + port: 80 ` Expect(kubernetes.Cluster.Install(YamlK8s(ingress))).To(Succeed()) @@ -100,6 +128,27 @@ spec: It("should route to service using Kuma DNS", func() { const ingressMeshDNS = ` --- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: echo + namespace: kic + annotations: + konghq.com/strip-path: 'true' +spec: + parentRefs: + - name: kong + namespace: kic + rules: + - matches: + - path: + type: PathPrefix + value: /test-server + backendRefs: + - name: test-server + kind: Service + port: 80 +--- apiVersion: v1 kind: Service metadata: @@ -109,24 +158,26 @@ spec: type: ExternalName externalName: test-server.kic.svc.80.mesh --- -apiVersion: networking.k8s.io/v1 -kind: Ingress +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute metadata: - namespace: kic name: k8s-ingress-dot-mesh + namespace: kic annotations: - kubernetes.io/ingress.class: kong + konghq.com/strip-path: 'true' spec: + parentRefs: + - name: kong + namespace: kic rules: - - http: - paths: - - path: /dot-mesh - pathType: Prefix - backend: - service: - name: test-server-externalname - port: - number: 80 + - matches: + - path: + type: PathPrefix + value: /dot-mesh + backendRefs: + - name: test-server-externalname + kind: Service + port: 80 ` Expect(kubernetes.Cluster.Install(YamlK8s(ingressMeshDNS))).To(Succeed()) diff --git a/test/e2e_env/kubernetes/kubernetes_suite_test.go b/test/e2e_env/kubernetes/kubernetes_suite_test.go index e958a6c87a01..cf7ad4b7661a 100644 --- a/test/e2e_env/kubernetes/kubernetes_suite_test.go +++ b/test/e2e_env/kubernetes/kubernetes_suite_test.go @@ -56,7 +56,7 @@ var ( _ = Describe("Gateway - Gateway API", gateway.GatewayAPI, Ordered) _ = Describe("Gateway - mTLS", gateway.Mtls, Ordered) _ = Describe("Gateway - Resources", gateway.Resources, Ordered) - _ = Describe("Delegated Gateway", Label("kind-not-supported", "ipv6-not-supported", "arm-not-supported"), gateway.Delegated, Ordered) + _ = Describe("Delegated Gateway", Label("kind-not-supported", "ipv6-not-supported"), gateway.Delegated, Ordered) _ = Describe("Graceful", graceful.Graceful, Ordered) _ = Describe("Eviction", graceful.Eviction, Ordered) _ = XDescribe("Change Service", graceful.ChangeService, Ordered) @@ -75,7 +75,7 @@ var ( _ = Describe("External Services Permissive MTLS", externalservices.PermissiveMTLS, Ordered) _ = Describe("ExternalName Services", externalname_services.ExternalNameServices, Ordered) _ = Describe("Virtual Outbound", virtualoutbound.VirtualOutbound, Ordered) - _ = Describe("Kong Ingress Controller", Label("arm-not-supported"), kic.KICKubernetes, Ordered) + _ = Describe("Kong Ingress Controller", kic.KICKubernetes, Ordered) _ = Describe("MeshTrafficPermission API", meshtrafficpermission.API, Ordered) _ = Describe("MeshRateLimit API", meshratelimit.API, Ordered) _ = Describe("MeshTimeout API", meshtimeout.MeshTimeout, Ordered) diff --git a/test/framework/deployments/kic/deployment.go b/test/framework/deployments/kic/deployment.go index 48bc0103d5c1..b2cf1d4086e5 100644 --- a/test/framework/deployments/kic/deployment.go +++ b/test/framework/deployments/kic/deployment.go @@ -22,6 +22,7 @@ type Deployment interface { type deployOptions struct { namespace string mesh string + name string } type deployOptionsFunc func(*deployOptions) @@ -47,6 +48,7 @@ func Install(fs ...deployOptionsFunc) framework.InstallFunc { deployment = &k8sDeployment{ ingressNamespace: opts.namespace, mesh: opts.mesh, + name: opts.name, } default: return errors.New("invalid cluster") @@ -67,6 +69,12 @@ func WithMesh(mesh string) deployOptionsFunc { } } +func WithName(name string) deployOptionsFunc { + return func(o *deployOptions) { + o.name = name + } +} + func KongIngressController(fs ...deployOptionsFunc) framework.InstallFunc { return Install(fs...) } @@ -85,7 +93,7 @@ metadata: spec: type: ClusterIP selector: - app: ingress-kong + app: %s-gateway ports: - name: proxy targetPort: 8000 @@ -94,5 +102,5 @@ spec: targetPort: 8443 port: 443 ` - return framework.YamlK8s(fmt.Sprintf(svc, opts.namespace)) + return framework.YamlK8s(fmt.Sprintf(svc, opts.namespace, opts.name)) } diff --git a/test/framework/deployments/kic/kubernetes.go b/test/framework/deployments/kic/kubernetes.go index cc782c8ab9bb..9f54aa313edc 100644 --- a/test/framework/deployments/kic/kubernetes.go +++ b/test/framework/deployments/kic/kubernetes.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/retry" "github.com/pkg/errors" @@ -16,37 +17,68 @@ import ( type k8sDeployment struct { ingressNamespace string mesh string + name string } var _ Deployment = &k8sDeployment{} -var ingressApp = "ingress-kong" - func (t *k8sDeployment) Name() string { return DeploymentName } func (t *k8sDeployment) Deploy(cluster framework.Cluster) error { - var yaml string var err error if t.ingressNamespace == "" { t.ingressNamespace = framework.Config.DefaultGatewayNamespace } - yaml, err = cluster.GetKumactlOptions().RunKumactlAndGetOutputV(framework.Verbose, - "install", "gateway", "kong", + opts := helm.Options{ + KubectlOptions: cluster.GetKubectlOptions(t.ingressNamespace), + } + _, err = helm.RunHelmCommandAndGetStdOutE(cluster.GetTesting(), &opts, "install", t.name, "--namespace", t.ingressNamespace, - "--mesh", t.mesh, + "--repo", "https://charts.konghq.com", + "--set", "controller.ingressController.ingressClass="+t.name, + "--set", "controller.podAnnotations.kuma\\.io/mesh="+t.mesh, + "--set", "gateway.podAnnotations.kuma\\.io/mesh="+t.mesh, + "ingress", ) if err != nil { return err } - err = k8s.KubectlApplyFromStringE(cluster.GetTesting(), - cluster.GetKubectlOptions(), - yaml) - if err != nil { - return err + for _, app := range []string{fmt.Sprintf("%s-controller", t.name), fmt.Sprintf("%s-gateway", t.name)} { + err := k8s.WaitUntilNumPodsCreatedE(cluster.GetTesting(), + cluster.GetKubectlOptions(t.ingressNamespace), + metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", app), + }, + 1, + framework.DefaultRetries, + framework.DefaultTimeout) + if err != nil { + return err + } + + pods := k8s.ListPods(cluster.GetTesting(), + cluster.GetKubectlOptions(t.ingressNamespace), + metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", app), + }, + ) + if len(pods) != 1 { + return errors.Errorf("counting KIC pods. Got: %d. Expected: 1", len(pods)) + } + + err = k8s.WaitUntilPodAvailableE(cluster.GetTesting(), + cluster.GetKubectlOptions(t.ingressNamespace), + pods[0].Name, + framework.DefaultRetries*3, // KIC is fetched from the internet. Increase the timeout to prevent long downloads of images. + framework.DefaultTimeout) + if err != nil { + return err + } } +<<<<<<< HEAD k8s.WaitUntilNumPodsCreated(cluster.GetTesting(), cluster.GetKubectlOptions(t.ingressNamespace), @@ -72,6 +104,9 @@ func (t *k8sDeployment) Deploy(cluster framework.Cluster) error { pods[0].Name, framework.DefaultRetries, framework.DefaultTimeout) +======= + return nil +>>>>>>> 6cf0b3eea (test(e2e): upgrade KIC (#9157)) } func (t *k8sDeployment) Delete(cluster framework.Cluster) error { From f3a1da8f648c73ad2a77883b238a67bc016bd2b0 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Thu, 19 Dec 2024 16:56:39 +0100 Subject: [PATCH 2/2] conflicts Signed-off-by: Lukasz Dziedziak --- .../gateway/delegated_meshproxypatch.go | 67 ------------------- test/framework/deployments/kic/kubernetes.go | 28 -------- 2 files changed, 95 deletions(-) delete mode 100644 test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go diff --git a/test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go b/test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go deleted file mode 100644 index c738647a5158..000000000000 --- a/test/e2e_env/kubernetes/gateway/delegated_meshproxypatch.go +++ /dev/null @@ -1,67 +0,0 @@ -package gateway - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/kumahq/kuma/test/framework" - "github.com/kumahq/kuma/test/framework/client" - "github.com/kumahq/kuma/test/framework/envs/kubernetes" - "github.com/kumahq/kuma/test/server/types" -) - -func MeshProxyPatch(config *delegatedE2EConfig) func() { - GinkgoHelper() - - return func() { - It("should add a header using Lua filter", func() { - // given - meshProxyPatch := fmt.Sprintf(` -apiVersion: kuma.io/v1alpha1 -kind: MeshProxyPatch -metadata: - name: backend-lua-filter - namespace: %s - labels: - kuma.io/mesh: %s -spec: - targetRef: - kind: Mesh - default: - appendModifications: - - httpFilter: - operation: AddBefore - match: - name: envoy.filters.http.router - origin: outbound - value: | - name: envoy.filters.http.lua - typedConfig: - '@type': type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua - inline_code: | - function envoy_on_request(request_handle) - request_handle:headers():add("X-Header", "test") - end -`, Config.KumaNamespace, config.mesh) - - // when - err := kubernetes.Cluster.Install(YamlK8s(meshProxyPatch)) - - // then - Expect(err).ToNot(HaveOccurred()) - Eventually(func() ([]types.EchoResponse, error) { - return client.CollectResponses( - kubernetes.Cluster, - "demo-client", - fmt.Sprintf("http://%s/test-server", config.kicIP), - client.FromKubernetesPod(config.namespaceOutsideMesh, "demo-client"), - ) - }, "30s", "1s").Should(ContainElement(HaveField( - `Received.Headers`, - HaveKeyWithValue("X-Header", ContainElement("test")), - ))) - }) - } -} diff --git a/test/framework/deployments/kic/kubernetes.go b/test/framework/deployments/kic/kubernetes.go index 9f54aa313edc..c30d6c64168b 100644 --- a/test/framework/deployments/kic/kubernetes.go +++ b/test/framework/deployments/kic/kubernetes.go @@ -78,35 +78,7 @@ func (t *k8sDeployment) Deploy(cluster framework.Cluster) error { return err } } -<<<<<<< HEAD - - k8s.WaitUntilNumPodsCreated(cluster.GetTesting(), - cluster.GetKubectlOptions(t.ingressNamespace), - metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s", ingressApp), - }, - 1, - framework.DefaultRetries, - framework.DefaultTimeout) - - pods := k8s.ListPods(cluster.GetTesting(), - cluster.GetKubectlOptions(t.ingressNamespace), - metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s", ingressApp), - }, - ) - if len(pods) != 1 { - return errors.Errorf("counting KIC pods. Got: %d. Expected: 1", len(pods)) - } - - return k8s.WaitUntilPodAvailableE(cluster.GetTesting(), - cluster.GetKubectlOptions(t.ingressNamespace), - pods[0].Name, - framework.DefaultRetries, - framework.DefaultTimeout) -======= return nil ->>>>>>> 6cf0b3eea (test(e2e): upgrade KIC (#9157)) } func (t *k8sDeployment) Delete(cluster framework.Cluster) error {