From a41fd62549beb0203c1aabfc4b4e239511c03320 Mon Sep 17 00:00:00 2001 From: Jay Chen <1180092+jijiechen@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:54:33 +0800 Subject: [PATCH 1/7] feat(bootstrap): support corefile template configured on cp Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- pkg/config/xds/bootstrap/config.go | 19 +++++-- pkg/config/xds/bootstrap/config_test.go | 3 + .../testdata/default-config.golden.yaml | 1 + .../testdata/valid-config.input.yaml | 1 + pkg/xds/bootstrap/generator.go | 7 +++ pkg/xds/bootstrap/generator_test.go | 57 +++++++++++++++---- pkg/xds/bootstrap/handler.go | 1 + pkg/xds/bootstrap/parameters.go | 1 + pkg/xds/bootstrap/testdata/corefile.template | 19 +++++++ pkg/xds/bootstrap/types/bootstrap_response.go | 3 +- 10 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 pkg/xds/bootstrap/testdata/corefile.template diff --git a/pkg/config/xds/bootstrap/config.go b/pkg/config/xds/bootstrap/config.go index 6ab6d3eab7b1..d782af55f5df 100644 --- a/pkg/config/xds/bootstrap/config.go +++ b/pkg/config/xds/bootstrap/config.go @@ -1,6 +1,7 @@ package bootstrap import ( + "github.com/kumahq/kuma/pkg/util/files" "net" "os" "time" @@ -57,6 +58,8 @@ type BootstrapParamsConfig struct { XdsPort uint32 `json:"xdsPort" envconfig:"kuma_bootstrap_server_params_xds_port"` // Connection timeout to the XDS Server XdsConnectTimeout config_types.Duration `json:"xdsConnectTimeout" envconfig:"kuma_bootstrap_server_params_xds_connect_timeout"` + // Path to the template of Corefile for data planes to use + CorefileTemplatePath string `json:"corefileTemplatePath" envconfig:"kuma_bootstrap_server_params_corefile_template_path"` } func (b *BootstrapParamsConfig) Validate() error { @@ -78,16 +81,20 @@ func (b *BootstrapParamsConfig) Validate() error { if b.XdsConnectTimeout.Duration < 0 { return errors.New("XdsConnectTimeout cannot be negative") } + if b.CorefileTemplatePath != "" && !files.FileExists(b.CorefileTemplatePath) { + return errors.New("CorefileTemplatePath must point to an existing file") + } return nil } func DefaultBootstrapParamsConfig() *BootstrapParamsConfig { return &BootstrapParamsConfig{ - AdminAddress: "127.0.0.1", // by default, Envoy Admin interface should listen on loopback address - AdminPort: 9901, - AdminAccessLogPath: os.DevNull, - XdsHost: "", // by default, it is the same host as the one used by kuma-dp to connect to the control plane - XdsPort: 0, // by default, it is autoconfigured from KUMA_XDS_SERVER_GRPC_PORT - XdsConnectTimeout: config_types.Duration{Duration: 1 * time.Second}, + AdminAddress: "127.0.0.1", // by default, Envoy Admin interface should listen on loopback address + AdminPort: 9901, + AdminAccessLogPath: os.DevNull, + XdsHost: "", // by default, it is the same host as the one used by kuma-dp to connect to the control plane + XdsPort: 0, // by default, it is autoconfigured from KUMA_XDS_SERVER_GRPC_PORT + XdsConnectTimeout: config_types.Duration{Duration: 1 * time.Second}, + CorefileTemplatePath: "", // by default, data plane will use the embedded Corefile to be the template } } diff --git a/pkg/config/xds/bootstrap/config_test.go b/pkg/config/xds/bootstrap/config_test.go index b1986864ee9f..19b3ca353b62 100644 --- a/pkg/config/xds/bootstrap/config_test.go +++ b/pkg/config/xds/bootstrap/config_test.go @@ -17,6 +17,8 @@ var _ = Describe("BootstrappServerConfig", func() { It("should be loadable from configuration file", func() { // given cfg := BootstrapServerConfig{} + fileError := os.WriteFile("/tmp/corefile", []byte("abc"), 0o600) + Expect(fileError).ToNot(HaveOccurred()) // when err := config.Load(filepath.Join("testdata", "valid-config.input.yaml"), &cfg) @@ -31,6 +33,7 @@ var _ = Describe("BootstrappServerConfig", func() { Expect(cfg.Params.XdsHost).To(Equal("kuma-control-plane.internal")) Expect(cfg.Params.XdsPort).To(Equal(uint32(10101))) Expect(cfg.Params.XdsConnectTimeout.Duration).To(Equal(2 * time.Second)) + Expect(cfg.Params.CorefileTemplatePath).To(Equal("/tmp/corefile")) }) Context("with modified environment variables", func() { diff --git a/pkg/config/xds/bootstrap/testdata/default-config.golden.yaml b/pkg/config/xds/bootstrap/testdata/default-config.golden.yaml index 8e76549396d3..ce314fe33408 100644 --- a/pkg/config/xds/bootstrap/testdata/default-config.golden.yaml +++ b/pkg/config/xds/bootstrap/testdata/default-config.golden.yaml @@ -5,3 +5,4 @@ params: xdsConnectTimeout: 1s xdsHost: "" xdsPort: 0 + corefileTemplatePath: "" diff --git a/pkg/config/xds/bootstrap/testdata/valid-config.input.yaml b/pkg/config/xds/bootstrap/testdata/valid-config.input.yaml index f98665564d3c..503938259ce7 100644 --- a/pkg/config/xds/bootstrap/testdata/valid-config.input.yaml +++ b/pkg/config/xds/bootstrap/testdata/valid-config.input.yaml @@ -6,5 +6,6 @@ params: xdsHost: kuma-control-plane.internal xdsPort: 10101 xdsConnectTimeout: 2s + corefileTemplatePath: /tmp/corefile tlsCertFile: "" tlsKeyFile: "" diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 73aef1a40415..1ee6e4e8bf3e 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -161,6 +161,13 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr params.IsGatewayDataplane = true } kumaDpBootstrap.NetworkingConfig.IsUsingTransparentProxy = dataplane.IsUsingTransparentProxy() + if b.config.Params.CorefileTemplatePath != "" { + corefileTemplate, err := os.ReadFile(b.config.Params.CorefileTemplatePath) + if err != nil { + return nil, kumaDpBootstrap, errors.Wrap(err, "could not read Corefile template") + } + kumaDpBootstrap.NetworkingConfig.CorefileTemplate = corefileTemplate + } params.Service = dataplane.Spec.GetIdentifyingService() setAdminPort(dataplane.Spec.GetNetworking().GetAdmin().GetPort()) diff --git a/pkg/xds/bootstrap/generator_test.go b/pkg/xds/bootstrap/generator_test.go index 11a068d7d69d..d8a2079588e2 100644 --- a/pkg/xds/bootstrap/generator_test.go +++ b/pkg/xds/bootstrap/generator_test.go @@ -96,14 +96,15 @@ var _ = Describe("bootstrapGenerator", func() { }) type testCase struct { - serverConfig *bootstrap_config.BootstrapServerConfig - proxyConfig *xds_config.Proxy - dataplane func() *core_mesh.DataplaneResource - dpAuthForProxyType map[string]bool - useTokenPath bool - request types.BootstrapRequest - expectedConfigFile string - hdsEnabled bool + serverConfig *bootstrap_config.BootstrapServerConfig + proxyConfig *xds_config.Proxy + dataplane func() *core_mesh.DataplaneResource + dpAuthForProxyType map[string]bool + useTokenPath bool + request types.BootstrapRequest + expectedConfigFile string + dpBootstrapVerifier func(KumaDpBootstrap) + hdsEnabled bool } DescribeTable("should generate bootstrap configuration", func(given testCase) { @@ -120,7 +121,7 @@ var _ = Describe("bootstrapGenerator", func() { Expect(err).ToNot(HaveOccurred()) // when - bootstrapConfig, _, err := generator.Generate(context.Background(), given.request) + bootstrapConfig, dpBootstrap, err := generator.Generate(context.Background(), given.request) // then Expect(err).ToNot(HaveOccurred()) @@ -128,7 +129,12 @@ var _ = Describe("bootstrapGenerator", func() { // and config is as expected actual, err := util_proto.ToYAML(bootstrapConfig) Expect(err).ToNot(HaveOccurred()) - Expect(actual).To(MatchGoldenYAML(filepath.Join("testdata", given.expectedConfigFile))) + if given.expectedConfigFile != "" { + Expect(actual).To(MatchGoldenYAML(filepath.Join("testdata", given.expectedConfigFile))) + } + if given.dpBootstrapVerifier != nil { + given.dpBootstrapVerifier(dpBootstrap) + } }, Entry("default config with minimal request", testCase{ dpAuthForProxyType: map[string]bool{}, @@ -423,6 +429,37 @@ var _ = Describe("bootstrapGenerator", func() { hdsEnabled: true, useTokenPath: true, }), + Entry("dns corefile template", testCase{ + dpAuthForProxyType: map[string]bool{}, + serverConfig: func() *bootstrap_config.BootstrapServerConfig { + return &bootstrap_config.BootstrapServerConfig{ + Params: &bootstrap_config.BootstrapParamsConfig{ + AdminAddress: "192.168.0.1", // by default, Envoy Admin interface should listen on loopback address + AdminAccessLogPath: "/var/log", + XdsHost: "localhost", + XdsPort: 15678, + XdsConnectTimeout: config_types.Duration{Duration: 2 * time.Second}, + CorefileTemplatePath: filepath.Join("testdata", "corefile.template"), + }, + } + }(), + dataplane: func() *core_mesh.DataplaneResource { + dp := defaultDataplane() + dp.Spec.Networking.Admin.Port = 9902 + return dp + }, + request: types.BootstrapRequest{ + Mesh: "mesh", + Name: "name.namespace", + Version: defaultVersion, + }, + dpBootstrapVerifier: func(dpBootstrap KumaDpBootstrap) { + expected, err := os.ReadFile(filepath.Join("testdata", "corefile.template")) + Expect(err).ToNot(HaveOccurred()) + Expect(dpBootstrap.NetworkingConfig.CorefileTemplate).To(Equal(expected)) + }, + hdsEnabled: true, + }), ) type errTestCase struct { diff --git a/pkg/xds/bootstrap/handler.go b/pkg/xds/bootstrap/handler.go index 098254b1e07b..d9ebe1ec8253 100644 --- a/pkg/xds/bootstrap/handler.go +++ b/pkg/xds/bootstrap/handler.go @@ -129,6 +129,7 @@ func createBootstrapResponse(bootstrap []byte, config *KumaDpBootstrap) *types.B }, Networking: types.NetworkingConfiguration{ IsUsingTransparentProxy: config.NetworkingConfig.IsUsingTransparentProxy, + CorefileTemplate: config.NetworkingConfig.CorefileTemplate, }, } return &bootstrapConfig diff --git a/pkg/xds/bootstrap/parameters.go b/pkg/xds/bootstrap/parameters.go index 2cd0a950dc3a..6e9171abdfde 100644 --- a/pkg/xds/bootstrap/parameters.go +++ b/pkg/xds/bootstrap/parameters.go @@ -14,6 +14,7 @@ type KumaDpBootstrap struct { type NetworkingConfig struct { IsUsingTransparentProxy bool + CorefileTemplate []byte } type AggregateMetricsConfig struct { diff --git a/pkg/xds/bootstrap/testdata/corefile.template b/pkg/xds/bootstrap/testdata/corefile.template new file mode 100644 index 000000000000..35ba34ec7ad9 --- /dev/null +++ b/pkg/xds/bootstrap/testdata/corefile.template @@ -0,0 +1,19 @@ +.:{{ .CoreDNSPort }} { + template IN AAAA . { + rcode NOERROR + fallthrough + } + forward . 127.0.0.1:{{ .EnvoyDNSPort }} + # We want all requests to be sent to the Envoy DNS Filter, unsuccessful responses should be forwarded to the original DNS server. + # For example: requests other than A, AAAA and SRV will return NOTIMP when hitting the envoy filter and should be sent to the original DNS server. + # Codes from: https://github.com/miekg/dns/blob/master/msg.go#L138 + alternate NOTIMP,FORMERR,NXDOMAIN,SERVFAIL,REFUSED . /etc/resolv.conf + prometheus localhost:{{ .PrometheusPort }} + errors +} + +.:{{ .CoreDNSEmptyPort }} { + template ANY ANY . { + rcode NXDOMAIN + } +} \ No newline at end of file diff --git a/pkg/xds/bootstrap/types/bootstrap_response.go b/pkg/xds/bootstrap/types/bootstrap_response.go index 785897bdd1d4..af33504552c1 100644 --- a/pkg/xds/bootstrap/types/bootstrap_response.go +++ b/pkg/xds/bootstrap/types/bootstrap_response.go @@ -22,7 +22,8 @@ type KumaSidecarConfiguration struct { } type NetworkingConfiguration struct { - IsUsingTransparentProxy bool `json:"isUsingTransparentProxy"` + IsUsingTransparentProxy bool `json:"isUsingTransparentProxy"` + CorefileTemplate []byte `json:"corefileTemplate"` } type MetricsConfiguration struct { From 5e887164b773c4c0df04df1d71c45fd7c3a176f2 Mon Sep 17 00:00:00 2001 From: Jay Chen <1180092+jijiechen@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:14:17 +0800 Subject: [PATCH 2/7] feat(bootstrap): support corefile template pushed from the cp on dp Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- app/kuma-dp/cmd/run.go | 69 +++++++---- app/kuma-dp/cmd/run_test.go | 109 ++++++++++++------ .../cmd/testdata/coredns-mock.sleep.sh | 5 + .../dnsserver/{config_file.go => corefile.go} | 4 +- .../{config_file_test.go => corefile_test.go} | 6 +- .../pkg/dataplane/dnsserver/dnsserver.go | 2 +- 6 files changed, 132 insertions(+), 63 deletions(-) rename app/kuma-dp/pkg/dataplane/dnsserver/{config_file.go => corefile.go} (77%) rename app/kuma-dp/pkg/dataplane/dnsserver/{config_file_test.go => corefile_test.go} (86%) diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index efbf4e371f27..d0ee8ea4ba62 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver" "os" "path/filepath" "time" @@ -11,7 +12,6 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/accesslogs" - "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics" kuma_cmd "github.com/kumahq/kuma/pkg/cmd" @@ -166,29 +166,6 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { OnFinish: cancelComponents, } - if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() { - dnsOpts := &dnsserver.Opts{ - Config: *cfg, - Stdout: cmd.OutOrStdout(), - Stderr: cmd.OutOrStderr(), - OnFinish: cancelComponents, - } - - dnsServer, err := dnsserver.New(dnsOpts) - if err != nil { - return err - } - - version, err := dnsServer.GetVersion() - if err != nil { - return err - } - - rootCtx.BootstrapDynamicMetadata[core_xds.FieldPrefixDependenciesVersion+".coredns"] = version - - components = append(components, dnsServer) - } - envoyVersion, err := envoy.GetEnvoyVersion(opts.Config.DataplaneRuntime.BinaryPath) if err != nil { return errors.Wrap(err, "failed to get Envoy version") @@ -226,6 +203,37 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { } opts.AdminPort = bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue() + if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() { + if cfg.DNS.CoreDNSConfigTemplatePath == "" && len(kumaSidecarConfiguration.Networking.CorefileTemplate) > 0 { + templateFile, err := writeTempFile("kuma-dp-corefile-template-", "Corefile", kumaSidecarConfiguration.Networking.CorefileTemplate, 0644) + if err != nil { + return errors.Errorf("Failed to write corefile template to disk. %v", err) + } + cfg.DNS.CoreDNSConfigTemplatePath = templateFile + } + + dnsOpts := &dnsserver.Opts{ + Config: *cfg, + Stdout: cmd.OutOrStdout(), + Stderr: cmd.OutOrStderr(), + OnFinish: cancelComponents, + } + + dnsServer, err := dnsserver.New(dnsOpts) + if err != nil { + return err + } + + version, err := dnsServer.GetVersion() + if err != nil { + return err + } + + rootCtx.BootstrapDynamicMetadata[core_xds.FieldPrefixDependenciesVersion+".coredns"] = version + + components = append(components, dnsServer) + } + envoyComponent, err := envoy.New(opts) if err != nil { return err @@ -336,3 +344,16 @@ func writeFile(filename string, data []byte, perm os.FileMode) error { } return os.WriteFile(filename, data, perm) } + +func writeTempFile(dirPattern, filename string, data []byte, perm os.FileMode) (string, error) { + path, err := os.MkdirTemp("", dirPattern) + fullFileName := filepath.Join(path, filename) + if err != nil { + return "", errors.Errorf("Failed to create a containing directory for %s. %v", filename, err) + } + err = os.WriteFile(fullFileName, data, perm) + if err != nil { + return "", errors.Errorf("Failed to write %s to disk. %v", filename, err) + } + return fullFileName, nil +} diff --git a/app/kuma-dp/cmd/run_test.go b/app/kuma-dp/cmd/run_test.go index 58810d14451c..051a3e25b38f 100644 --- a/app/kuma-dp/cmd/run_test.go +++ b/app/kuma-dp/cmd/run_test.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "fmt" "io" "os" "path/filepath" @@ -72,13 +73,17 @@ var _ = Describe("run", func() { given := givenFunc() // setup - pidFile := filepath.Join(tmpDir, "envoy-mock.pid") - cmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline") + envoyPidFile := filepath.Join(tmpDir, "envoy-mock.pid") + envoyCmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline") + corednsPidFile := filepath.Join(tmpDir, "coredns-mock.pid") + corednsCmdlineFile := filepath.Join(tmpDir, "coredns-mock.cmdline") // and env := given.envVars - env["ENVOY_MOCK_PID_FILE"] = pidFile - env["ENVOY_MOCK_CMDLINE_FILE"] = cmdlineFile + env["ENVOY_MOCK_PID_FILE"] = envoyPidFile + env["ENVOY_MOCK_CMDLINE_FILE"] = envoyCmdlineFile + env["COREDNS_MOCK_PID_FILE"] = corednsPidFile + env["COREDNS_MOCK_CMDLINE_FILE"] = corednsCmdlineFile for key, value := range env { Expect(os.Setenv(key, value)).To(Succeed()) } @@ -116,35 +121,22 @@ var _ = Describe("run", func() { }() // then - var pid int64 - By("waiting for dataplane (Envoy) to get started") - Eventually(func() bool { - data, err := os.ReadFile(pidFile) - if err != nil { - return false + var actualConfigFile string + envoyPid := verifyComponentProcess("Envoy", envoyPidFile, envoyCmdlineFile, func(actualArgs []string) { + Expect(actualArgs[0]).To(Equal("--version")) + Expect(actualArgs[1]).To(Equal("--config-path")) + actualConfigFile = actualArgs[2] + Expect(actualConfigFile).To(BeARegularFile()) + if given.expectedFile != "" { + Expect(actualArgs[2]).To(Equal(given.expectedFile)) } - pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32) - return err == nil - }, "5s", "100ms").Should(BeTrue()) - // and - Expect(pid).ToNot(BeZero()) - - By("verifying the arguments Envoy was launched with") - // when - cmdline, err := os.ReadFile(cmdlineFile) - // then - Expect(err).ToNot(HaveOccurred()) - // and - actualArgs := strings.Split(string(cmdline), "\n") - Expect(actualArgs[0]).To(Equal("--version")) - Expect(actualArgs[1]).To(Equal("--config-path")) - actualConfigFile := actualArgs[2] - Expect(actualConfigFile).To(BeARegularFile()) + }) - // then - if given.expectedFile != "" { - Expect(actualArgs[2]).To(Equal(given.expectedFile)) - } + corednsPid := verifyComponentProcess("coredns", corednsPidFile, corednsCmdlineFile, func(actualArgs []string) { + Expect(len(actualArgs)).To(Equal(3)) + Expect(actualArgs[0]).To(Equal("-conf")) + Expect(actualArgs[2]).To(Equal("-quiet")) + }) // when By("signaling the dataplane manager to stop") @@ -153,13 +145,20 @@ var _ = Describe("run", func() { cancel() // then - err = <-errCh + err := <-errCh Expect(err).ToNot(HaveOccurred()) By("waiting for dataplane (Envoy) to get stopped") Eventually(func() bool { // send sig 0 to check whether Envoy process still exists - err := syscall.Kill(int(pid), syscall.Signal(0)) + err := syscall.Kill(int(envoyPid), syscall.Signal(0)) + // we expect Envoy process to get killed by now + return err != nil + }, "5s", "100ms").Should(BeTrue()) + By("waiting for dataplane (coredns) to get stopped") + Eventually(func() bool { + // send sig 0 to check whether Envoy process still exists + err := syscall.Kill(int(corednsPid), syscall.Signal(0)) // we expect Envoy process to get killed by now return err != nil }, "5s", "100ms").Should(BeTrue()) @@ -288,6 +287,23 @@ var _ = Describe("run", func() { expectedFile: "", } }), + Entry("can be launched with given coredns configuration path", func() testCase { + corefileTemplate := filepath.Join(tmpDir, "Corefile") + _ = os.WriteFile(corefileTemplate, []byte("abcd"), 0o600) + return testCase{ + envVars: map[string]string{ + "KUMA_CONTROL_PLANE_API_SERVER_URL": "http://localhost:1234", + "KUMA_DATAPLANE_NAME": "example", + "KUMA_DATAPLANE_MESH": "default", + "KUMA_DATAPLANE_RUNTIME_BINARY_PATH": filepath.Join("testdata", "envoy-mock.sleep.sh"), + "KUMA_DATAPLANE_RUNTIME_CONFIG_DIR": tmpDir, + "KUMA_DNS_CORE_DNS_BINARY_PATH": filepath.Join("testdata", "coredns-mock.sleep.sh"), + "KUMA_DNS_CORE_DNS_CONFIG_TEMPLATE_PATH": corefileTemplate, + }, + args: []string{}, + expectedFile: filepath.Join(tmpDir, "bootstrap.yaml"), + } + }), ) It("should fail when name and mesh is provided with dataplane definition", func() { @@ -333,3 +349,30 @@ var _ = Describe("run", func() { Expect(err.Error()).To(ContainSubstring("invalid proxy type")) }) }, Ordered) + +func verifyComponentProcess(processDescription, pidfile string, cmdlinefile string, argsVerifier func(expectedArgs []string)) int64 { + var pid int64 + By(fmt.Sprintf("waiting for dataplane (%s) to get started", processDescription)) + Eventually(func() bool { + data, err := os.ReadFile(pidfile) + if err != nil { + return false + } + pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32) + return err == nil + }, "5s", "100ms").Should(BeTrue()) + // and + Expect(pid).ToNot(BeZero()) + + By(fmt.Sprintf("verifying the arguments %s was launched with", processDescription)) + // when + cmdline, err := os.ReadFile(cmdlinefile) + // then + Expect(err).ToNot(HaveOccurred()) + // and + if argsVerifier != nil { + actualArgs := strings.Split(string(cmdline), "\n") + argsVerifier(actualArgs) + } + return pid +} diff --git a/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh b/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh index b61273daa00f..08dea44e0247 100755 --- a/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh +++ b/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh @@ -6,6 +6,11 @@ then exit 0 fi +echo $$ >"${COREDNS_MOCK_PID_FILE}" +for arg in "$@"; do + echo "$arg" >> "${ENVOY_MOCK_CMDLINE_FILE}" +done + # Send logs for Cmd#Wait to finish while true; do diff --git a/app/kuma-dp/pkg/dataplane/dnsserver/config_file.go b/app/kuma-dp/pkg/dataplane/dnsserver/corefile.go similarity index 77% rename from app/kuma-dp/pkg/dataplane/dnsserver/config_file.go rename to app/kuma-dp/pkg/dataplane/dnsserver/corefile.go index cae9d6049617..b45cde93e481 100644 --- a/app/kuma-dp/pkg/dataplane/dnsserver/config_file.go +++ b/app/kuma-dp/pkg/dataplane/dnsserver/corefile.go @@ -13,10 +13,10 @@ import ( //go:embed Corefile var config embed.FS -func GenerateConfigFile(cfg kuma_dp.DNS, config []byte) (string, error) { +func GenerateCorefile(cfg kuma_dp.DNS, config []byte) (string, error) { configFile := filepath.Join(cfg.ConfigDir, "Corefile") if err := writeFile(configFile, config, 0o600); err != nil { - return "", errors.Wrap(err, "failed to persist Envoy bootstrap config on disk") + return "", errors.Wrap(err, "failed to persist coredns Corefile on disk") } return configFile, nil } diff --git a/app/kuma-dp/pkg/dataplane/dnsserver/config_file_test.go b/app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go similarity index 86% rename from app/kuma-dp/pkg/dataplane/dnsserver/config_file_test.go rename to app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go index b9ca7224cb94..fdc424345f19 100644 --- a/app/kuma-dp/pkg/dataplane/dnsserver/config_file_test.go +++ b/app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go @@ -10,8 +10,8 @@ import ( kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp" ) -var _ = Describe("Config File", func() { - Describe("GenerateConfigFile(..)", func() { +var _ = Describe("Corefile", func() { + Describe("GenerateCorefile(..)", func() { var configDir string BeforeEach(func() { @@ -39,7 +39,7 @@ var _ = Describe("Config File", func() { } // when - filename, err := GenerateConfigFile(dnsConfig, []byte(config)) + filename, err := GenerateCorefile(dnsConfig, []byte(config)) // then Expect(err).ToNot(HaveOccurred()) // and diff --git a/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go b/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go index d7aaca1d55fe..e2430bdaf164 100644 --- a/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go +++ b/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go @@ -117,7 +117,7 @@ func (s *DNSServer) Start(stop <-chan struct{}) error { return err } - configFile, err := GenerateConfigFile(dnsConfig, bs.Bytes()) + configFile, err := GenerateCorefile(dnsConfig, bs.Bytes()) if err != nil { return err } From e44287bbc2a45053a2fe9b97c34ac22919d82c87 Mon Sep 17 00:00:00 2001 From: Jay Chen <1180092+jijiechen@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:51:25 +0800 Subject: [PATCH 3/7] feat(bootstrap): fix failed unit tests Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- app/kuma-dp/cmd/run.go | 4 ++-- app/kuma-dp/cmd/run_test.go | 8 +++++--- app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh | 2 +- pkg/config/loader_test.go | 13 +++++++------ pkg/config/xds/bootstrap/config.go | 2 +- pkg/config/xds/bootstrap/config_test.go | 1 + 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index d0ee8ea4ba62..7efe0c484546 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver" "os" "path/filepath" "time" @@ -12,6 +11,7 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/accesslogs" + "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics" kuma_cmd "github.com/kumahq/kuma/pkg/cmd" @@ -205,7 +205,7 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() { if cfg.DNS.CoreDNSConfigTemplatePath == "" && len(kumaSidecarConfiguration.Networking.CorefileTemplate) > 0 { - templateFile, err := writeTempFile("kuma-dp-corefile-template-", "Corefile", kumaSidecarConfiguration.Networking.CorefileTemplate, 0644) + templateFile, err := writeTempFile("kuma-dp-corefile-template-", "Corefile", kumaSidecarConfiguration.Networking.CorefileTemplate, 0o644) if err != nil { return errors.Errorf("Failed to write corefile template to disk. %v", err) } diff --git a/app/kuma-dp/cmd/run_test.go b/app/kuma-dp/cmd/run_test.go index 051a3e25b38f..098fbc6fd72f 100644 --- a/app/kuma-dp/cmd/run_test.go +++ b/app/kuma-dp/cmd/run_test.go @@ -133,7 +133,7 @@ var _ = Describe("run", func() { }) corednsPid := verifyComponentProcess("coredns", corednsPidFile, corednsCmdlineFile, func(actualArgs []string) { - Expect(len(actualArgs)).To(Equal(3)) + Expect(len(actualArgs)).To(HaveLen(3)) Expect(actualArgs[0]).To(Equal("-conf")) Expect(actualArgs[2]).To(Equal("-quiet")) }) @@ -361,17 +361,19 @@ func verifyComponentProcess(processDescription, pidfile string, cmdlinefile stri pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32) return err == nil }, "5s", "100ms").Should(BeTrue()) - // and Expect(pid).ToNot(BeZero()) By(fmt.Sprintf("verifying the arguments %s was launched with", processDescription)) // when cmdline, err := os.ReadFile(cmdlinefile) + // then Expect(err).ToNot(HaveOccurred()) // and if argsVerifier != nil { - actualArgs := strings.Split(string(cmdline), "\n") + actualArgs := strings.FieldsFunc(string(cmdline), func(c rune) bool { + return c == '\n' + }) argsVerifier(actualArgs) } return pid diff --git a/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh b/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh index 08dea44e0247..66fe78d98412 100755 --- a/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh +++ b/app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh @@ -8,7 +8,7 @@ fi echo $$ >"${COREDNS_MOCK_PID_FILE}" for arg in "$@"; do - echo "$arg" >> "${ENVOY_MOCK_CMDLINE_FILE}" + echo "$arg" >> "${COREDNS_MOCK_CMDLINE_FILE}" done # Send logs for Cmd#Wait to finish diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 57f532673589..1da7c554abfe 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -717,12 +717,13 @@ tracing: }), Entry("from env variables", testCase{ envVars: map[string]string{ - "KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_PORT": "1234", - "KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_HOST": "kuma-control-plane", - "KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_PORT": "4321", - "KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_CONNECT_TIMEOUT": "13s", - "KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_ACCESS_LOG_PATH": "/access/log/test", - "KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_ADDRESS": "1.1.1.1", + "KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_PORT": "1234", + "KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_HOST": "kuma-control-plane", + "KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_PORT": "4321", + "KUMA_BOOTSTRAP_SERVER_PARAMS_XDS_CONNECT_TIMEOUT": "13s", + "KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_ACCESS_LOG_PATH": "/access/log/test", + "KUMA_BOOTSTRAP_SERVER_PARAMS_ADMIN_ADDRESS": "1.1.1.1", + "KUMA_BOOTSTRAP_SERVER_PARAMS_COREFILE_TEMPLATE_PATH": "/etc/resolv.conf", "KUMA_ENVIRONMENT": "kubernetes", "KUMA_STORE_TYPE": "postgres", "KUMA_STORE_UNSAFE_DELETE": "true", diff --git a/pkg/config/xds/bootstrap/config.go b/pkg/config/xds/bootstrap/config.go index d782af55f5df..8f6cd8a786ef 100644 --- a/pkg/config/xds/bootstrap/config.go +++ b/pkg/config/xds/bootstrap/config.go @@ -1,7 +1,6 @@ package bootstrap import ( - "github.com/kumahq/kuma/pkg/util/files" "net" "os" "time" @@ -11,6 +10,7 @@ import ( "github.com/kumahq/kuma/pkg/config" config_types "github.com/kumahq/kuma/pkg/config/types" + "github.com/kumahq/kuma/pkg/util/files" ) var _ config.Config = &BootstrapServerConfig{} diff --git a/pkg/config/xds/bootstrap/config_test.go b/pkg/config/xds/bootstrap/config_test.go index 19b3ca353b62..66851de78e18 100644 --- a/pkg/config/xds/bootstrap/config_test.go +++ b/pkg/config/xds/bootstrap/config_test.go @@ -17,6 +17,7 @@ var _ = Describe("BootstrappServerConfig", func() { It("should be loadable from configuration file", func() { // given cfg := BootstrapServerConfig{} + //nolint:gosec fileError := os.WriteFile("/tmp/corefile", []byte("abc"), 0o600) Expect(fileError).ToNot(HaveOccurred()) From fb463481673793552737f753e9138fe9e7327051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20S=C5=82onka?= Date: Mon, 18 Dec 2023 14:23:10 +0100 Subject: [PATCH 4/7] Update app/kuma-dp/cmd/run_test.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Krzysztof Słonka --- app/kuma-dp/cmd/run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/kuma-dp/cmd/run_test.go b/app/kuma-dp/cmd/run_test.go index 098fbc6fd72f..cf542d55f1b5 100644 --- a/app/kuma-dp/cmd/run_test.go +++ b/app/kuma-dp/cmd/run_test.go @@ -133,7 +133,7 @@ var _ = Describe("run", func() { }) corednsPid := verifyComponentProcess("coredns", corednsPidFile, corednsCmdlineFile, func(actualArgs []string) { - Expect(len(actualArgs)).To(HaveLen(3)) + Expect(actualArgs).To(HaveLen(3)) Expect(actualArgs[0]).To(Equal("-conf")) Expect(actualArgs[2]).To(Equal("-quiet")) }) From 670b91a67410399dddf532ad41d7c7e4ec022684 Mon Sep 17 00:00:00 2001 From: Jay Chen <1180092+jijiechen@users.noreply.github.com> Date: Fri, 22 Dec 2023 10:34:20 +0800 Subject: [PATCH 5/7] feat(bootstrap): refactor improve the configuration handling for coredns startup Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- app/kuma-dp/cmd/run.go | 16 ++-- .../pkg/dataplane/dnsserver/corefile.go | 2 +- .../pkg/dataplane/dnsserver/corefile_test.go | 4 +- .../pkg/dataplane/dnsserver/dnsserver.go | 84 +++++++++++-------- 4 files changed, 59 insertions(+), 47 deletions(-) diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index 7efe0c484546..27cb3c25e0c8 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -204,14 +204,6 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { opts.AdminPort = bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue() if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() { - if cfg.DNS.CoreDNSConfigTemplatePath == "" && len(kumaSidecarConfiguration.Networking.CorefileTemplate) > 0 { - templateFile, err := writeTempFile("kuma-dp-corefile-template-", "Corefile", kumaSidecarConfiguration.Networking.CorefileTemplate, 0o644) - if err != nil { - return errors.Errorf("Failed to write corefile template to disk. %v", err) - } - cfg.DNS.CoreDNSConfigTemplatePath = templateFile - } - dnsOpts := &dnsserver.Opts{ Config: *cfg, Stdout: cmd.OutOrStdout(), @@ -219,6 +211,10 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { OnFinish: cancelComponents, } + if len(kumaSidecarConfiguration.Networking.CorefileTemplate) > 0 { + dnsOpts.ProvidedCorefileTemplate = kumaSidecarConfiguration.Networking.CorefileTemplate + } + dnsServer, err := dnsserver.New(dnsOpts) if err != nil { return err @@ -349,11 +345,11 @@ func writeTempFile(dirPattern, filename string, data []byte, perm os.FileMode) ( path, err := os.MkdirTemp("", dirPattern) fullFileName := filepath.Join(path, filename) if err != nil { - return "", errors.Errorf("Failed to create a containing directory for %s. %v", filename, err) + return "", errors.Errorf("failed to create a containing directory for %s. %v", filename, err) } err = os.WriteFile(fullFileName, data, perm) if err != nil { - return "", errors.Errorf("Failed to write %s to disk. %v", filename, err) + return "", errors.Errorf("failed to write %s to disk. %v", filename, err) } return fullFileName, nil } diff --git a/app/kuma-dp/pkg/dataplane/dnsserver/corefile.go b/app/kuma-dp/pkg/dataplane/dnsserver/corefile.go index b45cde93e481..57d29255a316 100644 --- a/app/kuma-dp/pkg/dataplane/dnsserver/corefile.go +++ b/app/kuma-dp/pkg/dataplane/dnsserver/corefile.go @@ -13,7 +13,7 @@ import ( //go:embed Corefile var config embed.FS -func GenerateCorefile(cfg kuma_dp.DNS, config []byte) (string, error) { +func WriteCorefile(cfg kuma_dp.DNS, config []byte) (string, error) { configFile := filepath.Join(cfg.ConfigDir, "Corefile") if err := writeFile(configFile, config, 0o600); err != nil { return "", errors.Wrap(err, "failed to persist coredns Corefile on disk") diff --git a/app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go b/app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go index fdc424345f19..040ba97c35f6 100644 --- a/app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go +++ b/app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go @@ -11,7 +11,7 @@ import ( ) var _ = Describe("Corefile", func() { - Describe("GenerateCorefile(..)", func() { + Describe("WriteCorefile(..)", func() { var configDir string BeforeEach(func() { @@ -39,7 +39,7 @@ var _ = Describe("Corefile", func() { } // when - filename, err := GenerateCorefile(dnsConfig, []byte(config)) + filename, err := WriteCorefile(dnsConfig, []byte(config)) // then Expect(err).ToNot(HaveOccurred()) // and diff --git a/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go b/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go index e2430bdaf164..c57dcbd3d631 100644 --- a/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go +++ b/app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go @@ -30,10 +30,11 @@ type DNSServer struct { var _ component.GracefulComponent = &DNSServer{} type Opts struct { - Config kuma_dp.Config - Stdout io.Writer - Stderr io.Writer - OnFinish context.CancelFunc + Config kuma_dp.Config + Stdout io.Writer + Stderr io.Writer + OnFinish context.CancelFunc + ProvidedCorefileTemplate []byte } func lookupDNSServerPath(configuredPath string) (string, error) { @@ -89,39 +90,10 @@ func (s *DNSServer) Start(stop <-chan struct{}) error { dnsConfig := s.opts.Config.DNS - var tmpl *template.Template - - if dnsConfig.CoreDNSConfigTemplatePath != "" { - t, err := template.ParseFiles(dnsConfig.CoreDNSConfigTemplatePath) - if err != nil { - return err - } - - tmpl = t - } else { - corefile, err := config.ReadFile("Corefile") - if err != nil { - return errors.Wrap(err, "couldn't open embedded Corefile") - } - t, err := template.New("Corefile").Parse(string(corefile)) - if err != nil { - return err - } - - tmpl = t - } - - bs := bytes.NewBuffer([]byte{}) - - if err := tmpl.Execute(bs, dnsConfig); err != nil { - return err - } - - configFile, err := GenerateCorefile(dnsConfig, bs.Bytes()) + configFile, err := s.generateCoreFile(dnsConfig) if err != nil { return err } - runLog.Info("configuration saved to a file", "file", configFile) args := []string{ "-conf", configFile, @@ -151,6 +123,50 @@ func (s *DNSServer) Start(stop <-chan struct{}) error { return nil } +func (s *DNSServer) generateCoreFile(dnsConfig kuma_dp.DNS) (string, error) { + var tmpl *template.Template + + if dnsConfig.CoreDNSConfigTemplatePath != "" { + t, err := template.ParseFiles(dnsConfig.CoreDNSConfigTemplatePath) + if err != nil { + return "", err + } + + tmpl = t + } else { + var corefileTemplate []byte + if len(s.opts.ProvidedCorefileTemplate) > 0 { + corefileTemplate = s.opts.ProvidedCorefileTemplate + } else { + embedded, err := config.ReadFile("Corefile") + if err != nil { + return "", errors.Wrap(err, "couldn't open embedded Corefile") + } + corefileTemplate = embedded + } + + t, err := template.New("Corefile").Parse(string(corefileTemplate)) + if err != nil { + return "", err + } + + tmpl = t + } + + bs := bytes.NewBuffer([]byte{}) + + if err := tmpl.Execute(bs, dnsConfig); err != nil { + return "", err + } + + configFile, err := WriteCorefile(dnsConfig, bs.Bytes()) + if err != nil { + return "", err + } + runLog.Info("configuration saved to a file", "file", configFile) + return configFile, nil +} + func (s *DNSServer) WaitForDone() { s.wg.Wait() } From 84070af22abc60153f1dfad1bfb8c90dcbdefc34 Mon Sep 17 00:00:00 2001 From: Jay Chen <1180092+jijiechen@users.noreply.github.com> Date: Fri, 22 Dec 2023 15:54:45 +0800 Subject: [PATCH 6/7] feat(bootstrap): added E2E test case for pushing corefile template from cp to dp Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- app/kuma-dp/cmd/run.go | 13 --- .../kubernetes/bootstrap/corefile_template.go | 84 +++++++++++++++++++ .../kubernetes/kubernetes_suite_test.go | 2 + 3 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 test/e2e_env/kubernetes/bootstrap/corefile_template.go diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index 27cb3c25e0c8..14127515b404 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -340,16 +340,3 @@ func writeFile(filename string, data []byte, perm os.FileMode) error { } return os.WriteFile(filename, data, perm) } - -func writeTempFile(dirPattern, filename string, data []byte, perm os.FileMode) (string, error) { - path, err := os.MkdirTemp("", dirPattern) - fullFileName := filepath.Join(path, filename) - if err != nil { - return "", errors.Errorf("failed to create a containing directory for %s. %v", filename, err) - } - err = os.WriteFile(fullFileName, data, perm) - if err != nil { - return "", errors.Errorf("failed to write %s to disk. %v", filename, err) - } - return fullFileName, nil -} diff --git a/test/e2e_env/kubernetes/bootstrap/corefile_template.go b/test/e2e_env/kubernetes/bootstrap/corefile_template.go new file mode 100644 index 000000000000..2c7ae57da3d9 --- /dev/null +++ b/test/e2e_env/kubernetes/bootstrap/corefile_template.go @@ -0,0 +1,84 @@ +package bootstrap + +import ( + "fmt" + "strings" + "time" + + "github.com/gruntwork-io/terratest/modules/random" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kumahq/kuma/pkg/config/core" + k8s_util "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/util" + . "github.com/kumahq/kuma/test/framework" + "github.com/kumahq/kuma/test/framework/deployments/democlient" +) + +func CorefileTemplate() { + var zoneCluster Cluster + appNamespace := "dns-app" + appName := "demo-dp-app" + expectedTestText := "# this dummy corefile template is loaded from control plane" + configMapName := "corefile-template" + configMap := func(ns string) string { + return fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + namespace: %s + name: %s +data: + %s: | + .:{{ .CoreDNSPort }} { + %s + log + }`, ns, configMapName, configMapName, expectedTestText) + } + + dnsConfigDir := "/tmp/kuma-dp-config/coredns" + BeforeAll(func() { + zoneCluster = NewK8sCluster(NewTestingT(), Kuma2, Silent). + WithTimeout(6 * time.Second). + WithRetries(60) + + Expect(NewClusterSetup(). + Install(Namespace(Config.KumaNamespace)). + Install(YamlK8s(configMap(Config.KumaNamespace))). + Install(Kuma(core.Zone, + WithInstallationMode(HelmInstallationMode), + WithHelmReleaseName(fmt.Sprintf("kuma-%s", strings.ToLower(random.UniqueId()))), + WithHelmOpt("controlPlane.envVars.KUMA_BOOTSTRAP_SERVER_PARAMS_COREFILE_TEMPLATE_PATH", + dnsConfigDir+"/"+configMapName), + WithHelmOpt("controlPlane.extraConfigMaps[0].name", configMapName), + WithHelmOpt("controlPlane.extraConfigMaps[0].mountPath", dnsConfigDir), + WithHelmOpt("controlPlane.extraConfigMaps[0].readonly", "false"), + )). + Install(MeshKubernetes("default")). + Install(NamespaceWithSidecarInjection(appNamespace)). + Install(democlient.Install( + democlient.WithName(appName), + democlient.WithNamespace(appNamespace), + democlient.WithPodAnnotations(map[string]string{ + "kuma.io/sidecar-env-vars": fmt.Sprintf("KUMA_DNS_CONFIG_DIR=%s", dnsConfigDir), + }), + )). + Setup(zoneCluster), + ).To(Succeed()) + }) + + E2EAfterAll(func() { + Expect(zoneCluster.DeleteKuma()).To(Succeed()) + Expect(zoneCluster.DismissCluster()).To(Succeed()) + }) + + It("should use Corefile template from control plane at data plane", func() { + dpPod, err := PodNameOfApp(zoneCluster, appName, appNamespace) + Expect(err).ToNot(HaveOccurred()) + + stdout, stderr, err := zoneCluster.Exec( + appNamespace, dpPod, k8s_util.KumaSidecarContainerName, "cat", dnsConfigDir+"/Corefile") + Expect(err).ToNot(HaveOccurred()) + Expect(stderr).To(BeEmpty()) + Expect(stdout).To(ContainSubstring(expectedTestText)) + }) +} diff --git a/test/e2e_env/kubernetes/kubernetes_suite_test.go b/test/e2e_env/kubernetes/kubernetes_suite_test.go index e41c486a6425..65e99dbdda87 100644 --- a/test/e2e_env/kubernetes/kubernetes_suite_test.go +++ b/test/e2e_env/kubernetes/kubernetes_suite_test.go @@ -7,6 +7,7 @@ import ( "github.com/kumahq/kuma/pkg/test" "github.com/kumahq/kuma/test/e2e_env/kubernetes/api" + "github.com/kumahq/kuma/test/e2e_env/kubernetes/bootstrap" "github.com/kumahq/kuma/test/e2e_env/kubernetes/connectivity" "github.com/kumahq/kuma/test/e2e_env/kubernetes/container_patch" "github.com/kumahq/kuma/test/e2e_env/kubernetes/defaults" @@ -49,6 +50,7 @@ var _ = E2ESynchronizedBeforeSuite(kubernetes.SetupAndGetState, kubernetes.Resto var _ = SynchronizedAfterSuite(func() {}, func() {}) var ( + _ = Describe("Corefile Template", bootstrap.CorefileTemplate, Ordered) _ = Describe("Virtual Probes", healthcheck.VirtualProbes, Ordered) _ = Describe("Gateway", gateway.Gateway, Ordered) _ = Describe("Gateway - Cross-mesh", gateway.CrossMeshGatewayOnKubernetes, Ordered) From 2b2147e2a02fbb620f48ba6bbd1ce483c1f3e5a5 Mon Sep 17 00:00:00 2001 From: Jay Chen <1180092+jijiechen@users.noreply.github.com> Date: Sat, 23 Dec 2023 00:28:52 +0800 Subject: [PATCH 7/7] feat(bootstrap): start kuma-2 for runing tests involving control plane customization Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- mk/e2e.new.mk | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mk/e2e.new.mk b/mk/e2e.new.mk index 3ff1f6677896..a8ff9737750a 100644 --- a/mk/e2e.new.mk +++ b/mk/e2e.new.mk @@ -136,11 +136,9 @@ test/e2e: $(E2E_DEPS_TARGETS) $(E2E_K8S_BIN_DEPS) ## Run slower e2e tests (slowe .PHONY: test/e2e-kubernetes test/e2e-kubernetes: $(E2E_DEPS_TARGETS) $(E2E_K8S_BIN_DEPS) ## Run kubernetes e2e tests. Use DEBUG=1 to more easily find issues $(MAKE) docker/tag - $(MAKE) test/e2e/k8s/start/cluster/kuma-1 - $(MAKE) test/e2e/k8s/wait/kuma-1 - $(MAKE) test/e2e/k8s/load/images/kuma-1 + $(MAKE) test/e2e/k8s/start $(E2E_ENV_VARS) $(GINKGO_TEST_E2E) $(KUBE_E2E_PKG_LIST) || (ret=$$?; $(MAKE) test/e2e/k8s/stop/cluster/kuma-1 && exit $$ret) - $(MAKE) test/e2e/k8s/stop/cluster/kuma-1 + $(MAKE) test/e2e/k8s/stop .PHONY: test/e2e-universal test/e2e-universal: $(E2E_DEPS_TARGETS) $(E2E_UNIVERSAL_BIN_DEPS) k3d/network/create ## Run universal e2e tests. Use DEBUG=1 to more easily find issues