feat(bootstrap): support customizing corefile template from kuma-cp

app/kuma-dp/cmd/run.go

@@ -166,29 +166,6 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command {
 				OnFinish:  cancelComponents,
 			}
 
-			if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() {
-				dnsOpts := &dnsserver.Opts{
-					Config:   *cfg,
-					Stdout:   cmd.OutOrStdout(),
-					Stderr:   cmd.OutOrStderr(),
-					OnFinish: cancelComponents,
-				}
-
-				dnsServer, err := dnsserver.New(dnsOpts)
-				if err != nil {
-					return err
-				}
-
-				version, err := dnsServer.GetVersion()
-				if err != nil {
-					return err
-				}
-
-				rootCtx.BootstrapDynamicMetadata[core_xds.FieldPrefixDependenciesVersion+".coredns"] = version
-
-				components = append(components, dnsServer)
-			}
-
 			envoyVersion, err := envoy.GetEnvoyVersion(opts.Config.DataplaneRuntime.BinaryPath)
 			if err != nil {
 				return errors.Wrap(err, "failed to get Envoy version")
@@ -226,6 +203,33 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command {
 			}
 			opts.AdminPort = bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue()
 
+			if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() {
+				dnsOpts := &dnsserver.Opts{
+					Config:   *cfg,
+					Stdout:   cmd.OutOrStdout(),
+					Stderr:   cmd.OutOrStderr(),
+					OnFinish: cancelComponents,
+				}
+
+				if len(kumaSidecarConfiguration.Networking.CorefileTemplate) > 0 {
+					dnsOpts.ProvidedCorefileTemplate = kumaSidecarConfiguration.Networking.CorefileTemplate
+				}
+
+				dnsServer, err := dnsserver.New(dnsOpts)
+				if err != nil {
+					return err
+				}
+
+				version, err := dnsServer.GetVersion()
+				if err != nil {
+					return err
+				}
+
+				rootCtx.BootstrapDynamicMetadata[core_xds.FieldPrefixDependenciesVersion+".coredns"] = version
+
+				components = append(components, dnsServer)
+			}
+
 			envoyComponent, err := envoy.New(opts)
 			if err != nil {
 				return err

app/kuma-dp/cmd/run_test.go

@@ -5,6 +5,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -72,13 +73,17 @@ var _ = Describe("run", func() {
 			given := givenFunc()
 
 			// setup
-			pidFile := filepath.Join(tmpDir, "envoy-mock.pid")
-			cmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline")
+			envoyPidFile := filepath.Join(tmpDir, "envoy-mock.pid")
+			envoyCmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline")
+			corednsPidFile := filepath.Join(tmpDir, "coredns-mock.pid")
+			corednsCmdlineFile := filepath.Join(tmpDir, "coredns-mock.cmdline")
 
 			// and
 			env := given.envVars
-			env["ENVOY_MOCK_PID_FILE"] = pidFile
-			env["ENVOY_MOCK_CMDLINE_FILE"] = cmdlineFile
+			env["ENVOY_MOCK_PID_FILE"] = envoyPidFile
+			env["ENVOY_MOCK_CMDLINE_FILE"] = envoyCmdlineFile
+			env["COREDNS_MOCK_PID_FILE"] = corednsPidFile
+			env["COREDNS_MOCK_CMDLINE_FILE"] = corednsCmdlineFile
 			for key, value := range env {
 				Expect(os.Setenv(key, value)).To(Succeed())
 			}
@@ -116,35 +121,22 @@ var _ = Describe("run", func() {
 			}()
 
 			// then
-			var pid int64
-			By("waiting for dataplane (Envoy) to get started")
-			Eventually(func() bool {
-				data, err := os.ReadFile(pidFile)
-				if err != nil {
-					return false
+			var actualConfigFile string
+			envoyPid := verifyComponentProcess("Envoy", envoyPidFile, envoyCmdlineFile, func(actualArgs []string) {
+				Expect(actualArgs[0]).To(Equal("--version"))
+				Expect(actualArgs[1]).To(Equal("--config-path"))
+				actualConfigFile = actualArgs[2]
+				Expect(actualConfigFile).To(BeARegularFile())
+				if given.expectedFile != "" {
+					Expect(actualArgs[2]).To(Equal(given.expectedFile))
 				}
-				pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32)
-				return err == nil
-			}, "5s", "100ms").Should(BeTrue())
-			// and
-			Expect(pid).ToNot(BeZero())
+			})
 
-			By("verifying the arguments Envoy was launched with")
-			// when
-			cmdline, err := os.ReadFile(cmdlineFile)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			actualArgs := strings.Split(string(cmdline), "\n")
-			Expect(actualArgs[0]).To(Equal("--version"))
-			Expect(actualArgs[1]).To(Equal("--config-path"))
-			actualConfigFile := actualArgs[2]
-			Expect(actualConfigFile).To(BeARegularFile())
-
-			// then
-			if given.expectedFile != "" {
-				Expect(actualArgs[2]).To(Equal(given.expectedFile))
-			}
+			corednsPid := verifyComponentProcess("coredns", corednsPidFile, corednsCmdlineFile, func(actualArgs []string) {
+				Expect(actualArgs).To(HaveLen(3))
+				Expect(actualArgs[0]).To(Equal("-conf"))
+				Expect(actualArgs[2]).To(Equal("-quiet"))
+			})
 
 			// when
 			By("signaling the dataplane manager to stop")
@@ -153,13 +145,20 @@ var _ = Describe("run", func() {
 			cancel()
 
 			// then
-			err = <-errCh
+			err := <-errCh
 			Expect(err).ToNot(HaveOccurred())
 
 			By("waiting for dataplane (Envoy) to get stopped")
 			Eventually(func() bool {
 				// send sig 0 to check whether Envoy process still exists
-				err := syscall.Kill(int(pid), syscall.Signal(0))
+				err := syscall.Kill(int(envoyPid), syscall.Signal(0))
+				// we expect Envoy process to get killed by now
+				return err != nil
+			}, "5s", "100ms").Should(BeTrue())
+			By("waiting for dataplane (coredns) to get stopped")
+			Eventually(func() bool {
+				// send sig 0 to check whether Envoy process still exists
+				err := syscall.Kill(int(corednsPid), syscall.Signal(0))
 				// we expect Envoy process to get killed by now
 				return err != nil
 			}, "5s", "100ms").Should(BeTrue())
@@ -288,6 +287,23 @@ var _ = Describe("run", func() {
 				expectedFile: "",
 			}
 		}),
+		Entry("can be launched with given coredns configuration path", func() testCase {
+			corefileTemplate := filepath.Join(tmpDir, "Corefile")
+			_ = os.WriteFile(corefileTemplate, []byte("abcd"), 0o600)
+			return testCase{
+				envVars: map[string]string{
+					"KUMA_CONTROL_PLANE_API_SERVER_URL":      "http://localhost:1234",
+					"KUMA_DATAPLANE_NAME":                    "example",
+					"KUMA_DATAPLANE_MESH":                    "default",
+					"KUMA_DATAPLANE_RUNTIME_BINARY_PATH":     filepath.Join("testdata", "envoy-mock.sleep.sh"),
+					"KUMA_DATAPLANE_RUNTIME_CONFIG_DIR":      tmpDir,
+					"KUMA_DNS_CORE_DNS_BINARY_PATH":          filepath.Join("testdata", "coredns-mock.sleep.sh"),
+					"KUMA_DNS_CORE_DNS_CONFIG_TEMPLATE_PATH": corefileTemplate,
+				},
+				args:         []string{},
+				expectedFile: filepath.Join(tmpDir, "bootstrap.yaml"),
+			}
+		}),
 	)
 
 	It("should fail when name and mesh is provided with dataplane definition", func() {
@@ -333,3 +349,32 @@ var _ = Describe("run", func() {
 		Expect(err.Error()).To(ContainSubstring("invalid proxy type"))
 	})
 }, Ordered)
+
+func verifyComponentProcess(processDescription, pidfile string, cmdlinefile string, argsVerifier func(expectedArgs []string)) int64 {
+	var pid int64
+	By(fmt.Sprintf("waiting for dataplane (%s) to get started", processDescription))
+	Eventually(func() bool {
+		data, err := os.ReadFile(pidfile)
+		if err != nil {
+			return false
+		}
+		pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32)
+		return err == nil
+	}, "5s", "100ms").Should(BeTrue())
+	Expect(pid).ToNot(BeZero())
+
+	By(fmt.Sprintf("verifying the arguments %s was launched with", processDescription))
+	// when
+	cmdline, err := os.ReadFile(cmdlinefile)
+
+	// then
+	Expect(err).ToNot(HaveOccurred())
+	// and
+	if argsVerifier != nil {
+		actualArgs := strings.FieldsFunc(string(cmdline), func(c rune) bool {
+			return c == '\n'
+		})
+		argsVerifier(actualArgs)
+	}
+	return pid
+}

app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh

@@ -6,6 +6,11 @@ then
     exit 0
 fi
 
+echo $$ >"${COREDNS_MOCK_PID_FILE}"
+for arg in "$@"; do
+    echo "$arg" >> "${COREDNS_MOCK_CMDLINE_FILE}"
+done
+
 # Send logs for Cmd#Wait to finish
 while true;
 do

app/kuma-dp/pkg/dataplane/dnsserver/config_file.go → app/kuma-dp/pkg/dataplane/dnsserver/corefile.go

@@ -13,10 +13,10 @@ import (
 //go:embed Corefile
 var config embed.FS
 
-func GenerateConfigFile(cfg kuma_dp.DNS, config []byte) (string, error) {
+func WriteCorefile(cfg kuma_dp.DNS, config []byte) (string, error) {
 	configFile := filepath.Join(cfg.ConfigDir, "Corefile")
 	if err := writeFile(configFile, config, 0o600); err != nil {
-		return "", errors.Wrap(err, "failed to persist Envoy bootstrap config on disk")
+		return "", errors.Wrap(err, "failed to persist coredns Corefile on disk")
 	}
 	return configFile, nil
 }

app/kuma-dp/pkg/dataplane/dnsserver/config_file_test.go → app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go

@@ -10,8 +10,8 @@ import (
 	kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
 )
 
-var _ = Describe("Config File", func() {
-	Describe("GenerateConfigFile(..)", func() {
+var _ = Describe("Corefile", func() {
+	Describe("WriteCorefile(..)", func() {
 		var configDir string
 
 		BeforeEach(func() {
@@ -39,7 +39,7 @@ var _ = Describe("Config File", func() {
 			}
 
 			// when
-			filename, err := GenerateConfigFile(dnsConfig, []byte(config))
+			filename, err := WriteCorefile(dnsConfig, []byte(config))
 			// then
 			Expect(err).ToNot(HaveOccurred())
 			// and

app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go

@@ -30,10 +30,11 @@ type DNSServer struct {
 var _ component.GracefulComponent = &DNSServer{}
 
 type Opts struct {
-	Config   kuma_dp.Config
-	Stdout   io.Writer
-	Stderr   io.Writer
-	OnFinish context.CancelFunc
+	Config                   kuma_dp.Config
+	Stdout                   io.Writer
+	Stderr                   io.Writer
+	OnFinish                 context.CancelFunc
+	ProvidedCorefileTemplate []byte
 }
 
 func lookupDNSServerPath(configuredPath string) (string, error) {
@@ -89,39 +90,10 @@ func (s *DNSServer) Start(stop <-chan struct{}) error {
 
 	dnsConfig := s.opts.Config.DNS
 
-	var tmpl *template.Template
-
-	if dnsConfig.CoreDNSConfigTemplatePath != "" {
-		t, err := template.ParseFiles(dnsConfig.CoreDNSConfigTemplatePath)
-		if err != nil {
-			return err
-		}
-
-		tmpl = t
-	} else {
-		corefile, err := config.ReadFile("Corefile")
-		if err != nil {
-			return errors.Wrap(err, "couldn't open embedded Corefile")
-		}
-		t, err := template.New("Corefile").Parse(string(corefile))
-		if err != nil {
-			return err
-		}
-
-		tmpl = t
-	}
-
-	bs := bytes.NewBuffer([]byte{})
-
-	if err := tmpl.Execute(bs, dnsConfig); err != nil {
-		return err
-	}
-
-	configFile, err := GenerateConfigFile(dnsConfig, bs.Bytes())
+	configFile, err := s.generateCoreFile(dnsConfig)
 	if err != nil {
 		return err
 	}
-	runLog.Info("configuration saved to a file", "file", configFile)
 
 	args := []string{
 		"-conf", configFile,
@@ -151,6 +123,50 @@ func (s *DNSServer) Start(stop <-chan struct{}) error {
 	return nil
 }
 
+func (s *DNSServer) generateCoreFile(dnsConfig kuma_dp.DNS) (string, error) {
+	var tmpl *template.Template
+
+	if dnsConfig.CoreDNSConfigTemplatePath != "" {
+		t, err := template.ParseFiles(dnsConfig.CoreDNSConfigTemplatePath)
+		if err != nil {
+			return "", err
+		}
+
+		tmpl = t
+	} else {
+		var corefileTemplate []byte
+		if len(s.opts.ProvidedCorefileTemplate) > 0 {
+			corefileTemplate = s.opts.ProvidedCorefileTemplate
+		} else {
+			embedded, err := config.ReadFile("Corefile")
+			if err != nil {
+				return "", errors.Wrap(err, "couldn't open embedded Corefile")
+			}
+			corefileTemplate = embedded
+		}
+
+		t, err := template.New("Corefile").Parse(string(corefileTemplate))
+		if err != nil {
+			return "", err
+		}
+
+		tmpl = t
+	}
+
+	bs := bytes.NewBuffer([]byte{})
+
+	if err := tmpl.Execute(bs, dnsConfig); err != nil {
+		return "", err
+	}
+
+	configFile, err := WriteCorefile(dnsConfig, bs.Bytes())
+	if err != nil {
+		return "", err
+	}
+	runLog.Info("configuration saved to a file", "file", configFile)
+	return configFile, nil
+}
+
 func (s *DNSServer) WaitForDone() {
 	s.wg.Wait()
 }

mk/e2e.new.mk

@@ -136,11 +136,9 @@ test/e2e: $(E2E_DEPS_TARGETS) $(E2E_K8S_BIN_DEPS) ## Run slower e2e tests (slowe
 .PHONY: test/e2e-kubernetes
 test/e2e-kubernetes: $(E2E_DEPS_TARGETS) $(E2E_K8S_BIN_DEPS) ## Run kubernetes e2e tests. Use DEBUG=1 to more easily find issues
 	$(MAKE) docker/tag
-	$(MAKE) test/e2e/k8s/start/cluster/kuma-1
-	$(MAKE) test/e2e/k8s/wait/kuma-1
-	$(MAKE) test/e2e/k8s/load/images/kuma-1
+	$(MAKE) test/e2e/k8s/start
 	$(E2E_ENV_VARS) $(GINKGO_TEST_E2E) $(KUBE_E2E_PKG_LIST) || (ret=$$?; $(MAKE) test/e2e/k8s/stop/cluster/kuma-1 && exit $$ret)
-	$(MAKE) test/e2e/k8s/stop/cluster/kuma-1
+	$(MAKE) test/e2e/k8s/stop
 
 .PHONY: test/e2e-universal
 test/e2e-universal: $(E2E_DEPS_TARGETS) $(E2E_UNIVERSAL_BIN_DEPS) k3d/network/create ## Run universal e2e tests. Use DEBUG=1 to more easily find issues