feat(bootstrap): support corefile template pushed from the cp on dp

Signed-off-by: Jay Chen <[email protected]>

app/kuma-dp/cmd/run.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver"
 	"os"
 	"path/filepath"
 	"time"
@@ -11,7 +12,6 @@ import (
 
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/accesslogs"
-	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics"
 	kuma_cmd "github.com/kumahq/kuma/pkg/cmd"
@@ -166,29 +166,6 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command {
 				OnFinish:  cancelComponents,
 			}
 
-			if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() {
-				dnsOpts := &dnsserver.Opts{
-					Config:   *cfg,
-					Stdout:   cmd.OutOrStdout(),
-					Stderr:   cmd.OutOrStderr(),
-					OnFinish: cancelComponents,
-				}
-
-				dnsServer, err := dnsserver.New(dnsOpts)
-				if err != nil {
-					return err
-				}
-
-				version, err := dnsServer.GetVersion()
-				if err != nil {
-					return err
-				}
-
-				rootCtx.BootstrapDynamicMetadata[core_xds.FieldPrefixDependenciesVersion+".coredns"] = version
-
-				components = append(components, dnsServer)
-			}
-
 			envoyVersion, err := envoy.GetEnvoyVersion(opts.Config.DataplaneRuntime.BinaryPath)
 			if err != nil {
 				return errors.Wrap(err, "failed to get Envoy version")
@@ -226,6 +203,37 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command {
 			}
 			opts.AdminPort = bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue()
 
+			if cfg.DNS.Enabled && !cfg.Dataplane.IsZoneProxy() {
+				if cfg.DNS.CoreDNSConfigTemplatePath == "" && len(kumaSidecarConfiguration.Networking.CorefileTemplate) > 0 {
+					templateFile, err := writeTempFile("kuma-dp-corefile-template-", "Corefile", kumaSidecarConfiguration.Networking.CorefileTemplate, 0644)
+					if err != nil {
+						return errors.Errorf("Failed to write corefile template to disk. %v", err)
+					}
+					cfg.DNS.CoreDNSConfigTemplatePath = templateFile
+				}
+
+				dnsOpts := &dnsserver.Opts{
+					Config:   *cfg,
+					Stdout:   cmd.OutOrStdout(),
+					Stderr:   cmd.OutOrStderr(),
+					OnFinish: cancelComponents,
+				}
+
+				dnsServer, err := dnsserver.New(dnsOpts)
+				if err != nil {
+					return err
+				}
+
+				version, err := dnsServer.GetVersion()
+				if err != nil {
+					return err
+				}
+
+				rootCtx.BootstrapDynamicMetadata[core_xds.FieldPrefixDependenciesVersion+".coredns"] = version
+
+				components = append(components, dnsServer)
+			}
+
 			envoyComponent, err := envoy.New(opts)
 			if err != nil {
 				return err
@@ -338,3 +346,16 @@ func writeFile(filename string, data []byte, perm os.FileMode) error {
 	}
 	return os.WriteFile(filename, data, perm)
 }
+
+func writeTempFile(dirPattern, filename string, data []byte, perm os.FileMode) (string, error) {
+	path, err := os.MkdirTemp("", dirPattern)
+	fullFileName := filepath.Join(path, filename)
+	if err != nil {
+		return "", errors.Errorf("Failed to create a containing directory for %s. %v", filename, err)
+	}
+	err = os.WriteFile(fullFileName, data, perm)
+	if err != nil {
+		return "", errors.Errorf("Failed to write %s to disk. %v", filename, err)
+	}
+	return fullFileName, nil
+}

app/kuma-dp/cmd/run_test.go

@@ -5,6 +5,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -72,13 +73,17 @@ var _ = Describe("run", func() {
 			given := givenFunc()
 
 			// setup
-			pidFile := filepath.Join(tmpDir, "envoy-mock.pid")
-			cmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline")
+			envoyPidFile := filepath.Join(tmpDir, "envoy-mock.pid")
+			envoyCmdlineFile := filepath.Join(tmpDir, "envoy-mock.cmdline")
+			corednsPidFile := filepath.Join(tmpDir, "coredns-mock.pid")
+			corednsCmdlineFile := filepath.Join(tmpDir, "coredns-mock.cmdline")
 
 			// and
 			env := given.envVars
-			env["ENVOY_MOCK_PID_FILE"] = pidFile
-			env["ENVOY_MOCK_CMDLINE_FILE"] = cmdlineFile
+			env["ENVOY_MOCK_PID_FILE"] = envoyPidFile
+			env["ENVOY_MOCK_CMDLINE_FILE"] = envoyCmdlineFile
+			env["COREDNS_MOCK_PID_FILE"] = corednsPidFile
+			env["COREDNS_MOCK_CMDLINE_FILE"] = corednsCmdlineFile
 			for key, value := range env {
 				Expect(os.Setenv(key, value)).To(Succeed())
 			}
@@ -116,35 +121,22 @@ var _ = Describe("run", func() {
 			}()
 
 			// then
-			var pid int64
-			By("waiting for dataplane (Envoy) to get started")
-			Eventually(func() bool {
-				data, err := os.ReadFile(pidFile)
-				if err != nil {
-					return false
+			var actualConfigFile string
+			envoyPid := verifyComponentProcess("Envoy", envoyPidFile, envoyCmdlineFile, func(actualArgs []string) {
+				Expect(actualArgs[0]).To(Equal("--version"))
+				Expect(actualArgs[1]).To(Equal("--config-path"))
+				actualConfigFile = actualArgs[2]
+				Expect(actualConfigFile).To(BeARegularFile())
+				if given.expectedFile != "" {
+					Expect(actualArgs[2]).To(Equal(given.expectedFile))
 				}
-				pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32)
-				return err == nil
-			}, "5s", "100ms").Should(BeTrue())
-			// and
-			Expect(pid).ToNot(BeZero())
-
-			By("verifying the arguments Envoy was launched with")
-			// when
-			cmdline, err := os.ReadFile(cmdlineFile)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			actualArgs := strings.Split(string(cmdline), "\n")
-			Expect(actualArgs[0]).To(Equal("--version"))
-			Expect(actualArgs[1]).To(Equal("--config-path"))
-			actualConfigFile := actualArgs[2]
-			Expect(actualConfigFile).To(BeARegularFile())
+			})
 
-			// then
-			if given.expectedFile != "" {
-				Expect(actualArgs[2]).To(Equal(given.expectedFile))
-			}
+			corednsPid := verifyComponentProcess("coredns", corednsPidFile, corednsCmdlineFile, func(actualArgs []string) {
+				Expect(len(actualArgs)).To(Equal(3))
+				Expect(actualArgs[0]).To(Equal("-conf"))
+				Expect(actualArgs[2]).To(Equal("-quiet"))
+			})
 
 			// when
 			By("signaling the dataplane manager to stop")
@@ -153,13 +145,20 @@ var _ = Describe("run", func() {
 			cancel()
 
 			// then
-			err = <-errCh
+			err := <-errCh
 			Expect(err).ToNot(HaveOccurred())
 
 			By("waiting for dataplane (Envoy) to get stopped")
 			Eventually(func() bool {
 				// send sig 0 to check whether Envoy process still exists
-				err := syscall.Kill(int(pid), syscall.Signal(0))
+				err := syscall.Kill(int(envoyPid), syscall.Signal(0))
+				// we expect Envoy process to get killed by now
+				return err != nil
+			}, "5s", "100ms").Should(BeTrue())
+			By("waiting for dataplane (coredns) to get stopped")
+			Eventually(func() bool {
+				// send sig 0 to check whether Envoy process still exists
+				err := syscall.Kill(int(corednsPid), syscall.Signal(0))
 				// we expect Envoy process to get killed by now
 				return err != nil
 			}, "5s", "100ms").Should(BeTrue())
@@ -288,6 +287,23 @@ var _ = Describe("run", func() {
 				expectedFile: "",
 			}
 		}),
+		Entry("can be launched with given coredns configuration path", func() testCase {
+			corefileTemplate := filepath.Join(tmpDir, "Corefile")
+			_ = os.WriteFile(corefileTemplate, []byte("abcd"), 0o600)
+			return testCase{
+				envVars: map[string]string{
+					"KUMA_CONTROL_PLANE_API_SERVER_URL":      "http://localhost:1234",
+					"KUMA_DATAPLANE_NAME":                    "example",
+					"KUMA_DATAPLANE_MESH":                    "default",
+					"KUMA_DATAPLANE_RUNTIME_BINARY_PATH":     filepath.Join("testdata", "envoy-mock.sleep.sh"),
+					"KUMA_DATAPLANE_RUNTIME_CONFIG_DIR":      tmpDir,
+					"KUMA_DNS_CORE_DNS_BINARY_PATH":          filepath.Join("testdata", "coredns-mock.sleep.sh"),
+					"KUMA_DNS_CORE_DNS_CONFIG_TEMPLATE_PATH": corefileTemplate,
+				},
+				args:         []string{},
+				expectedFile: filepath.Join(tmpDir, "bootstrap.yaml"),
+			}
+		}),
 	)
 
 	It("should fail when name and mesh is provided with dataplane definition", func() {
@@ -333,3 +349,30 @@ var _ = Describe("run", func() {
 		Expect(err.Error()).To(ContainSubstring("invalid proxy type"))
 	})
 }, Ordered)
+
+func verifyComponentProcess(processDescription, pidfile string, cmdlinefile string, argsVerifier func(expectedArgs []string)) int64 {
+	var pid int64
+	By(fmt.Sprintf("waiting for dataplane (%s) to get started", processDescription))
+	Eventually(func() bool {
+		data, err := os.ReadFile(pidfile)
+		if err != nil {
+			return false
+		}
+		pid, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 32)
+		return err == nil
+	}, "5s", "100ms").Should(BeTrue())
+	// and
+	Expect(pid).ToNot(BeZero())
+
+	By(fmt.Sprintf("verifying the arguments %s was launched with", processDescription))
+	// when
+	cmdline, err := os.ReadFile(cmdlinefile)
+	// then
+	Expect(err).ToNot(HaveOccurred())
+	// and
+	if argsVerifier != nil {
+		actualArgs := strings.Split(string(cmdline), "\n")
+		argsVerifier(actualArgs)
+	}
+	return pid
+}

app/kuma-dp/cmd/testdata/coredns-mock.sleep.sh

@@ -6,6 +6,11 @@ then
     exit 0
 fi
 
+echo $$ >"${COREDNS_MOCK_PID_FILE}"
+for arg in "$@"; do
+    echo "$arg" >> "${ENVOY_MOCK_CMDLINE_FILE}"
+done
+
 # Send logs for Cmd#Wait to finish
 while true;
 do

app/kuma-dp/pkg/dataplane/dnsserver/config_file.go → app/kuma-dp/pkg/dataplane/dnsserver/corefile.go

@@ -13,10 +13,10 @@ import (
 //go:embed Corefile
 var config embed.FS
 
-func GenerateConfigFile(cfg kuma_dp.DNS, config []byte) (string, error) {
+func GenerateCorefile(cfg kuma_dp.DNS, config []byte) (string, error) {
 	configFile := filepath.Join(cfg.ConfigDir, "Corefile")
 	if err := writeFile(configFile, config, 0o600); err != nil {
-		return "", errors.Wrap(err, "failed to persist Envoy bootstrap config on disk")
+		return "", errors.Wrap(err, "failed to persist coredns Corefile on disk")
 	}
 	return configFile, nil
 }

app/kuma-dp/pkg/dataplane/dnsserver/config_file_test.go → app/kuma-dp/pkg/dataplane/dnsserver/corefile_test.go

@@ -10,8 +10,8 @@ import (
 	kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
 )
 
-var _ = Describe("Config File", func() {
-	Describe("GenerateConfigFile(..)", func() {
+var _ = Describe("Corefile", func() {
+	Describe("GenerateCorefile(..)", func() {
 		var configDir string
 
 		BeforeEach(func() {
@@ -39,7 +39,7 @@ var _ = Describe("Config File", func() {
 			}
 
 			// when
-			filename, err := GenerateConfigFile(dnsConfig, []byte(config))
+			filename, err := GenerateCorefile(dnsConfig, []byte(config))
 			// then
 			Expect(err).ToNot(HaveOccurred())
 			// and

app/kuma-dp/pkg/dataplane/dnsserver/dnsserver.go

@@ -117,7 +117,7 @@ func (s *DNSServer) Start(stop <-chan struct{}) error {
 		return err
 	}
 
-	configFile, err := GenerateConfigFile(dnsConfig, bs.Bytes())
+	configFile, err := GenerateCorefile(dnsConfig, bs.Bytes())
 	if err != nil {
 		return err
 	}