From 63a7a38d3966198dade09ffcf5932c872e9ead9a Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Sun, 1 Dec 2024 18:47:35 +0100
Subject: [PATCH 01/10] test: bind mount gomodcache to cache go dependencies

try to speed up ci jobs integration tests
---
 dev-tools/mage/integtest_docker.go | 6 ++++++
 metricbeat/docker-compose.yml      | 1 +
 2 files changed, 7 insertions(+)

diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go
index 8c5fb6d3603f..ccb8f7b5522f 100644
--- a/dev-tools/mage/integtest_docker.go
+++ b/dev-tools/mage/integtest_docker.go
@@ -449,9 +449,15 @@ func integTestDockerComposeEnvVars() (map[string]string, error) {
 		return nil, err
 	}
 
+	goModCache := os.Getenv("GOMODCACHE")
+	if goModCache == "" {
+		goModCache = os.Getenv("GOPATH") + "/pkg/mod"
+	}
+
 	return map[string]string{
 		"ES_BEATS":          esBeatsDir,
 		"STACK_ENVIRONMENT": StackEnvironment,
+		"GOMODCACHE":        goModCache,
 		// Deprecated use STACK_ENVIRONMENT instead (it's more descriptive).
 		"TESTING_ENVIRONMENT": StackEnvironment,
 	}, nil
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 03b48240814b..afcc7ea49d2c 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -11,6 +11,7 @@ services:
     working_dir: /go/src/github.com/elastic/beats/metricbeat
     volumes:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
+      - ${GOMODCACHE}:/go/pkg/mod/:ro
       - /var/run/docker.sock:/var/run/docker.sock
     network_mode: host
     command: make

From ed1e67480f83372d81470c82989053b3c2ad8c48 Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Sun, 1 Dec 2024 22:32:34 +0100
Subject: [PATCH 02/10] test(metricbeat): do not make gomodcache readonly

---
 metricbeat/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index afcc7ea49d2c..67f136d4e05a 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -11,7 +11,7 @@ services:
     working_dir: /go/src/github.com/elastic/beats/metricbeat
     volumes:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
-      - ${GOMODCACHE}:/go/pkg/mod/:ro
+      - ${GOMODCACHE}:/go/pkg/mod/
       - /var/run/docker.sock:/var/run/docker.sock
     network_mode: host
     command: make

From 279f2bc1acfa7666481954af30de0ccb172c194e Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Sun, 1 Dec 2024 22:33:58 +0100
Subject: [PATCH 03/10] lint: replace ioutil usage

---
 dev-tools/mage/integtest_docker.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go
index ccb8f7b5522f..c9773ff519a5 100644
--- a/dev-tools/mage/integtest_docker.go
+++ b/dev-tools/mage/integtest_docker.go
@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"go/build"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -356,7 +355,7 @@ func StartIntegTestContainers() error {
 
 func StopIntegTestContainers() error {
 	// Docker-compose rm is noisy. So only pass through stderr when in verbose.
-	out := ioutil.Discard
+	out := io.Discard
 	if mg.Verbose() {
 		out = os.Stderr
 	}
@@ -368,7 +367,7 @@ func StopIntegTestContainers() error {
 
 	_, err = sh.Exec(
 		composeEnv,
-		ioutil.Discard,
+		io.Discard,
 		out,
 		"docker-compose",
 		"-p", DockerComposeProjectName(),

From fe9168f57eecd9e6eb5eace9c2c6054a80230800 Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Sun, 1 Dec 2024 23:47:51 +0100
Subject: [PATCH 04/10] test: fail on empty gomodcache

---
 dev-tools/mage/integtest_docker.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go
index c9773ff519a5..20c3f1e3ae5b 100644
--- a/dev-tools/mage/integtest_docker.go
+++ b/dev-tools/mage/integtest_docker.go
@@ -453,6 +453,11 @@ func integTestDockerComposeEnvVars() (map[string]string, error) {
 		goModCache = os.Getenv("GOPATH") + "/pkg/mod"
 	}
 
+	if goModCache == "" {
+		_, file, no, _ := runtime.Caller(1)
+		return nil, fmt.Errorf("empty gomodcache: %s:%d", file, no)
+	}
+
 	return map[string]string{
 		"ES_BEATS":          esBeatsDir,
 		"STACK_ENVIRONMENT": StackEnvironment,

From 537ca3917e274912342a849166e5556c453fb2f4 Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Mon, 2 Dec 2024 01:00:02 +0100
Subject: [PATCH 05/10] test: rework cache bind approach

instead of binding the volume on each dockerfile, run the container
with the proper volume arg
---
 dev-tools/mage/integtest_docker.go | 21 +++------------------
 metricbeat/docker-compose.yml      |  1 -
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go
index 20c3f1e3ae5b..ea29d993b272 100644
--- a/dev-tools/mage/integtest_docker.go
+++ b/dev-tools/mage/integtest_docker.go
@@ -98,11 +98,9 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 	if err != nil {
 		return err
 	}
-	dockerRepoRoot := filepath.Join("/go/src", repo.CanonicalRootImportPath)
-	dockerGoCache := filepath.Join(dockerRepoRoot, "build/docker-gocache")
 	magePath := filepath.Join("/go/src", repo.CanonicalRootImportPath, repo.SubDir, "build/mage-linux-"+GOARCH)
-	goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod/cache/download")
-	dockerGoPkgCache := "/gocache"
+	goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod")
+	dockerGoPkgCache := "/go/pkg/mod"
 
 	// Execute the inside of docker-compose.
 	args := []string{"-p", DockerComposeProjectName(), "run",
@@ -113,10 +111,8 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 		// compose.EnsureUp needs to know the environment type.
 		"-e", "STACK_ENVIRONMENT=" + StackEnvironment,
 		"-e", "TESTING_ENVIRONMENT=" + StackEnvironment,
-		"-e", "GOCACHE=" + dockerGoCache,
 		// Use the host machine's pkg cache to minimize external downloads.
-		"-v", goPkgCache + ":" + dockerGoPkgCache + ":ro",
-		"-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct",
+		"-v", goPkgCache + ":" + dockerGoPkgCache,
 	}
 	args, err = addUidGidEnvArgs(args)
 	if err != nil {
@@ -448,20 +444,9 @@ func integTestDockerComposeEnvVars() (map[string]string, error) {
 		return nil, err
 	}
 
-	goModCache := os.Getenv("GOMODCACHE")
-	if goModCache == "" {
-		goModCache = os.Getenv("GOPATH") + "/pkg/mod"
-	}
-
-	if goModCache == "" {
-		_, file, no, _ := runtime.Caller(1)
-		return nil, fmt.Errorf("empty gomodcache: %s:%d", file, no)
-	}
-
 	return map[string]string{
 		"ES_BEATS":          esBeatsDir,
 		"STACK_ENVIRONMENT": StackEnvironment,
-		"GOMODCACHE":        goModCache,
 		// Deprecated use STACK_ENVIRONMENT instead (it's more descriptive).
 		"TESTING_ENVIRONMENT": StackEnvironment,
 	}, nil
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 67f136d4e05a..03b48240814b 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -11,7 +11,6 @@ services:
     working_dir: /go/src/github.com/elastic/beats/metricbeat
     volumes:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
-      - ${GOMODCACHE}:/go/pkg/mod/
       - /var/run/docker.sock:/var/run/docker.sock
     network_mode: host
     command: make

From b16c71f745b57b30c65b891e040f1bbfc159ad39 Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Mon, 2 Dec 2024 03:12:35 +0100
Subject: [PATCH 06/10] test: bind gomodcache in kubernetes runner

---
 dev-tools/mage/kubernetes/kuberemote.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/dev-tools/mage/kubernetes/kuberemote.go b/dev-tools/mage/kubernetes/kuberemote.go
index e3062f00d1ad..dbdd34b62c8b 100644
--- a/dev-tools/mage/kubernetes/kuberemote.go
+++ b/dev-tools/mage/kubernetes/kuberemote.go
@@ -26,12 +26,14 @@ import (
 	"encoding/hex"
 	"encoding/pem"
 	"fmt"
+	"go/build"
 	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -491,6 +493,10 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 							Name:      "destdir",
 							MountPath: destDir,
 						},
+						{
+							Name:      "gomodcache",
+							MountPath: "/go/pkg/mod",
+						},
 					},
 				},
 			},
@@ -510,6 +516,14 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 						EmptyDir: &apiv1.EmptyDirVolumeSource{},
 					},
 				},
+				{
+					Name: "gomodcache",
+					VolumeSource: apiv1.VolumeSource{
+						HostPath: &apiv1.HostPathVolumeSource{
+							Path: filepath.Join(build.Default.GOPATH, "pkg", "mod"),
+						},
+					},
+				},
 			},
 		},
 	}

From 20dde32ada594d3d31169c1a77f1bb856488213b Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Mon, 2 Dec 2024 04:06:53 +0100
Subject: [PATCH 07/10] test: sync the gomodcache with rsync and fix linter
 issues

can't really bind using hostpath in k8s
---
 dev-tools/mage/kubernetes/kuberemote.go | 58 +++++++++++--------------
 1 file changed, 25 insertions(+), 33 deletions(-)

diff --git a/dev-tools/mage/kubernetes/kuberemote.go b/dev-tools/mage/kubernetes/kuberemote.go
index dbdd34b62c8b..5dff700cfd73 100644
--- a/dev-tools/mage/kubernetes/kuberemote.go
+++ b/dev-tools/mage/kubernetes/kuberemote.go
@@ -23,15 +23,15 @@ import (
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
-	"encoding/hex"
 	"encoding/pem"
 	"fmt"
 	"go/build"
 	"io"
-	"io/ioutil"
+	"log"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
@@ -129,11 +129,20 @@ func (r *KubeRemote) Run(env map[string]string, stdout io.Writer, stderr io.Writ
 	if err != nil {
 		return err
 	}
-	go f.ForwardPorts()
+	go func() {
+		if err := f.ForwardPorts(); err != nil {
+			log.Printf("forward port error: %v\n", err)
+		}
+	}()
 	<-readyChannel
 
 	// perform the rsync
-	r.rsync(randomPort, stderr, stderr)
+	if err := r.rsync(randomPort, stderr, stderr, r.syncDir, r.destDir); err != nil {
+		return fmt.Errorf("rsync failed: %w", err)
+	}
+	if err := r.rsync(randomPort, stderr, stderr, filepath.Join(build.Default.GOPATH, "pkg", "mod"), "/go/pkg/mod"); err != nil {
+		return fmt.Errorf("gomodcache rsync failed: %w", err)
+	}
 
 	// stop port forwarding
 	close(stopChannel)
@@ -260,14 +269,14 @@ func (r *KubeRemote) portForward(ports []string, stopChannel, readyChannel chan
 	}
 
 	path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", r.namespace, r.name)
-	hostIP := strings.TrimLeft(r.cfg.Host, "https://")
+	hostIP := strings.TrimPrefix(r.cfg.Host, "https://")
 	serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP}
 	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL)
 	return portforward.New(dialer, ports, stopChannel, readyChannel, stdout, stderr)
 }
 
 // rsync performs the rsync of sync directory to destination directory inside of the pod.
-func (r *KubeRemote) rsync(port uint16, stdout, stderr io.Writer) error {
+func (r *KubeRemote) rsync(port uint16, stdout, stderr io.Writer, src string, dst string) error {
 	privateKeyFile, err := createTempFile(r.privateKey)
 	if err != nil {
 		return err
@@ -276,8 +285,8 @@ func (r *KubeRemote) rsync(port uint16, stdout, stderr io.Writer) error {
 	rsh := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -p %d -i %s", port, privateKeyFile)
 	args := []string{
 		"--rsh", rsh,
-		"-a", fmt.Sprintf("%s/", r.syncDir),
-		fmt.Sprintf("root@localhost:%s", r.destDir),
+		"-a", fmt.Sprintf("%s/", src),
+		fmt.Sprintf("root@localhost:%s", dst),
 	}
 	cmd := exec.Command("rsync", args...)
 	cmd.Stdout = stdout
@@ -493,10 +502,6 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 							Name:      "destdir",
 							MountPath: destDir,
 						},
-						{
-							Name:      "gomodcache",
-							MountPath: "/go/pkg/mod",
-						},
 					},
 				},
 			},
@@ -516,14 +521,6 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 						EmptyDir: &apiv1.EmptyDirVolumeSource{},
 					},
 				},
-				{
-					Name: "gomodcache",
-					VolumeSource: apiv1.VolumeSource{
-						HostPath: &apiv1.HostPathVolumeSource{
-							Path: filepath.Join(build.Default.GOPATH, "pkg", "mod"),
-						},
-					},
-				},
 			},
 		},
 	}
@@ -556,7 +553,7 @@ func isInitContainersReady(pod *apiv1.Pod) bool {
 }
 
 func isScheduled(pod *apiv1.Pod) bool {
-	if &pod.Status != nil && len(pod.Status.Conditions) > 0 {
+	if len(pod.Status.Conditions) > 0 {
 		for _, condition := range pod.Status.Conditions {
 			if condition.Type == apiv1.PodScheduled &&
 				condition.Status == apiv1.ConditionTrue {
@@ -568,18 +565,15 @@ func isScheduled(pod *apiv1.Pod) bool {
 }
 
 func isInitContainersRunning(pod *apiv1.Pod) bool {
-	if &pod.Status != nil {
-		if len(pod.Spec.InitContainers) != len(pod.Status.InitContainerStatuses) {
+	if len(pod.Spec.InitContainers) != len(pod.Status.InitContainerStatuses) {
+		return false
+	}
+	for _, status := range pod.Status.InitContainerStatuses {
+		if status.State.Running == nil {
 			return false
 		}
-		for _, status := range pod.Status.InitContainerStatuses {
-			if status.State.Running == nil {
-				return false
-			}
-		}
-		return true
 	}
-	return false
+	return true
 }
 
 func containerRunning(containerName string) func(watch.Event) (bool, error) {
@@ -634,9 +628,7 @@ func podDone(event watch.Event) (bool, error) {
 }
 
 func createTempFile(content []byte) (string, error) {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	tmpfile, err := ioutil.TempFile("", hex.EncodeToString(randBytes))
+	tmpfile, err := os.CreateTemp("", "kuberemote-")
 	if err != nil {
 		return "", err
 	}

From 467cdd45cd3ec1b96f4ca1a600f99639ddf26354 Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Mon, 2 Dec 2024 04:35:50 +0100
Subject: [PATCH 08/10] test: rework bind approach with k8s runner

since it's a local cluster, use kind to bind the hostpath to the node
then bind the node path to the container path
---
 dev-tools/mage/kubernetes/kind.go       | 28 ++++++++++++++++++++-----
 dev-tools/mage/kubernetes/kuberemote.go | 15 ++++++++++---
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/dev-tools/mage/kubernetes/kind.go b/dev-tools/mage/kubernetes/kind.go
index 133f305fedd2..dd02227a5ea2 100644
--- a/dev-tools/mage/kubernetes/kind.go
+++ b/dev-tools/mage/kubernetes/kind.go
@@ -19,7 +19,8 @@ package kubernetes
 
 import (
 	"fmt"
-	"io/ioutil"
+	"go/build"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -66,8 +67,8 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 	}
 
 	clusterName := kubernetesClusterName()
-	stdOut := ioutil.Discard
-	stdErr := ioutil.Discard
+	stdOut := io.Discard
+	stdErr := io.Discard
 	if mg.Verbose() {
 		stdOut = os.Stdout
 		stdErr = os.Stderr
@@ -86,9 +87,26 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 		return err
 	}
 
+	cfg, err := os.CreateTemp("", "kind-")
+	if _, err := cfg.WriteString(fmt.Sprintf(`
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: %s
+        containerPath: /go/pkg/mod
+`, filepath.Join(build.Default.GOPATH, "pkg", "mod"))); err != nil {
+		return err
+	}
+	if err := cfg.Close(); err != nil {
+		return err
+	}
+
 	args := []string{
 		"create",
 		"cluster",
+		"--config", cfg.Name(),
 		"--name", clusterName,
 		"--kubeconfig", kubeConfig,
 		"--wait",
@@ -116,8 +134,8 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 
 // Teardown destroys the kubernetes cluster.
 func (m *KindIntegrationTestStep) Teardown(env map[string]string) error {
-	stdOut := ioutil.Discard
-	stdErr := ioutil.Discard
+	stdOut := io.Discard
+	stdErr := io.Discard
 	if mg.Verbose() {
 		stdOut = os.Stdout
 		stdErr = os.Stderr
diff --git a/dev-tools/mage/kubernetes/kuberemote.go b/dev-tools/mage/kubernetes/kuberemote.go
index 5dff700cfd73..0df91eb5790e 100644
--- a/dev-tools/mage/kubernetes/kuberemote.go
+++ b/dev-tools/mage/kubernetes/kuberemote.go
@@ -140,9 +140,6 @@ func (r *KubeRemote) Run(env map[string]string, stdout io.Writer, stderr io.Writ
 	if err := r.rsync(randomPort, stderr, stderr, r.syncDir, r.destDir); err != nil {
 		return fmt.Errorf("rsync failed: %w", err)
 	}
-	if err := r.rsync(randomPort, stderr, stderr, filepath.Join(build.Default.GOPATH, "pkg", "mod"), "/go/pkg/mod"); err != nil {
-		return fmt.Errorf("gomodcache rsync failed: %w", err)
-	}
 
 	// stop port forwarding
 	close(stopChannel)
@@ -502,6 +499,10 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 							Name:      "destdir",
 							MountPath: destDir,
 						},
+						{
+							Name:      "gomodcache",
+							MountPath: "/go/pkg/mod",
+						},
 					},
 				},
 			},
@@ -521,6 +522,14 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 						EmptyDir: &apiv1.EmptyDirVolumeSource{},
 					},
 				},
+				{
+					Name: "gomodcache",
+					VolumeSource: apiv1.VolumeSource{
+						HostPath: &apiv1.HostPathVolumeSource{
+							Path: "/go/pkg/mod",
+						},
+					},
+				},
 			},
 		},
 	}

From 83978e63fbc8f03ff86f71b2eaed9fd90251bf44 Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Mon, 2 Dec 2024 04:42:12 +0100
Subject: [PATCH 09/10] lint: remove unused imports

---
 dev-tools/mage/kubernetes/kuberemote.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dev-tools/mage/kubernetes/kuberemote.go b/dev-tools/mage/kubernetes/kuberemote.go
index 0df91eb5790e..fb028dc94db3 100644
--- a/dev-tools/mage/kubernetes/kuberemote.go
+++ b/dev-tools/mage/kubernetes/kuberemote.go
@@ -25,7 +25,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"go/build"
 	"io"
 	"log"
 	"net"
@@ -33,7 +32,6 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 	"time"
 

From efe3d2f1a7a9849f8f745fb1f16e0a19483cef4b Mon Sep 17 00:00:00 2001
From: kruskal <99559985+kruskall@users.noreply.github.com>
Date: Mon, 2 Dec 2024 05:22:27 +0100
Subject: [PATCH 10/10] lint: fix linter issues

---
 dev-tools/mage/kubernetes/kind.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dev-tools/mage/kubernetes/kind.go b/dev-tools/mage/kubernetes/kind.go
index dd02227a5ea2..43ef37e10a4a 100644
--- a/dev-tools/mage/kubernetes/kind.go
+++ b/dev-tools/mage/kubernetes/kind.go
@@ -88,6 +88,9 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 	}
 
 	cfg, err := os.CreateTemp("", "kind-")
+	if err != nil {
+		return err
+	}
 	if _, err := cfg.WriteString(fmt.Sprintf(`
 apiVersion: kind.x-k8s.io/v1alpha4
 kind: Cluster