feat(kuma-cni): add a init container to validate iptables rules are a…

…pplied (#9699)


---------

Signed-off-by: Jay Jijie Chen <[email protected]>
Co-authored-by: Krzysztof Słonka <[email protected]>

api/common/v1alpha1/ref.go

@@ -43,9 +43,17 @@ func (k TargetRefKind) Less(o TargetRefKind) bool {
 }
 
 func AllTargetRefKinds() []TargetRefKind {
-	return maps.Keys(order)
+	keys := maps.Keys(order)
+	sort.Sort(TargetRefKindSlice(keys))
+	return keys
 }
 
+type TargetRefKindSlice []TargetRefKind
+
+func (x TargetRefKindSlice) Len() int           { return len(x) }
+func (x TargetRefKindSlice) Less(i, j int) bool { return string(x[i]) < string(x[j]) }
+func (x TargetRefKindSlice) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
+
 // TargetRef defines structure that allows attaching policy to various objects
 type TargetRef struct {
 	// Kind of the referenced resource

app/cni/pkg/install/main.go

@@ -129,7 +129,9 @@ func setupChainedPlugin(mountedCniNetDir, cniConfName, kumaCniConfig string) err
 	backoff := retry.WithMaxDuration(5*time.Minute, retry.NewConstant(time.Second))
 	err := retry.Do(context.Background(), backoff, func(ctx context.Context) error {
 		if !files.FileExists(cniConfPath) {
-			return retry.RetryableError(errors.Errorf("CNI config %v not found. Kuma CNI won't be chained", cniConfPath))
+			err := errors.Errorf("CNI config '%s' not found.", cniConfPath)
+			log.Error(err, "error chaining Kuma CNI config, will retry...")
+			return retry.RetryableError(err)
 		}
 		return nil
 	})

app/kumactl/cmd/completion/testdata/bash.golden

@@ -5595,6 +5595,41 @@ _kumactl_install_transparent-proxy()
     noun_aliases=()
 }
 
+_kumactl_install_transparent-proxy-validator()
+{
+    last_command="kumactl_install_transparent-proxy-validator"
+
+    command_aliases=()
+
+    commands=()
+
+    flags=()
+    two_word_flags=()
+    local_nonpersistent_flags=()
+    flags_with_completion=()
+    flags_completion=()
+
+    flags+=("--ip-family-mode=")
+    two_word_flags+=("--ip-family-mode")
+    local_nonpersistent_flags+=("--ip-family-mode")
+    local_nonpersistent_flags+=("--ip-family-mode=")
+    flags+=("--validation-server-port=")
+    two_word_flags+=("--validation-server-port")
+    local_nonpersistent_flags+=("--validation-server-port")
+    local_nonpersistent_flags+=("--validation-server-port=")
+    flags+=("--api-timeout=")
+    two_word_flags+=("--api-timeout")
+    flags+=("--config-file=")
+    two_word_flags+=("--config-file")
+    flags+=("--log-level=")
+    two_word_flags+=("--log-level")
+    flags+=("--no-config")
+
+    must_have_one_flag=()
+    must_have_one_noun=()
+    noun_aliases=()
+}
+
 _kumactl_install()
 {
     last_command="kumactl_install"
@@ -5607,6 +5642,7 @@ _kumactl_install()
     commands+=("demo")
     commands+=("observability")
     commands+=("transparent-proxy")
+    commands+=("transparent-proxy-validator")
 
     flags=()
     two_word_flags=()

app/kumactl/cmd/install/install.go

@@ -19,6 +19,7 @@ func NewInstallCmd(pctx *kumactl_cmd.RootContext) *cobra.Command {
 	cmd.AddCommand(newInstallObservability(pctx))
 	cmd.AddCommand(newInstallDemoCmd(&pctx.InstallDemoContext))
 	cmd.AddCommand(newInstallTransparentProxy())
+	cmd.AddCommand(newInstallTransparentProxyValidator())
 
 	return cmd
 }

app/kumactl/cmd/install/install_transparent_proxy_validator.go

@@ -0,0 +1,56 @@
+//go:build !windows
+// +build !windows
+
+package install
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/kumahq/kuma/pkg/core"
+	kuma_log "github.com/kumahq/kuma/pkg/log"
+	"github.com/kumahq/kuma/pkg/transparentproxy"
+	"github.com/kumahq/kuma/pkg/transparentproxy/validate"
+)
+
+const defaultLogName = "validator"
+
+type transparentProxyValidatorArgs struct {
+	IpFamilyMode         string
+	ValidationServerPort uint16
+}
+
+func newInstallTransparentProxyValidator() *cobra.Command {
+	args := transparentProxyValidatorArgs{
+		IpFamilyMode:         "dualstack",
+		ValidationServerPort: validate.ValidationServerPort,
+	}
+	cmd := &cobra.Command{
+		Use:   "transparent-proxy-validator",
+		Short: "Validates if transparent proxy has been set up successfully",
+		Long: `Validates the transparent proxy setup by testing if the applied 
+iptables rules are working correctly onto the pod.
+
+Follow the following steps to validate:
+ 1) install the transparent proxy using 'kumactl install transparent-proxy'
+ 2) run this command
+
+The result will be shown as text in stdout as well as the exit code.
+`,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			log := core.NewLoggerTo(os.Stdout, kuma_log.InfoLevel).WithName(defaultLogName)
+
+			ipv6Supported, _ := transparentproxy.HasLocalIPv6()
+			useIPv6 := ipv6Supported && args.IpFamilyMode == "ipv6"
+
+			validator := validate.NewValidator(useIPv6, args.ValidationServerPort, log)
+
+			return validator.Run()
+		},
+	}
+
+	cmd.Flags().StringVar(&args.IpFamilyMode, "ip-family-mode", args.IpFamilyMode, "The IP family mode that has enabled traffic redirection for when setting up transparent proxy. Can be 'dualstack' or 'ipv4'")
+	cmd.Flags().Uint16Var(&args.ValidationServerPort, "validation-server-port", args.ValidationServerPort, "The port that the validation server will listen")
+	return cmd
+}

app/kumactl/cmd/install/install_transparent_proxy_validator_windows.go

@@ -0,0 +1,16 @@
+package install
+
+import (
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+func newInstallTransparentProxyValidator() *cobra.Command {
+	return &cobra.Command{
+		Use:   "transparent-proxy-validator",
+		Short: "Validates if transparent proxy has been set up successfully",
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return errors.New("This command is not supported on your operating system")
+		},
+	}
+}

pkg/plugins/runtime/k8s/util/names.go

@@ -1,7 +1,8 @@
 package util
 
 const (
-	KumaSidecarContainerName = "kuma-sidecar"
-	KumaGatewayContainerName = "kuma-gateway"
-	KumaInitContainerName    = "kuma-init"
+	KumaSidecarContainerName       = "kuma-sidecar"
+	KumaGatewayContainerName       = "kuma-gateway"
+	KumaInitContainerName          = "kuma-init"
+	KumaCniValidationContainerName = "kuma-validation"
 )

pkg/plugins/runtime/k8s/webhooks/injector/injector.go

@@ -160,9 +160,13 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error
 		})
 	}
 
+	podRedirect, err := tp_k8s.NewPodRedirectForPod(pod)
+	if err != nil {
+		return err
+	}
 	// init container
 	if !i.cfg.CNIEnabled {
-		ic, err := i.NewInitContainer(pod)
+		ic, err := i.NewInitContainer(podRedirect)
 		if err != nil {
 			return err
 		}
@@ -180,6 +184,22 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error
 		} else {
 			pod.Spec.InitContainers = append(pod.Spec.InitContainers, patchedIc)
 		}
+	} else if podRedirect.RedirectInbound {
+		ic := i.NewValidationContainer(podRedirect.IpFamilyMode, fmt.Sprintf("%d", podRedirect.RedirectPortInbound), sidecarTmp.Name)
+		patchedIc, err := i.applyCustomPatches(logger, ic, initPatches)
+		if err != nil {
+			return err
+		}
+		enabled, _, err := metadata.Annotations(pod.Annotations).GetEnabled(metadata.KumaInitFirst)
+		if err != nil {
+			return err
+		}
+		if enabled {
+			log.V(1).Info("injecting kuma cni validation container first because kuma.io/init-first is set")
+			pod.Spec.InitContainers = append([]kube_core.Container{patchedIc}, pod.Spec.InitContainers...)
+		} else {
+			pod.Spec.InitContainers = append(pod.Spec.InitContainers, patchedIc)
+		}
 	}
 
 	if i.sidecarContainersEnabled {
@@ -364,12 +384,7 @@ func (i *KumaInjector) FindServiceAccountToken(podSpec *kube_core.PodSpec) *kube
 	return nil
 }
 
-func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container, error) {
-	podRedirect, err := tp_k8s.NewPodRedirectForPod(pod)
-	if err != nil {
-		return kube_core.Container{}, err
-	}
-
+func (i *KumaInjector) NewInitContainer(podRedirect *tp_k8s.PodRedirect) (kube_core.Container, error) {
 	container := kube_core.Container{
 		Name:            k8s_util.KumaInitContainerName,
 		Image:           i.cfg.InitContainer.Image,
@@ -444,6 +459,46 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 	return container, nil
 }
 
+func (i *KumaInjector) NewValidationContainer(ipFamilyMode, inboundRedirectPort string, tmpVolumeName string) kube_core.Container {
+	container := kube_core.Container{
+		Name:            k8s_util.KumaCniValidationContainerName,
+		Image:           i.cfg.InitContainer.Image,
+		ImagePullPolicy: kube_core.PullIfNotPresent,
+		Command:         []string{"/usr/bin/kumactl", "install", "transparent-proxy-validator"},
+		Args: []string{
+			"--config-file", "/tmp/.kumactl",
+			"--ip-family-mode", ipFamilyMode,
+			"--validation-server-port", inboundRedirectPort,
+		},
+		SecurityContext: &kube_core.SecurityContext{
+			RunAsUser:  &i.cfg.SidecarContainer.DataplaneContainer.UID,
+			RunAsGroup: &i.cfg.SidecarContainer.DataplaneContainer.GID,
+			Capabilities: &kube_core.Capabilities{
+				Drop: []kube_core.Capability{
+					"ALL",
+				},
+			},
+		},
+		Resources: kube_core.ResourceRequirements{
+			Limits: kube_core.ResourceList{
+				kube_core.ResourceCPU:    *kube_api.NewScaledQuantity(100, kube_api.Milli),
+				kube_core.ResourceMemory: *kube_api.NewScaledQuantity(50, kube_api.Mega),
+			},
+			Requests: kube_core.ResourceList{
+				kube_core.ResourceCPU:    *kube_api.NewScaledQuantity(20, kube_api.Milli),
+				kube_core.ResourceMemory: *kube_api.NewScaledQuantity(20, kube_api.Mega),
+			},
+		},
+	}
+	container.VolumeMounts = append(container.VolumeMounts, kube_core.VolumeMount{
+		Name:      tmpVolumeName,
+		MountPath: "/tmp",
+		ReadOnly:  false,
+	})
+
+	return container
+}
+
 func (i *KumaInjector) NewAnnotations(pod *kube_core.Pod, mesh string, logger logr.Logger) (map[string]string, error) {
 	annotations := map[string]string{
 		metadata.KumaMeshAnnotation:                            mesh, // either user-defined value or default

pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go

@@ -708,6 +708,23 @@ spec:
                   kuma.io/sidecar-injection: enabled`,
 			cfgFile: "inject.config-ipv6-disabled.yaml",
 		}),
+		Entry("34. cni enabled", testCase{
+			num: "34",
+			mesh: `
+              apiVersion: kuma.io/v1alpha1
+              kind: Mesh
+              metadata:
+                name: default
+              spec: {}`,
+			namespace: `
+              apiVersion: v1
+              kind: Namespace
+              metadata:
+                name: default
+                labels:
+                  kuma.io/sidecar-injection: enabled`,
+			cfgFile: "inject.config-cni.yaml",
+		}),
 	)
 
 	DescribeTable("should not inject Kuma into a Pod",