diff --git a/api/common/v1alpha1/ref.go b/api/common/v1alpha1/ref.go index f8ed8f9c8464..a838fca55cb3 100644 --- a/api/common/v1alpha1/ref.go +++ b/api/common/v1alpha1/ref.go @@ -43,9 +43,17 @@ func (k TargetRefKind) Less(o TargetRefKind) bool { } func AllTargetRefKinds() []TargetRefKind { - return maps.Keys(order) + keys := maps.Keys(order) + sort.Sort(TargetRefKindSlice(keys)) + return keys } +type TargetRefKindSlice []TargetRefKind + +func (x TargetRefKindSlice) Len() int { return len(x) } +func (x TargetRefKindSlice) Less(i, j int) bool { return string(x[i]) < string(x[j]) } +func (x TargetRefKindSlice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + // TargetRef defines structure that allows attaching policy to various objects type TargetRef struct { // Kind of the referenced resource diff --git a/api/common/v1alpha1/zz_generated.deepcopy.go b/api/common/v1alpha1/zz_generated.deepcopy.go index 43e6c9eac4cf..b17fd9576b21 100644 --- a/api/common/v1alpha1/zz_generated.deepcopy.go +++ b/api/common/v1alpha1/zz_generated.deepcopy.go @@ -144,3 +144,22 @@ func (in *TargetRef) DeepCopy() *TargetRef { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in TargetRefKindSlice) DeepCopyInto(out *TargetRefKindSlice) { + { + in := &in + *out = make(TargetRefKindSlice, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetRefKindSlice. +func (in TargetRefKindSlice) DeepCopy() TargetRefKindSlice { + if in == nil { + return nil + } + out := new(TargetRefKindSlice) + in.DeepCopyInto(out) + return *out +} diff --git a/app/cni/pkg/install/main.go b/app/cni/pkg/install/main.go index 02bc4402eda1..2295e48fac70 100644 --- a/app/cni/pkg/install/main.go +++ b/app/cni/pkg/install/main.go @@ -129,7 +129,9 @@ func setupChainedPlugin(mountedCniNetDir, cniConfName, kumaCniConfig string) err backoff := retry.WithMaxDuration(5*time.Minute, retry.NewConstant(time.Second)) err := retry.Do(context.Background(), backoff, func(ctx context.Context) error { if !files.FileExists(cniConfPath) { - return retry.RetryableError(errors.Errorf("CNI config %v not found. Kuma CNI won't be chained", cniConfPath)) + err := errors.Errorf("CNI config '%s' not found.", cniConfPath) + log.Error(err, "error chaining Kuma CNI config, will retry...") + return retry.RetryableError(err) } return nil }) diff --git a/app/kumactl/cmd/completion/testdata/bash.golden b/app/kumactl/cmd/completion/testdata/bash.golden index 04e644dd1514..d973bd99d0a3 100644 --- a/app/kumactl/cmd/completion/testdata/bash.golden +++ b/app/kumactl/cmd/completion/testdata/bash.golden @@ -5595,6 +5595,41 @@ _kumactl_install_transparent-proxy() noun_aliases=() } +_kumactl_install_transparent-proxy-validator() +{ + last_command="kumactl_install_transparent-proxy-validator" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--ip-family-mode=") + two_word_flags+=("--ip-family-mode") + local_nonpersistent_flags+=("--ip-family-mode") + local_nonpersistent_flags+=("--ip-family-mode=") + flags+=("--validation-server-port=") + two_word_flags+=("--validation-server-port") + local_nonpersistent_flags+=("--validation-server-port") + local_nonpersistent_flags+=("--validation-server-port=") + flags+=("--api-timeout=") + two_word_flags+=("--api-timeout") + flags+=("--config-file=") + two_word_flags+=("--config-file") + flags+=("--log-level=") + two_word_flags+=("--log-level") + flags+=("--no-config") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + _kumactl_install() { last_command="kumactl_install" @@ -5607,6 +5642,7 @@ _kumactl_install() commands+=("demo") commands+=("observability") commands+=("transparent-proxy") + commands+=("transparent-proxy-validator") flags=() two_word_flags=() diff --git a/app/kumactl/cmd/install/install.go b/app/kumactl/cmd/install/install.go index 31414b754f7d..3b0b2bc5f0b7 100644 --- a/app/kumactl/cmd/install/install.go +++ b/app/kumactl/cmd/install/install.go @@ -19,6 +19,7 @@ func NewInstallCmd(pctx *kumactl_cmd.RootContext) *cobra.Command { cmd.AddCommand(newInstallObservability(pctx)) cmd.AddCommand(newInstallDemoCmd(&pctx.InstallDemoContext)) cmd.AddCommand(newInstallTransparentProxy()) + cmd.AddCommand(newInstallTransparentProxyValidator()) return cmd } diff --git a/app/kumactl/cmd/install/install_transparent_proxy_validator.go b/app/kumactl/cmd/install/install_transparent_proxy_validator.go new file mode 100644 index 000000000000..f6c1e2bbd0de --- /dev/null +++ b/app/kumactl/cmd/install/install_transparent_proxy_validator.go @@ -0,0 +1,56 @@ +//go:build !windows +// +build !windows + +package install + +import ( + "os" + + "github.com/spf13/cobra" + + "github.com/kumahq/kuma/pkg/core" + kuma_log "github.com/kumahq/kuma/pkg/log" + "github.com/kumahq/kuma/pkg/transparentproxy" + "github.com/kumahq/kuma/pkg/transparentproxy/validate" +) + +const defaultLogName = "validator" + +type transparentProxyValidatorArgs struct { + IpFamilyMode string + ValidationServerPort uint16 +} + +func newInstallTransparentProxyValidator() *cobra.Command { + args := transparentProxyValidatorArgs{ + IpFamilyMode: "dualstack", + ValidationServerPort: validate.ValidationServerPort, + } + cmd := &cobra.Command{ + Use: "transparent-proxy-validator", + Short: "Validates if transparent proxy has been set up successfully", + Long: `Validates the transparent proxy setup by testing if the applied +iptables rules are working correctly onto the pod. + +Follow the following steps to validate: + 1) install the transparent proxy using 'kumactl install transparent-proxy' + 2) run this command + +The result will be shown as text in stdout as well as the exit code. +`, + RunE: func(cmd *cobra.Command, _ []string) error { + log := core.NewLoggerTo(os.Stdout, kuma_log.InfoLevel).WithName(defaultLogName) + + ipv6Supported, _ := transparentproxy.HasLocalIPv6() + useIPv6 := ipv6Supported && args.IpFamilyMode == "ipv6" + + validator := validate.NewValidator(useIPv6, args.ValidationServerPort, log) + + return validator.Run() + }, + } + + cmd.Flags().StringVar(&args.IpFamilyMode, "ip-family-mode", args.IpFamilyMode, "The IP family mode that has enabled traffic redirection for when setting up transparent proxy. Can be 'dualstack' or 'ipv4'") + cmd.Flags().Uint16Var(&args.ValidationServerPort, "validation-server-port", args.ValidationServerPort, "The port that the validation server will listen") + return cmd +} diff --git a/app/kumactl/cmd/install/install_transparent_proxy_validator_windows.go b/app/kumactl/cmd/install/install_transparent_proxy_validator_windows.go new file mode 100644 index 000000000000..88c3cdfd4890 --- /dev/null +++ b/app/kumactl/cmd/install/install_transparent_proxy_validator_windows.go @@ -0,0 +1,16 @@ +package install + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +func newInstallTransparentProxyValidator() *cobra.Command { + return &cobra.Command{ + Use: "transparent-proxy-validator", + Short: "Validates if transparent proxy has been set up successfully", + RunE: func(_ *cobra.Command, _ []string) error { + return errors.New("This command is not supported on your operating system") + }, + } +} diff --git a/pkg/plugins/runtime/k8s/util/names.go b/pkg/plugins/runtime/k8s/util/names.go index 5579f97703b9..d2558e4fef85 100644 --- a/pkg/plugins/runtime/k8s/util/names.go +++ b/pkg/plugins/runtime/k8s/util/names.go @@ -1,7 +1,8 @@ package util const ( - KumaSidecarContainerName = "kuma-sidecar" - KumaGatewayContainerName = "kuma-gateway" - KumaInitContainerName = "kuma-init" + KumaSidecarContainerName = "kuma-sidecar" + KumaGatewayContainerName = "kuma-gateway" + KumaInitContainerName = "kuma-init" + KumaCniValidationContainerName = "kuma-validation" ) diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go index d67d412a2126..36e0c701ec92 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go +++ b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go @@ -160,9 +160,13 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error }) } + podRedirect, err := tp_k8s.NewPodRedirectForPod(pod) + if err != nil { + return err + } // init container if !i.cfg.CNIEnabled { - ic, err := i.NewInitContainer(pod) + ic, err := i.NewInitContainer(podRedirect) if err != nil { return err } @@ -180,6 +184,22 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error } else { pod.Spec.InitContainers = append(pod.Spec.InitContainers, patchedIc) } + } else if podRedirect.RedirectInbound { + ic := i.NewValidationContainer(podRedirect.IpFamilyMode, fmt.Sprintf("%d", podRedirect.RedirectPortInbound), sidecarTmp.Name) + patchedIc, err := i.applyCustomPatches(logger, ic, initPatches) + if err != nil { + return err + } + enabled, _, err := metadata.Annotations(pod.Annotations).GetEnabled(metadata.KumaInitFirst) + if err != nil { + return err + } + if enabled { + log.V(1).Info("injecting kuma cni validation container first because kuma.io/init-first is set") + pod.Spec.InitContainers = append([]kube_core.Container{patchedIc}, pod.Spec.InitContainers...) + } else { + pod.Spec.InitContainers = append(pod.Spec.InitContainers, patchedIc) + } } if i.sidecarContainersEnabled { @@ -364,12 +384,7 @@ func (i *KumaInjector) FindServiceAccountToken(podSpec *kube_core.PodSpec) *kube return nil } -func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container, error) { - podRedirect, err := tp_k8s.NewPodRedirectForPod(pod) - if err != nil { - return kube_core.Container{}, err - } - +func (i *KumaInjector) NewInitContainer(podRedirect *tp_k8s.PodRedirect) (kube_core.Container, error) { container := kube_core.Container{ Name: k8s_util.KumaInitContainerName, Image: i.cfg.InitContainer.Image, @@ -444,6 +459,46 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container return container, nil } +func (i *KumaInjector) NewValidationContainer(ipFamilyMode, inboundRedirectPort string, tmpVolumeName string) kube_core.Container { + container := kube_core.Container{ + Name: k8s_util.KumaCniValidationContainerName, + Image: i.cfg.InitContainer.Image, + ImagePullPolicy: kube_core.PullIfNotPresent, + Command: []string{"/usr/bin/kumactl", "install", "transparent-proxy-validator"}, + Args: []string{ + "--config-file", "/tmp/.kumactl", + "--ip-family-mode", ipFamilyMode, + "--validation-server-port", inboundRedirectPort, + }, + SecurityContext: &kube_core.SecurityContext{ + RunAsUser: &i.cfg.SidecarContainer.DataplaneContainer.UID, + RunAsGroup: &i.cfg.SidecarContainer.DataplaneContainer.GID, + Capabilities: &kube_core.Capabilities{ + Drop: []kube_core.Capability{ + "ALL", + }, + }, + }, + Resources: kube_core.ResourceRequirements{ + Limits: kube_core.ResourceList{ + kube_core.ResourceCPU: *kube_api.NewScaledQuantity(100, kube_api.Milli), + kube_core.ResourceMemory: *kube_api.NewScaledQuantity(50, kube_api.Mega), + }, + Requests: kube_core.ResourceList{ + kube_core.ResourceCPU: *kube_api.NewScaledQuantity(20, kube_api.Milli), + kube_core.ResourceMemory: *kube_api.NewScaledQuantity(20, kube_api.Mega), + }, + }, + } + container.VolumeMounts = append(container.VolumeMounts, kube_core.VolumeMount{ + Name: tmpVolumeName, + MountPath: "/tmp", + ReadOnly: false, + }) + + return container +} + func (i *KumaInjector) NewAnnotations(pod *kube_core.Pod, mesh string, logger logr.Logger) (map[string]string, error) { annotations := map[string]string{ metadata.KumaMeshAnnotation: mesh, // either user-defined value or default diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go index 03b9046ea2b8..308c6efcae26 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go +++ b/pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go @@ -708,6 +708,23 @@ spec: kuma.io/sidecar-injection: enabled`, cfgFile: "inject.config-ipv6-disabled.yaml", }), + Entry("34. cni enabled", testCase{ + num: "34", + mesh: ` + apiVersion: kuma.io/v1alpha1 + kind: Mesh + metadata: + name: default + spec: {}`, + namespace: ` + apiVersion: v1 + kind: Namespace + metadata: + name: default + labels: + kuma.io/sidecar-injection: enabled`, + cfgFile: "inject.config-cni.yaml", + }), ) DescribeTable("should not inject Kuma into a Pod", diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml new file mode 100644 index 000000000000..0382ee7fe963 --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.golden.yaml @@ -0,0 +1,175 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: kuma-cni + kubectl.kubernetes.io/default-container: busybox + kuma.io/envoy-admin-port: "9901" + kuma.io/init-first: "true" + kuma.io/mesh: default + kuma.io/sidecar-injected: "true" + kuma.io/sidecar-uid: "5678" + kuma.io/transparent-proxying: enabled + kuma.io/transparent-proxying-ebpf: disabled + kuma.io/transparent-proxying-inbound-port: "15055" + kuma.io/transparent-proxying-ip-family-mode: ipv4 + kuma.io/transparent-proxying-outbound-port: "15001" + kuma.io/virtual-probes: enabled + kuma.io/virtual-probes-port: "9000" + creationTimestamp: null + labels: + run: busybox + name: busybox +spec: + containers: + - args: + - run + - --log-level=info + - --concurrency=2 + env: + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUMA_CONTROL_PLANE_CA_CERT + value: | + -----BEGIN CERTIFICATE----- + MIIDMzCCAhugAwIBAgIQDhlInfsXYHamKN+29qnQvzANBgkqhkiG9w0BAQsFADAP + MQ0wCwYDVQQDEwRrdW1hMB4XDTIxMDQwMjEwMjIyNloXDTMxMDMzMTEwMjIyNlow + DzENMAsGA1UEAxMEa3VtYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB + AL4GGg+e2O7eA12F0F6v2rr8j2iVSFKepnZtL15lrCds6lqK50sXWOw8PKZp2ihA + XJVTSZzKasyLDTAR9VYQjTpE526EzvtdthSagf32QWW+wY6LMpEdexKOOCx2se55 + Rd97L33yYPfgX15OYliHPD056jjhotHLdN2lpy7+STDvQyRnXAu73YkY37Ed4hI4 + t/V6soHyEGNcDhm9p5fBGqz0njBbQkp2lTY5/kj42qB7Q6rCM2tbPsEMooeAAw5m + hyY4xj0tP9ucqlUz8gc+6o8HDNst8NeJXZktWn+COytjr/NzGgS22kvSDphisJot + o0FyoIOdAtxC1qxXXR+XuUUCAwEAAaOBijCBhzAOBgNVHQ8BAf8EBAMCAqQwHQYD + VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYD + VR0OBBYEFKRLkgIzX/OjKw9idepuQ/RMtT+AMCYGA1UdEQQfMB2CCWxvY2FsaG9z + dIcQ/QChIwAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAPs5yJZhoYlGW + CpA8dSISivM8/8iBNQ3fVwP63ft0EJLMVGu2RFZ4/UAJ/rUPSGN8xhXSk5+1d56a + /kaH9rX0HaRIHHlxA7iPUKxAj44x9LKmqPHToL3XlWY1AXzvicW9d+GM2FaQee+I + leaqLbz0AZvlnu271Z1CeaACuU9GljujvyiTTE9naHUEqvHgSpPtilJalyJ5/zIl + Z9F0+UWt3TOYMs5g+SCt0MwHTNbisbmewpcFFJzjt2kvtrc9t9dkF81xhcS19w7q + h1AeP3RRlLl7bv9EAVXEmIavih/29PA3ZSy+pbYNW7jNJHjMQ4hQ0E+xcCazU/O4 + ypWGaanvPg== + -----END CERTIFICATE----- + - name: KUMA_CONTROL_PLANE_URL + value: http://kuma-control-plane.kuma-system:5681 + - name: KUMA_DATAPLANE_DRAIN_TIME + value: 31s + - name: KUMA_DATAPLANE_MESH + value: default + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUMA_DNS_ENABLED + value: "false" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kuma/kuma-sidecar:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 212 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 260 + periodSeconds: 25 + successThreshold: 1 + timeoutSeconds: 23 + name: kuma-sidecar + readinessProbe: + failureThreshold: 112 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 11 + periodSeconds: 15 + successThreshold: 11 + timeoutSeconds: 13 + resources: + limits: + cpu: 1100m + ephemeral-storage: 1G + memory: 1512Mi + requests: + cpu: 150m + ephemeral-storage: 50M + memory: 164Mi + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 5678 + runAsUser: 5678 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + - mountPath: /tmp + name: kuma-sidecar-tmp + - image: busybox + name: busybox + resources: {} + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + initContainers: + - args: + - --config-file + - /tmp/.kumactl + - --ip-family-mode + - ipv4 + - --validation-server-port + - "15055" + command: + - /usr/bin/kumactl + - install + - transparent-proxy-validator + image: kuma/kuma-init:latest + imagePullPolicy: IfNotPresent + name: kuma-validation + resources: + limits: + cpu: 100m + memory: 50M + requests: + cpu: 20m + memory: 20M + securityContext: + capabilities: + drop: + - ALL + runAsGroup: 5678 + runAsUser: 5678 + volumeMounts: + - mountPath: /tmp + name: kuma-sidecar-tmp + - command: + - sh + - -c + - sleep 5 + image: busybox + name: init + resources: {} + volumes: + - name: default-token-w7dxf + secret: + secretName: default-token-w7dxf + - emptyDir: + sizeLimit: 10M + name: kuma-init-tmp + - emptyDir: + sizeLimit: 10M + name: kuma-sidecar-tmp +status: {} diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.input.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.input.yaml new file mode 100644 index 000000000000..f7a03aa0390f --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.34.input.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + name: busybox + labels: + run: busybox + annotations: + kuma.io/transparent-proxying-ip-family-mode: "ipv4" + kuma.io/init-first: "true" +spec: + volumes: + - name: default-token-w7dxf + secret: + secretName: default-token-w7dxf + containers: + - name: busybox + image: busybox + resources: {} + volumeMounts: + - name: default-token-w7dxf + readOnly: true + mountPath: "/var/run/secrets/kubernetes.io/serviceaccount" + initContainers: + - name: init + image: busybox + command: ['sh', '-c', 'sleep 5'] diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml new file mode 100644 index 000000000000..54cf2808e49b --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-cni.yaml @@ -0,0 +1,39 @@ +controlPlane: + apiServer: + url: http://kuma-control-plane.kuma-system:5681 +cniEnabled: true +sidecarContainer: + image: kuma/kuma-sidecar:latest + ipFamilyMode: ipv4 + redirectPortOutbound: 15001 + redirectPortInbound: 15055 + uid: 5678 + gid: 5678 + adminPort: 9901 + drainTime: 31s + + readinessProbe: + initialDelaySeconds: 11 + timeoutSeconds: 13 + periodSeconds: 15 + successThreshold: 11 + failureThreshold: 112 + livenessProbe: + initialDelaySeconds: 260 + timeoutSeconds: 23 + periodSeconds: 25 + failureThreshold: 212 + resources: + requests: + cpu: 150m + memory: 164Mi + limits: + cpu: 1100m + memory: 1512Mi +initContainer: + image: kuma/kuma-init:latest +virtualProbesEnabled: true +virtualProbesPort: 9000 +exceptions: + labels: + "openshift.io/deployer-pod-for.name": "*" diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml index b45f26f25f78..4e92d804a7f8 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ipv6-disabled.yaml @@ -30,7 +30,6 @@ sidecarContainer: cpu: 1100m memory: 1512Mi initContainer: - enabled: true image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml index 07b34c33a25c..17b3583ea624 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config-ports.yaml @@ -31,7 +31,6 @@ sidecarContainer: cpu: 1100m memory: 1512Mi initContainer: - enabled: true image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml index 82181946768a..d59369d29150 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.config.yaml @@ -36,7 +36,6 @@ sidecarContainer: cpu: 1100m memory: 1512Mi initContainer: - enabled: true image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml index 08740b49a337..7d0d7ecd4b73 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.ebpf.config.yaml @@ -31,7 +31,6 @@ sidecarContainer: cpu: 1100m memory: 1512Mi initContainer: - enabled: true image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml index de841300700e..29602d2f949a 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.env-vars.config.yaml @@ -35,7 +35,6 @@ sidecarContainer: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH: "/some/other/path" # a var that will be overridden by this config KUMA_DATAPLANE_DRAIN_TIME: 1s # a var that should be overridden by this config, but will be overridden by annotation on pod initContainer: - enabled: true image: kuma/kuma-init:latest virtualProbesEnabled: true virtualProbesPort: 9000 diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml index 3d81502c5d3c..d5a3f84c732f 100644 --- a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.high-resources.config.yaml @@ -30,7 +30,6 @@ sidecarContainer: cpu: 8500m memory: 1512Mi initContainer: - enabled: true image: kuma/kuma-init:latest builtinDNS: enabled: true diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml new file mode 100644 index 000000000000..c60b4e5105ab --- /dev/null +++ b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.sidecar-feature.34.golden.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: kuma-cni + kubectl.kubernetes.io/default-container: busybox + kuma.io/envoy-admin-port: "9901" + kuma.io/init-first: "true" + kuma.io/mesh: default + kuma.io/sidecar-injected: "true" + kuma.io/sidecar-uid: "5678" + kuma.io/transparent-proxying: enabled + kuma.io/transparent-proxying-ebpf: disabled + kuma.io/transparent-proxying-inbound-port: "15055" + kuma.io/transparent-proxying-ip-family-mode: ipv4 + kuma.io/transparent-proxying-outbound-port: "15001" + kuma.io/virtual-probes: enabled + kuma.io/virtual-probes-port: "9000" + creationTimestamp: null + labels: + run: busybox + name: busybox +spec: + containers: + - image: busybox + name: busybox + resources: {} + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + initContainers: + - args: + - --config-file + - /tmp/.kumactl + - --ip-family-mode + - ipv4 + - --validation-server-port + - "15055" + command: + - /usr/bin/kumactl + - install + - transparent-proxy-validator + image: kuma/kuma-init:latest + imagePullPolicy: IfNotPresent + name: kuma-validation + resources: + limits: + cpu: 100m + memory: 50M + requests: + cpu: 20m + memory: 20M + securityContext: + capabilities: + drop: + - ALL + runAsGroup: 5678 + runAsUser: 5678 + volumeMounts: + - mountPath: /tmp + name: kuma-sidecar-tmp + - command: + - sh + - -c + - sleep 5 + image: busybox + name: init + resources: {} + - args: + - run + - --log-level=info + - --concurrency=2 + env: + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUMA_CONTROL_PLANE_CA_CERT + value: | + -----BEGIN CERTIFICATE----- + MIIDMzCCAhugAwIBAgIQDhlInfsXYHamKN+29qnQvzANBgkqhkiG9w0BAQsFADAP + MQ0wCwYDVQQDEwRrdW1hMB4XDTIxMDQwMjEwMjIyNloXDTMxMDMzMTEwMjIyNlow + DzENMAsGA1UEAxMEa3VtYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB + AL4GGg+e2O7eA12F0F6v2rr8j2iVSFKepnZtL15lrCds6lqK50sXWOw8PKZp2ihA + XJVTSZzKasyLDTAR9VYQjTpE526EzvtdthSagf32QWW+wY6LMpEdexKOOCx2se55 + Rd97L33yYPfgX15OYliHPD056jjhotHLdN2lpy7+STDvQyRnXAu73YkY37Ed4hI4 + t/V6soHyEGNcDhm9p5fBGqz0njBbQkp2lTY5/kj42qB7Q6rCM2tbPsEMooeAAw5m + hyY4xj0tP9ucqlUz8gc+6o8HDNst8NeJXZktWn+COytjr/NzGgS22kvSDphisJot + o0FyoIOdAtxC1qxXXR+XuUUCAwEAAaOBijCBhzAOBgNVHQ8BAf8EBAMCAqQwHQYD + VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYD + VR0OBBYEFKRLkgIzX/OjKw9idepuQ/RMtT+AMCYGA1UdEQQfMB2CCWxvY2FsaG9z + dIcQ/QChIwAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAPs5yJZhoYlGW + CpA8dSISivM8/8iBNQ3fVwP63ft0EJLMVGu2RFZ4/UAJ/rUPSGN8xhXSk5+1d56a + /kaH9rX0HaRIHHlxA7iPUKxAj44x9LKmqPHToL3XlWY1AXzvicW9d+GM2FaQee+I + leaqLbz0AZvlnu271Z1CeaACuU9GljujvyiTTE9naHUEqvHgSpPtilJalyJ5/zIl + Z9F0+UWt3TOYMs5g+SCt0MwHTNbisbmewpcFFJzjt2kvtrc9t9dkF81xhcS19w7q + h1AeP3RRlLl7bv9EAVXEmIavih/29PA3ZSy+pbYNW7jNJHjMQ4hQ0E+xcCazU/O4 + ypWGaanvPg== + -----END CERTIFICATE----- + - name: KUMA_CONTROL_PLANE_URL + value: http://kuma-control-plane.kuma-system:5681 + - name: KUMA_DATAPLANE_DRAIN_TIME + value: 31s + - name: KUMA_DATAPLANE_MESH + value: default + - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUMA_DNS_ENABLED + value: "false" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kuma/kuma-sidecar:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 212 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 260 + periodSeconds: 25 + successThreshold: 1 + timeoutSeconds: 23 + name: kuma-sidecar + readinessProbe: + failureThreshold: 112 + httpGet: + path: /ready + port: 9901 + initialDelaySeconds: 11 + periodSeconds: 15 + successThreshold: 11 + timeoutSeconds: 13 + resources: + limits: + cpu: 1100m + ephemeral-storage: 1G + memory: 1512Mi + requests: + cpu: 150m + ephemeral-storage: 50M + memory: 164Mi + restartPolicy: Always + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 5678 + runAsUser: 5678 + startupProbe: + httpGet: + path: /ready + port: 9901 + successThreshold: 1 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: default-token-w7dxf + readOnly: true + - mountPath: /tmp + name: kuma-sidecar-tmp + volumes: + - name: default-token-w7dxf + secret: + secretName: default-token-w7dxf + - emptyDir: + sizeLimit: 10M + name: kuma-init-tmp + - emptyDir: + sizeLimit: 10M + name: kuma-sidecar-tmp +status: {} diff --git a/pkg/transparentproxy/transparentproxy_v2.go b/pkg/transparentproxy/transparentproxy_v2.go index e200c60d315b..bff6f12c31ba 100644 --- a/pkg/transparentproxy/transparentproxy_v2.go +++ b/pkg/transparentproxy/transparentproxy_v2.go @@ -21,7 +21,7 @@ func V2() TransparentProxy { return &TransparentProxyV2{} } -func hasLocalIPv6() (bool, error) { +func HasLocalIPv6() (bool, error) { addrs, err := net.InterfaceAddrs() if err != nil { return false, err @@ -45,7 +45,7 @@ func ShouldEnableIPv6(port uint16) (bool, error) { return false, nil } - hasIPv6Address, err := hasLocalIPv6() + hasIPv6Address, err := HasLocalIPv6() if !hasIPv6Address || err != nil { return false, err } diff --git a/pkg/transparentproxy/validate/validator.go b/pkg/transparentproxy/validate/validator.go new file mode 100644 index 000000000000..9ef21dd7d112 --- /dev/null +++ b/pkg/transparentproxy/validate/validator.go @@ -0,0 +1,191 @@ +package validate + +import ( + "context" + "crypto/rand" + "fmt" + "math/big" + "net" + "net/netip" + "time" + + "github.com/go-logr/logr" + "github.com/sethvargo/go-retry" +) + +const ( + ValidationServerPort uint16 = 15006 + validationRetries = 10 + validationInterval = 1 * time.Second +) + +type Validator struct { + Config *Config + Logger logr.Logger +} + +type Config struct { + ServerListenIP netip.Addr + ServerListenPort uint16 + ClientConnectIP netip.Addr + ClientConnectPort uint16 + ClientRetryInterval time.Duration +} + +func NewValidator(useIpv6 bool, port uint16, logger logr.Logger) *Validator { + // Traffic to lo (but not 127.0.0.1) by sidecar will be redirected to KUMA_MESH_INBOUND_REDIRECT, so: + // connect to 127.0.0.6 should be redirected to 127.0.0.1 + // connect to ::6 should be redirected to ::1 + serverListenIP := netip.MustParseAddr("127.0.0.1") + clientConnectIP := netip.MustParseAddr("127.0.0.6") + + if useIpv6 { + serverListenIP = netip.MustParseAddr("::1") + clientConnectIP = netip.MustParseAddr("::6") + } + + return &Validator{ + Config: &Config{ + ServerListenIP: serverListenIP, + ServerListenPort: port, + ClientConnectIP: clientConnectIP, + ClientRetryInterval: validationInterval, + }, + Logger: logger, + } +} + +func (validator *Validator) Run() error { + validator.Logger.Info("starting iptables validation") + sExit := make(chan struct{}) + + sError := validator.runServer(sExit) + select { + case serverErr := <-sError: + if serverErr == nil { + serverErr = fmt.Errorf("server exited unexpectedly") + } + serverErr = fmt.Errorf("validation failed: %w", serverErr) + return serverErr + default: + } + + clientErr := validator.runClient() + if clientErr != nil { + clientErr = fmt.Errorf("validation failed, client failed to connect to the verification server: %w", clientErr) + close(sExit) + return clientErr + } else { + close(sExit) + validator.Logger.Info("Validation passed, iptables rules applied correctly") + return nil + } +} + +func (validator *Validator) runServer(sExit chan struct{}) chan error { + s := LocalServer{ + logger: validator.Logger, + config: validator.Config, + } + + sReady := make(chan struct{}, 1) + sError := make(chan error, 1) + go func() { + sError <- s.Run(sReady, sExit) + }() + + <-sReady + return sError +} + +type LocalServer struct { + logger logr.Logger + config *Config +} + +func (s *LocalServer) Run(readiness chan struct{}, exit chan struct{}) error { + addr := net.JoinHostPort(s.config.ServerListenIP.String(), fmt.Sprintf("%d", s.config.ServerListenPort)) + s.logger.Info(fmt.Sprintf("Listening on %v", addr)) + + config := &net.ListenConfig{} + l, err := config.Listen(context.Background(), "tcp", addr) + if err != nil { + s.logger.Error(err, fmt.Sprintf("error listening on %v", s.config.ServerListenIP)) + return err + } + + go s.handleTcpConnections(l, exit) + + readiness <- struct{}{} + <-exit + l.Close() + return nil +} + +func (s *LocalServer) handleTcpConnections(l net.Listener, cExit chan struct{}) { + for { + conn, err := l.Accept() + if err != nil { + s.logger.Error(err, "Listener failed to accept connection") + return + } + + s.logger.Error(err, "Server: a connection has been established") + _, _ = conn.Write([]byte(s.config.ServerListenIP.String())) + _ = conn.Close() + + select { + case <-cExit: + return + default: + } + } +} + +func (validator *Validator) runClient() error { + c := LocalClient{ServerIP: validator.Config.ClientConnectIP, ServerPort: validator.Config.ClientConnectPort} + backoff := retry.WithMaxRetries(validationRetries, retry.NewConstant(validator.Config.ClientRetryInterval)) + return retry.Do(context.Background(), backoff, func(ctx context.Context) error { + e := c.Run() + if e != nil { + validator.Logger.Error(e, "Client failed to connect to server") + return retry.RetryableError(e) + } + validator.Logger.Info("Client: connection established") + return nil + }) +} + +type LocalClient struct { + ServerIP netip.Addr + ServerPort uint16 +} + +func (c *LocalClient) Run() error { + laddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0") + if err != nil { + return err + } + if c.ServerIP.Is6() { + laddr, err = net.ResolveTCPAddr("tcp", "[::1]:0") + if err != nil { + return err + } + } + + // connections to all ports should be redirected to the server + // we support a pre-configured port for testing purposes + if c.ServerPort == 0 { + randPort, _ := rand.Int(rand.Reader, big.NewInt(10000)) + c.ServerPort = uint16(20000 + randPort.Int64()) + } + serverAddr := net.JoinHostPort(c.ServerIP.String(), fmt.Sprintf("%d", c.ServerPort)) + + dailer := net.Dialer{LocalAddr: laddr, Timeout: 50 * time.Millisecond} + conn, err := dailer.Dial("tcp", serverAddr) + if err != nil { + return err + } + conn.Close() + return nil +} diff --git a/pkg/transparentproxy/validate/validator_suite_test.go b/pkg/transparentproxy/validate/validator_suite_test.go new file mode 100644 index 000000000000..796283deb66e --- /dev/null +++ b/pkg/transparentproxy/validate/validator_suite_test.go @@ -0,0 +1,11 @@ +package validate + +import ( + "testing" + + "github.com/kumahq/kuma/pkg/test" +) + +func TestTransparentProxyValidator(t *testing.T) { + test.RunSpecs(t, "Transparent Proxy Validator") +} diff --git a/pkg/transparentproxy/validate/validator_test.go b/pkg/transparentproxy/validate/validator_test.go new file mode 100644 index 000000000000..54e29c5f09b7 --- /dev/null +++ b/pkg/transparentproxy/validate/validator_test.go @@ -0,0 +1,74 @@ +package validate + +import ( + "net/netip" + "os" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kumahq/kuma/pkg/core" + kuma_log "github.com/kumahq/kuma/pkg/log" +) + +var _ = Describe("Should Validate iptables rules", func() { + Describe("generate default validator config", func() { + It("ipv4", func() { + // when + validator := createValidator(false) + + // then + Expect(validator.Config.ServerListenIP.String()).To(Equal("127.0.0.1")) + Expect(validator.Config.ServerListenPort).To(Equal(uint16(15006))) + }) + + It("ipv6", func() { + // when + validator := createValidator(true) + + // then + serverIP := validator.Config.ServerListenIP.String() + Expect(serverIP).To(Equal("::1")) + + splitByCon := strings.Split(serverIP, ":") + Expect(len(splitByCon)).To(BeNumerically(">", 2)) + }) + }) + + It("should return pass when connect to correct address", func() { + // when + validator := createValidator(false) + ipAddr := "127.0.0.1" + addr, _ := netip.ParseAddr(ipAddr) + validator.Config.ServerListenIP = addr + validator.Config.ClientConnectIP = addr + validator.Config.ClientConnectPort = ValidationServerPort + + err := validator.Run() + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return fail when no iptables rules setup", func() { + // given + validator := createValidator(false) + validator.Config.ClientRetryInterval = 30 * time.Millisecond // just to make test faster and there should be no flakiness here because the connection will never establish successfully without the redirection + + // when + err := validator.Run() + + // then + Expect(err).To(HaveOccurred()) + errMsg := err.Error() + containsTimeout := strings.Contains(errMsg, "i/o timeout") + containsRefused := strings.Contains(errMsg, "connection refused") + Expect(containsTimeout || containsRefused).To(BeTrue()) + }) +}) + +func createValidator(ipv6Enabled bool) *Validator { + return NewValidator(ipv6Enabled, ValidationServerPort, core.NewLoggerTo(os.Stdout, kuma_log.InfoLevel).WithName("validator")) +} diff --git a/test/e2e/cni/taint_controller_race.go b/test/e2e/cni/taint_controller_race.go index 3bebbaeb0326..cefa9dedc2aa 100644 --- a/test/e2e/cni/taint_controller_race.go +++ b/test/e2e/cni/taint_controller_race.go @@ -74,6 +74,7 @@ metadata: Expect(k8sCluster.LoadImages( Config.KumaDPImageRepo, Config.KumaCNIImageRepo, + Config.KumaInitImageRepo, Config.KumaUniversalImageRepo, )).ToNot(HaveOccurred())