feat(transparent-proxy): support readOnlyFileSystem and drop all capa…

…bilities on kuma-init (#9688) --------- Signed-off-by: Jay Chen <[email protected]>
kumahq · Mar 26, 2024 · 5a09493 · 5a09493
1 parent eb56ab7
commit 5a09493
Show file tree

Hide file tree

Showing 81 changed files with 963 additions and 21 deletions.
diff --git a/app/kumactl/cmd/install/install_transparent_proxy.go b/app/kumactl/cmd/install/install_transparent_proxy.go
@@ -76,7 +76,7 @@ func newInstallTransparentProxy() *cobra.Command {
 		StoreFirewalld:                 false,
 		SkipDNSConntrackZoneSplit:      false,
 		EbpfEnabled:                    false,
-		EbpfProgramsSourcePath:         "/kuma/ebpf",
+		EbpfProgramsSourcePath:         "/tmp/kuma-ebpf",
 		EbpfBPFFSPath:                  "/sys/fs/bpf",
 		EbpfCgroupPath:                 "/sys/fs/cgroup",
 		EbpfTCAttachIface:              "",

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml
@@ -693,7 +693,7 @@ experimental:
     # -- Name of the network interface which TC programs should be attached to, we'll try to automatically determine it if empty
     tcAttachIface: ""
     # -- Path where compiled eBPF programs which will be installed can be found
-    programsSourcePath: /kuma/ebpf
+    programsSourcePath: /tmp/kuma-ebpf
   # -- If false, it uses legacy API for resource synchronization
   deltaKds: true
   # -- If true, enable native Kubernetes sidecars. This requires at least

diff --git a/...l/cmd/install/testdata/install-control-plane.tproxy-ebpf-experimental-enabled.golden.yaml b/...l/cmd/install/testdata/install-control-plane.tproxy-ebpf-experimental-enabled.golden.yaml
@@ -434,7 +434,7 @@ spec:
             - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_INSTANCE_IP_ENV_VAR_NAME
               value: "INSTANCE_IP"
             - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH
-              value: "/kuma/ebpf"
+              value: "/tmp/kuma-ebpf"
             - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_IMAGE
               value: "docker.io/kumahq/kuma-dp:0.0.1"
             - name: KUMA_RUNTIME_KUBERNETES_SERVICE_ACCOUNT_NAME

diff --git a/deployments/charts/kuma/README.md b/deployments/charts/kuma/README.md
@@ -204,7 +204,7 @@ A Helm chart for the Kuma Control Plane
 | experimental.ebpf.bpffsPath | string | `"/sys/fs/bpf"` | Path where BPF file system should be mounted |
 | experimental.ebpf.cgroupPath | string | `"/sys/fs/cgroup"` | Host's cgroup2 path |
 | experimental.ebpf.tcAttachIface | string | `""` | Name of the network interface which TC programs should be attached to, we'll try to automatically determine it if empty |
-| experimental.ebpf.programsSourcePath | string | `"/kuma/ebpf"` | Path where compiled eBPF programs which will be installed can be found |
+| experimental.ebpf.programsSourcePath | string | `"/tmp/kuma-ebpf"` | Path where compiled eBPF programs which will be installed can be found |
 | experimental.deltaKds | bool | `true` | If false, it uses legacy API for resource synchronization |
 | experimental.sidecarContainers | bool | `false` | If true, enable native Kubernetes sidecars. This requires at least Kubernetes v1.29 |
 | postgres.port | string | `"5432"` | Postgres port, password should be provided as a secret reference in "controlPlane.secrets" with the Env value "KUMA_STORE_POSTGRES_PASSWORD". Example: controlPlane:   secrets:     - Secret: postgres-postgresql       Key: postgresql-password       Env: KUMA_STORE_POSTGRES_PASSWORD |

diff --git a/deployments/charts/kuma/values.yaml b/deployments/charts/kuma/values.yaml
@@ -693,7 +693,7 @@ experimental:
     # -- Name of the network interface which TC programs should be attached to, we'll try to automatically determine it if empty
     tcAttachIface: ""
     # -- Path where compiled eBPF programs which will be installed can be found
-    programsSourcePath: /kuma/ebpf
+    programsSourcePath: /tmp/kuma-ebpf
   # -- If false, it uses legacy API for resource synchronization
   deltaKds: true
   # -- If true, enable native Kubernetes sidecars. This requires at least

diff --git a/docs/generated/raw/helm-values.yaml b/docs/generated/raw/helm-values.yaml
@@ -693,7 +693,7 @@ experimental:
     # -- Name of the network interface which TC programs should be attached to, we'll try to automatically determine it if empty
     tcAttachIface: ""
     # -- Path where compiled eBPF programs which will be installed can be found
-    programsSourcePath: /kuma/ebpf
+    programsSourcePath: /tmp/kuma-ebpf
   # -- If false, it uses legacy API for resource synchronization
   deltaKds: true
   # -- If true, enable native Kubernetes sidecars. This requires at least

diff --git a/docs/generated/raw/kuma-cp.yaml b/docs/generated/raw/kuma-cp.yaml
@@ -371,7 +371,7 @@ runtime:
         # when not specified, we will try to automatically determine it
         tcAttachIface: "" # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_TC_ATTACH_IFACE
         # Path where compiled eBPF programs are placed
-        programsSourcePath: /kuma/ebpf # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH
+        programsSourcePath: /tmp/kuma-ebpf # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH
       # IgnoredServiceSelectorLabels defines a list ignored labels in Service selector.
       # If Pod matches a Service with ignored labels, but does not match it fully, it gets Ignored inbound.
       # It is useful when you change Service selector and expect traffic to be sent immediately.

diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
@@ -371,7 +371,7 @@ runtime:
         # when not specified, we will try to automatically determine it
         tcAttachIface: "" # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_TC_ATTACH_IFACE
         # Path where compiled eBPF programs are placed
-        programsSourcePath: /kuma/ebpf # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH
+        programsSourcePath: /tmp/kuma-ebpf # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_PROGRAMS_SOURCE_PATH
       # IgnoredServiceSelectorLabels defines a list ignored labels in Service selector.
       # If Pod matches a Service with ignored labels, but does not match it fully, it gets Ignored inbound.
       # It is useful when you change Service selector and expect traffic to be sent immediately.

diff --git a/pkg/config/plugins/runtime/k8s/config.go b/pkg/config/plugins/runtime/k8s/config.go
@@ -95,7 +95,7 @@ func DefaultKubernetesRuntimeConfig() *KubernetesRuntimeConfig {
 				InstanceIPEnvVarName: "INSTANCE_IP",
 				BPFFSPath:            "/sys/fs/bpf",
 				CgroupPath:           "/sys/fs/cgroup",
-				ProgramsSourcePath:   "/kuma/ebpf",
+				ProgramsSourcePath:   "/tmp/kuma-ebpf",
 			},
 			IgnoredServiceSelectorLabels: []string{},
 			// topology labels that are useful for, for example, MeshLoadBalancingStrategy policy.

diff --git a/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml b/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml
@@ -20,7 +20,7 @@ injector:
     cgroupPath: /sys/fs/cgroup
     enabled: false
     instanceIPEnvVarName: INSTANCE_IP
-    programsSourcePath: /kuma/ebpf
+    programsSourcePath: /tmp/kuma-ebpf
   exceptions:
     labels:
       openshift.io/build.name: '*'

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go
@@ -95,6 +95,14 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error
 	if err != nil {
 		return err
 	}
+	initTmp := kube_core.Volume{
+		Name: "kuma-init-tmp",
+		VolumeSource: kube_core.VolumeSource{
+			EmptyDir: &kube_core.EmptyDirVolumeSource{
+				SizeLimit: kube_api.NewScaledQuantity(10, kube_api.Mega),
+			},
+		},
+	}
 	sidecarTmp := kube_core.Volume{
 		Name: "kuma-sidecar-tmp",
 		VolumeSource: kube_core.VolumeSource{
@@ -103,7 +111,7 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error
 			},
 		},
 	}
-	pod.Spec.Volumes = append(pod.Spec.Volumes, sidecarTmp)
+	pod.Spec.Volumes = append(pod.Spec.Volumes, initTmp, sidecarTmp)
 
 	container.VolumeMounts = append(container.VolumeMounts, kube_core.VolumeMount{
 		Name:      sidecarTmp.Name,
@@ -368,6 +376,14 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 		ImagePullPolicy: kube_core.PullIfNotPresent,
 		Command:         []string{"/usr/bin/kumactl", "install", "transparent-proxy"},
 		Args:            podRedirect.AsKumactlCommandLine(),
+		Env: []kube_core.EnvVar{
+			// iptables needs this lock file to be writable:
+			// source: https://git.netfilter.org/iptables/tree/iptables/xshared.c?h=v1.8.7#n258
+			{
+				Name:  "XTABLES_LOCKFILE",
+				Value: "/tmp/xtables.lock",
+			},
+		},
 		SecurityContext: &kube_core.SecurityContext{
 			RunAsUser:  new(int64), // way to get pointer to int64(0)
 			RunAsGroup: new(int64),
@@ -376,7 +392,11 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 					"NET_ADMIN",
 					"NET_RAW",
 				},
+				Drop: []kube_core.Capability{
+					"ALL",
+				},
 			},
+			ReadOnlyRootFilesystem: pointer.To(true),
 		},
 		Resources: kube_core.ResourceRequirements{
 			Limits: kube_core.ResourceList{
@@ -388,6 +408,9 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 				kube_core.ResourceMemory: *kube_api.NewScaledQuantity(20, kube_api.Mega),
 			},
 		},
+		VolumeMounts: []kube_core.VolumeMount{
+			{Name: "kuma-init-tmp", MountPath: "/tmp", ReadOnly: false},
+		},
 	}
 
 	if i.cfg.EBPF.Enabled {
@@ -412,10 +435,10 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 			kube_core.ResourceMemory: *kube_api.NewScaledQuantity(80, kube_api.Mega),
 		}
 
-		container.VolumeMounts = []kube_core.VolumeMount{
-			{Name: "sys-fs-cgroup", MountPath: i.cfg.EBPF.CgroupPath},
-			{Name: "bpf-fs", MountPath: i.cfg.EBPF.BPFFSPath, MountPropagation: &bidirectional},
-		}
+		container.VolumeMounts = append(container.VolumeMounts,
+			kube_core.VolumeMount{Name: "sys-fs-cgroup", MountPath: i.cfg.EBPF.CgroupPath},
+			kube_core.VolumeMount{Name: "bpf-fs", MountPath: i.cfg.EBPF.BPFFSPath, MountPropagation: &bidirectional},
+		)
 	}
 
 	return container, nil

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml
@@ -124,6 +124,8 @@ spec:
       readOnly: true
   initContainers:
   - args:
+    - --config-file
+    - /tmp/kumactl/config
     - --redirect-outbound-port
     - "15001"
     - --redirect-inbound=true
@@ -142,6 +144,9 @@ spec:
     - /usr/bin/kumactl
     - install
     - transparent-proxy
+    env:
+    - name: XTABLES_LOCKFILE
+      value: /tmp/xtables.lock
     image: kuma/kuma-init:latest
     imagePullPolicy: IfNotPresent
     name: kuma-init
@@ -157,12 +162,21 @@ spec:
         add:
         - NET_ADMIN
         - NET_RAW
+        drop:
+        - ALL
+      readOnlyRootFilesystem: true
       runAsGroup: 0
       runAsUser: 0
+    volumeMounts:
+    - mountPath: /tmp
+      name: kuma-init-tmp
   volumes:
   - name: default-token-w7dxf
     secret:
       secretName: default-token-w7dxf
+  - emptyDir:
+      sizeLimit: 10M
+    name: kuma-init-tmp
   - emptyDir:
       sizeLimit: 10M
     name: kuma-sidecar-tmp

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml
@@ -132,6 +132,8 @@ spec:
     name: init
     resources: {}
   - args:
+    - --config-file
+    - /tmp/kumactl/config
     - --redirect-outbound-port
     - "15001"
     - --redirect-inbound=true
@@ -150,6 +152,9 @@ spec:
     - /usr/bin/kumactl
     - install
     - transparent-proxy
+    env:
+    - name: XTABLES_LOCKFILE
+      value: /tmp/xtables.lock
     image: kuma/kuma-init:latest
     imagePullPolicy: IfNotPresent
     name: kuma-init
@@ -165,12 +170,21 @@ spec:
         add:
         - NET_ADMIN
         - NET_RAW
+        drop:
+        - ALL
+      readOnlyRootFilesystem: true
       runAsGroup: 0
       runAsUser: 0
+    volumeMounts:
+    - mountPath: /tmp
+      name: kuma-init-tmp
   volumes:
   - name: default-token-w7dxf
     secret:
       secretName: default-token-w7dxf
+  - emptyDir:
+      sizeLimit: 10M
+    name: kuma-init-tmp
   - emptyDir:
       sizeLimit: 10M
     name: kuma-sidecar-tmp

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml
@@ -186,6 +186,8 @@ spec:
   enableServiceLinks: true
   initContainers:
   - args:
+    - --config-file
+    - /tmp/kumactl/config
     - --redirect-outbound-port
     - "15001"
     - --redirect-inbound=true
@@ -204,6 +206,9 @@ spec:
     - /usr/bin/kumactl
     - install
     - transparent-proxy
+    env:
+    - name: XTABLES_LOCKFILE
+      value: /tmp/xtables.lock
     image: kuma/kuma-init:latest
     imagePullPolicy: IfNotPresent
     name: kuma-init
@@ -219,8 +224,14 @@ spec:
         add:
         - NET_ADMIN
         - NET_RAW
+        drop:
+        - ALL
+      readOnlyRootFilesystem: true
       runAsGroup: 0
       runAsUser: 0
+    volumeMounts:
+    - mountPath: /tmp
+      name: kuma-init-tmp
   nodeSelector:
     beta.kubernetes.io/os: linux
   priority: 2000000000
@@ -255,6 +266,9 @@ spec:
   - name: coredns-token-9gmrh
     secret:
       secretName: coredns-token-9gmrh
+  - emptyDir:
+      sizeLimit: 10M
+    name: kuma-init-tmp
   - emptyDir:
       sizeLimit: 10M
     name: kuma-sidecar-tmp

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml
@@ -124,6 +124,8 @@ spec:
       readOnly: true
   initContainers:
   - args:
+    - --config-file
+    - /tmp/kumactl/config
     - --redirect-outbound-port
     - "15001"
     - --redirect-inbound=true
@@ -142,6 +144,9 @@ spec:
     - /usr/bin/kumactl
     - install
     - transparent-proxy
+    env:
+    - name: XTABLES_LOCKFILE
+      value: /tmp/xtables.lock
     image: kuma/kuma-init:latest
     imagePullPolicy: IfNotPresent
     name: kuma-init
@@ -157,12 +162,21 @@ spec:
         add:
         - NET_ADMIN
         - NET_RAW
+        drop:
+        - ALL
+      readOnlyRootFilesystem: true
       runAsGroup: 0
       runAsUser: 0
+    volumeMounts:
+    - mountPath: /tmp
+      name: kuma-init-tmp
   volumes:
   - name: default-token-w7dxf
     secret:
       secretName: default-token-w7dxf
+  - emptyDir:
+      sizeLimit: 10M
+    name: kuma-init-tmp
   - emptyDir:
       sizeLimit: 10M
     name: kuma-sidecar-tmp

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml
@@ -118,6 +118,8 @@ spec:
     resources: {}
   initContainers:
   - args:
+    - --config-file
+    - /tmp/kumactl/config
     - --redirect-outbound-port
     - "15001"
     - --redirect-inbound=true
@@ -136,6 +138,9 @@ spec:
     - /usr/bin/kumactl
     - install
     - transparent-proxy
+    env:
+    - name: XTABLES_LOCKFILE
+      value: /tmp/xtables.lock
     image: kuma/kuma-init:latest
     imagePullPolicy: IfNotPresent
     name: kuma-init
@@ -151,9 +156,18 @@ spec:
         add:
         - NET_ADMIN
         - NET_RAW
+        drop:
+        - ALL
+      readOnlyRootFilesystem: true
       runAsGroup: 0
       runAsUser: 0
+    volumeMounts:
+    - mountPath: /tmp
+      name: kuma-init-tmp
   volumes:
+  - emptyDir:
+      sizeLimit: 10M
+    name: kuma-init-tmp
   - emptyDir:
       sizeLimit: 10M
     name: kuma-sidecar-tmp