nodeinstaller: add nydus-pull container

.github/workflows/release.yml

@@ -251,6 +251,7 @@ jobs:
           serviceMeshImg=$(nix run .#containers.push-service-mesh-proxy -- "$container_registry/contrast/service-mesh-proxy")
           tardevSnapshotterImg=$(nix run .#containers.push-tardev-snapshotter -- "$container_registry/contrast/tardev-snapshotter")
           nydusSnapshotterImg=$(nix run .#containers.push-nydus-snapshotter -- "$container_registry/contrast/nydus-snapshotter")
+          nydusPullImg=$(nix run .#containers.push-nydus-pull -- "$container_registry/contrast/nydus-pull")
           cryptsetupImg=$(nix run .#containers.push-cryptsetup -- "$container_registry/contrast/cryptsetup")
           echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV"
           echo "nodeInstallerMsftImg=$nodeInstallerMsftImg" | tee -a "$GITHUB_ENV"
@@ -259,6 +260,7 @@ jobs:
           echo "serviceMeshImg=$serviceMeshImg" | tee -a "$GITHUB_ENV"
           echo "tardevSnapshotterImg=$tardevSnapshotterImg" | tee -a "$GITHUB_ENV"
           echo "nydusSnapshotterImg=$nydusSnapshotterImg" | tee -a "$GITHUB_ENV"
+          echo "nydusPullImg=$nydusPullImg" | tee -a "$GITHUB_ENV"
           echo "cryptsetupImg=$cryptsetupImg" | tee -a "$GITHUB_ENV"
       - name: Add tag to Coordinator image
         run: |
@@ -272,6 +274,7 @@ jobs:
           echo "nodeInstallerKataImgTagged=$(tag "$nodeInstallerKataImg")" | tee -a "$GITHUB_ENV"
           echo "initializerImgTagged=$(tag "$initializerImg")" | tee -a "$GITHUB_ENV"
           echo "serviceMeshImgTagged=$(tag "$serviceMeshImg")" | tee -a "$GITHUB_ENV"
+          echo "nydusPullImgTagged=$(tag "$nydusPullImg")" | tee -a "$GITHUB_ENV"
           echo "cryptsetupImgTagged=$(tag "$cryptsetupImg")" | tee -a "$GITHUB_ENV"
 
           tardevVer=$(nix eval --impure --raw --expr "(builtins.getFlake \"git+file://$(pwd)?shallow=1\").outputs.legacyPackages.x86_64-linux.microsoft.tardev-snapshotter.version")
@@ -293,6 +296,7 @@ jobs:
             echo "ghcr.io/edgelesssys/contrast/node-installer-kata:latest=$nodeInstallerKataImgTagged"
             echo "ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest=$tardevSnapshotterImgTagged"
             echo "ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest=$nydusSnapshotterImgTagged"
+            echo "ghcr.io/edgelesssys/contrast/nydus-pull:latest=$nydusPullImgTagged"
             echo "ghcr.io/edgelesssys/contrast/cryptsetup:latest=$cryptsetupImgTagged"
           } > image-replacements.txt
       - name: Upload image replacements file (for main branch PR)

go.work

@@ -4,4 +4,5 @@ use (
 	.
 	./service-mesh
 	./tools/tdx-measure
+	./tools/nydus-pull
 )

internal/kuberesource/mutators.go

@@ -13,6 +13,7 @@ import (
 	applybatchv1 "k8s.io/client-go/applyconfigurations/batch/v1"
 	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
 	applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1"
+	applyrbacv1 "k8s.io/client-go/applyconfigurations/rbac/v1"
 )
 
 const (
@@ -293,6 +294,10 @@ func PatchNamespaces(resources []any, namespace string) []any {
 			r.Namespace = nsPtr
 		case *applycorev1.ServiceAccountApplyConfiguration:
 			r.Namespace = nsPtr
+		case *applyrbacv1.ClusterRoleBindingApplyConfiguration:
+			for i := range len(r.Subjects) {
+				r.Subjects[i].Namespace = nsPtr
+			}
 		}
 	}
 	return resources

internal/kuberesource/parts.go

@@ -15,6 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
 	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
+	applyrbacv1 "k8s.io/client-go/applyconfigurations/rbac/v1"
 )
 
 // ContrastRuntimeClass creates a new RuntimeClassConfig.
@@ -39,6 +40,9 @@ func ContrastRuntimeClass(platform platforms.Platform) (*RuntimeClassConfig, err
 // NodeInstallerConfig wraps a DaemonSetApplyConfiguration for a node installer.
 type NodeInstallerConfig struct {
 	*applyappsv1.DaemonSetApplyConfiguration
+	*applycorev1.ServiceAccountApplyConfiguration
+	*applyrbacv1.ClusterRoleApplyConfiguration
+	*applyrbacv1.ClusterRoleBindingApplyConfiguration
 }
 
 // NodeInstaller constructs a node installer daemon set.
@@ -118,33 +122,70 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 			),
 	}
 
+	nydusPull := Container().
+		WithName("nydus-pull").
+		WithImage("ghcr.io/edgelesssys/contrast/nydus-pull:latest").
+		WithArgs(runtimeHandler).
+		WithEnv(
+			EnvVar().
+				WithName("NODE_NAME").
+				WithValueFrom(
+					applycorev1.EnvVarSource().
+						WithFieldRef(
+							applycorev1.ObjectFieldSelector().
+								WithFieldPath("spec.nodeName"),
+						),
+				),
+		).
+		WithVolumeMounts(
+			VolumeMount().
+				WithName("containerd-socket").
+				WithMountPath("/run/containerd/containerd.sock"),
+		)
+
 	var nodeInstallerImageURL string
-	var snapshotter *applycorev1.ContainerApplyConfiguration
+	var containers []*applycorev1.ContainerApplyConfiguration
 	var snapshotterVolumes []*applycorev1.VolumeApplyConfiguration
 	switch platform {
 	case platforms.AKSCloudHypervisorSNP:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest"
-		snapshotter = tardevSnapshotter
+		containers = append(containers, tardevSnapshotter)
 		snapshotterVolumes = tardevSnapshotterVolumes
 	case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.MetalQEMUSNPGPU:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
-		snapshotter = nydusSnapshotter
-		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
-			WithName("var-lib-containerd").
-			WithHostPath(HostPathVolumeSource().
-				WithPath("/var/lib/containerd").
-				WithType(corev1.HostPathDirectory),
-			))
+		containers = append(containers, nydusSnapshotter, nydusPull)
+		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes,
+			Volume().
+				WithName("var-lib-containerd").
+				WithHostPath(HostPathVolumeSource().
+					WithPath("/var/lib/containerd").
+					WithType(corev1.HostPathDirectory),
+				),
+			Volume().
+				WithName("containerd-socket").
+				WithHostPath(HostPathVolumeSource().
+					WithPath("/run/containerd/containerd.sock").
+					WithType(corev1.HostPathSocket),
+				),
+		)
 		snapshotterVolumes = nydusSnapshotterVolumes
 	case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
-		snapshotter = nydusSnapshotter
-		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
-			WithName("var-lib-containerd").
-			WithHostPath(HostPathVolumeSource().
-				WithPath("/var/lib/rancher/k3s/agent/containerd").
-				WithType(corev1.HostPathDirectory),
-			))
+		containers = append(containers, nydusSnapshotter, nydusPull)
+		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes,
+			Volume().
+				WithName("var-lib-containerd").
+				WithHostPath(HostPathVolumeSource().
+					WithPath("/var/lib/rancher/k3s/agent/containerd").
+					WithType(corev1.HostPathDirectory),
+				),
+			Volume().
+				WithName("containerd-socket").
+				WithHostPath(HostPathVolumeSource().
+					WithPath("/run/k3s/containerd/containerd.sock").
+					WithType(corev1.HostPathSocket),
+				),
+		)
 		snapshotterVolumes = nydusSnapshotterVolumes
 	default:
 		return nil, fmt.Errorf("unsupported platform %q", platform)
@@ -163,6 +204,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 					"contrast.edgeless.systems/platform": platform.String(),
 				}).
 				WithSpec(PodSpec().
+					WithServiceAccountName("nodeinstaller-serviceaccount").
 					WithHostPID(true).
 					WithInitContainers(Container().
 						WithName("installer").
@@ -177,7 +219,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 						WithCommand("/bin/node-installer", platform.String()),
 					).
 					WithContainers(
-						snapshotter,
+						containers...,
 					).
 					WithVolumes(append(
 						snapshotterVolumes,
@@ -193,7 +235,36 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 			),
 		)
 
-	return &NodeInstallerConfig{d}, nil
+	serviceAccount := applycorev1.ServiceAccount("nodeinstaller-serviceaccount", "")
+
+	clusterRole := applyrbacv1.ClusterRole("nodeinstaller-clusterrole").
+		WithRules(
+			applyrbacv1.PolicyRule().
+				WithAPIGroups("").
+				WithResources("pods").
+				WithVerbs("watch"),
+		)
+
+	clusterRoleBinding := applyrbacv1.ClusterRoleBinding("nodeinstaller-clusterrole-binding").
+		WithSubjects(
+			applyrbacv1.Subject().
+				WithKind("ServiceAccount").
+				WithName("nodeinstaller-serviceaccount").
+				WithNamespace(namespace),
+		).
+		WithRoleRef(
+			applyrbacv1.RoleRef().
+				WithKind("ClusterRole").
+				WithName("nodeinstaller-clusterrole").
+				WithAPIGroup("rbac.authorization.k8s.io"),
+		)
+
+	return &NodeInstallerConfig{
+		DaemonSetApplyConfiguration:          d,
+		ServiceAccountApplyConfiguration:     serviceAccount,
+		ClusterRoleApplyConfiguration:        clusterRole,
+		ClusterRoleBindingApplyConfiguration: clusterRoleBinding,
+	}, nil
 }
 
 // PortForwarderConfig wraps a PodApplyConfiguration for a port forwarder.

internal/kuberesource/sets.go

@@ -47,6 +47,9 @@ func Runtime(platform platforms.Platform) ([]any, error) {
 	return []any{
 		runtimeClassApplyConfig,
 		nodeInstaller.DaemonSetApplyConfiguration,
+		nodeInstaller.ServiceAccountApplyConfiguration,
+		nodeInstaller.ClusterRoleApplyConfiguration,
+		nodeInstaller.ClusterRoleBindingApplyConfiguration,
 	}, nil
 }
 

justfile

@@ -33,6 +33,9 @@ tardev-snapshotter: (push "tardev-snapshotter")
 # Build the nydus-snapshotter, containerize and push it.
 nydus-snapshotter: (push "nydus-snapshotter")
 
+# Build the nydus-pull container and push it.
+nydus-pull: (push "nydus-pull")
+
 default_cli := "contrast.cli"
 default_deploy_target := "openssl"
 default_platform := "${default_platform}"
@@ -49,6 +52,7 @@ node-installer platform=default_platform:
         ;;
         "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"Metal-QEMU-SNP-GPU"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
             just push "nydus-snapshotter"
+            just push "nydus-pull"
             just push "node-installer-kata"
         ;;
         "AKS-PEER-SNP")

packages/by-name/contrast/package.nix

@@ -176,7 +176,7 @@ buildGoModule rec {
         (fileset.difference (fileset.fileFilter (file: hasSuffix ".go" file.name) root) (
           fileset.unions [
             (path.append root "service-mesh")
-            (path.append root "tools/tdx-measure")
+            (path.append root "tools")
           ]
         ))
       ];

packages/by-name/nydus-pull/package.nix

@@ -0,0 +1,34 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+{ buildGoModule }:
+
+buildGoModule rec {
+  pname = "nydus-pull";
+  version = builtins.readFile ../../../version.txt;
+
+  src = ../../../tools/nydus-pull;
+
+  proxyVendor = true;
+  vendorHash = "sha256-bzCdcDfdivf52CerJ+9Nf5i+/laqjBWKNhhyLS8eBs4=";
+
+  subPackages = [ "." ];
+
+  CGO_ENABLED = 0;
+  ldflags = [
+    "-s"
+    "-X main.version=v${version}"
+  ];
+
+  preCheck = ''
+    export CGO_ENABLED=1
+  '';
+
+  checkPhase = ''
+    runHook preCheck
+    go test -race ./...
+    runHook postCheck
+  '';
+
+  meta.mainProgram = "nydus-pull";
+}

packages/containers.nix

@@ -178,6 +178,15 @@ let
         Cmd = [ "${lib.getExe pkgs.cloud-api-adaptor.entrypoint}" ];
       };
     };
+
+    nydus-pull = dockerTools.buildImage {
+      name = "nydus-pull";
+      tag = "v${pkgs.nydus-pull.version}";
+      copyToRoot = with dockerTools; [ caCertificates ];
+      config = {
+        Entrypoint = [ "${lib.getExe pkgs.nydus-pull}" ];
+      };
+    };
   };
 in
 containers

packages/scripts.nix

@@ -122,6 +122,8 @@
       nix-update --version=skip --flake legacyPackages.x86_64-linux.service-mesh
       echo "Updating vendorHash of contrast.cli package" >&2
       nix-update --version=skip --flake legacyPackages.x86_64-linux.contrast
+      echo "Updating vendorHash of nydus-pull package" >&2
+      nix-update --version=skip --flake legacyPackages.x86_64-linux.nydus-pull
 
       echo "Updating src hash of kata.kata-kernel-uvm.configfile" >&2
       nix-update --version=skip --flake legacyPackages.x86_64-linux.kata.kata-kernel-uvm.configfile