diff --git a/.dockerignore b/.dockerignore index c0e601f..cc6a6de 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ .git/ **/.DS_Store bin/ -deploy/images/ test/ Dockerfile diff --git a/Dockerfile b/Dockerfile index 4d712ce..d112333 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,12 +13,11 @@ RUN go mod download # Copy the project's source code (except for whatever is included in the .dockerignore) COPY . . - # Build release -#RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o /hpk ./cmd/hpk-kubelet +#RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o /hpk ./cmd/hpk # Build dev -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -race -a -o /hpk ./cmd/hpk-kubelet +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -race -a -o /hpk ./cmd/hpk # Super minimal image just to package the hpk binary. It does not include anything. diff --git a/Makefile b/Makefile index 55310cc..654575c 100644 --- a/Makefile +++ b/Makefile @@ -84,12 +84,18 @@ help: ## Display this help ##@ Build -build: ## Build HPK binary - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(VERSION_FLAGS) -ldflags '-extldflags "-static"' -o bin/hpk-kubelet ./cmd/hpk - +build: hpk-kubelet hpk-pause ## Build HPK binary build-race: ## Build HPK binary with race condition detector GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(VERSION_FLAGS) -race -o bin/hpk-kubelet ./cmd/hpk +hpk-kubelet: + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(VERSION_FLAGS) -ldflags '-extldflags "-static"' -o bin/hpk-kubelet ./cmd/hpk + +hpk-pause: + GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $(VERSION_FLAGS) -ldflags '-extldflags "-static"' -o bin/hpk-pause ./cmd/pause + +docker-pause: + DOCKER_BUILDKIT=1 docker build . -t malvag/pause:apptainer -f deploy/images/pause-apptainer-agent/pause.apptainer.Dockerfile ##@ Deployment diff --git a/cmd/pause/main.go b/cmd/pause/main.go new file mode 100644 index 0000000..70d3aab --- /dev/null +++ b/cmd/pause/main.go @@ -0,0 +1,766 @@ +package main + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/carv-ics-forth/hpk/compute/endpoint" + "github.com/carv-ics-forth/hpk/compute/image" + "github.com/carv-ics-forth/hpk/compute/podhandler" + "github.com/rs/zerolog/log" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func getPodDetails(clientset *kubernetes.Clientset, namespace string, podID string) (*v1.Pod, error) { + // Create a context with a 5-minute timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pod, err := clientset.CoreV1().Pods(namespace).Get(ctx, podID, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return pod, nil +} + +func fileExists(filename string) bool { + if filename == "" { + return false + } + info, err := os.Stat(filename) + if err != nil { + if os.IsNotExist(err) { + return false + } + panic(err) + } + return !info.IsDir() // Ensure it's a file, not a directory +} + +func main() { + var podID string + var namespaceID string + var wg sync.WaitGroup + + flag.StringVar(&podID, "pod", "", "Pod ID to query Kubernetes") + flag.StringVar(&namespaceID, "namespace", "", "Pod ID to query Kubernetes") + flag.Parse() + + if podID == "" || namespaceID == "" { + log.Fatal().Msg("Please provide both the pod and namespace.") + } + + config, err := clientcmd.BuildConfigFromFlags("", filepath.Join("/k8s-data", "admin.conf")) + if err != nil { + log.Fatal().Err(err).Msg("Error building kubeconfig") + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatal().Err(err).Msg("Error creating Kubernetes client") + } + + // Main loop to keep asking for pod details + timeout := time.After(5 * time.Minute) + +acquire_pod_loop: + for { + select { + case <-timeout: + log.Error().Msg("Timeout reached. Exiting.") + os.Exit(1) + default: + _, err := getPodDetails(clientset, namespaceID, podID) + if err != nil { + log.Error().Err(err).Msg("Error getting pod details. Retrying...") + time.Sleep(5 * time.Second) // Adjust retry interval as needed + continue + } + + break acquire_pod_loop + } + } + + pod, err := getPodDetails(clientset, namespaceID, podID) + if err != nil { + panic(err) + } + + if err := prepareContainers(pod); err != nil { + log.Error().Err(err).Msg("Error preparing container environment") + return + } + + if len(pod.Spec.InitContainers) > 0 { + if err := handleInitContainers(pod, true); err != nil { + log.Error().Err(err).Msg("Error executing init containers") + return + } + } + + if err := handleContainers(pod, &wg, true); err != nil { + log.Error().Err(err).Msg("Error executing main containers") + return + } + + ctx, cancel := context.WithCancel(context.Background()) + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGCHLD) + + go func() { + for { + select { + case signo := <-signalChan: + switch signo { + case syscall.SIGINT, syscall.SIGTERM: + log.Info().Msgf("Received %v. Cleaning up...\n", signo) + cancel() // Initiate cleanup + + // Wait for containers to exit before fully exiting the goroutine + wg.Wait() + + case syscall.SIGCHLD: + // SIGCHLD handling - reap zombie processes + log.Info().Msg("Received SIGCHLD. Containers have terminated. ") + for { + pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil) + if pid <= 0 || err != nil { + break + } + log.Info().Msgf("pid: %v", pid) + } + cancel() // Initiate cleanup after SIGCHLD handling + + // Wait for containers to exit before fully exiting the goroutine + wg.Wait() + } + case <-ctx.Done(): + log.Info().Msg("Context was cancelled. Waiting for containers to exit...") + wg.Wait() // Ensure completion of all containers + log.Info().Msg("Containers have terminated. Exiting...") + return // Terminate the goroutine once containers finish + } + + } + }() + + log.Info().Msg("Containers have started. Now waiting on context or signals") + <-ctx.Done() + +} + +func prepareContainers(pod *v1.Pod) error { + if err := prepareDNS(pod); err != nil { + return fmt.Errorf("could not prepare DNS : %v", err) + } + if err := announceIP(pod); err != nil { + return fmt.Errorf("could not announce ip : %v", err) + } + if err := cleanEnvironment(); err != nil { + return fmt.Errorf("could not clear the environment : %v", err) + } + return nil +} + +func announceIP(pod *v1.Pod) error { + podKey := client.ObjectKeyFromObject(pod) + hpk := endpoint.HPK(pod.Annotations["workingDirectory"]) + podPath := hpk.Pod(podKey) + + addrs, err := net.InterfaceAddrs() + if err != nil { + return fmt.Errorf("could not get interfaces from host: %v", err) + } + var ipAddresses []string + for _, addr := range addrs { + // Add only if the address is an IP address + if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() && ipNet.IP.To4() != nil { + ipAddresses = append(ipAddresses, ipNet.IP.String()) + } + } + ipString := strings.Join(ipAddresses, " ") + + if err := os.WriteFile(podPath.IPAddressPath(), []byte(ipString), os.ModePerm); err != nil { + return fmt.Errorf("error writing to .ip file: %v", err) + } + return nil +} + +func cleanEnvironment() error { + + envVars := []string{ + "LD_LIBRARY_PATH", + "SINGULARITY_COMMAND", + "SINGULARITY_CONTAINER", + "SINGULARITY_ENVIRONMENT", + "SINGULARITY_NAME", + "APPTAINER_APPNAME", + "APPTAINER_COMMAND", + "APPTAINER_CONTAINER", + "APPTAINER_ENVIRONMENT", + "APPTAINER_NAME", + "APPTAINER_BIND", + "SINGULARITY_BIND", + } + + for _, name := range envVars { + if err := os.Unsetenv(name); err != nil { + return fmt.Errorf("could not clear the environment variable %s: %v", name, err) + } + } + return nil +} + +func prepareDNS(pod *v1.Pod) error { + if err := os.MkdirAll("/scratch/etc", 0644); err != nil { + return fmt.Errorf("could not create /scratch/etc folder: %v", err) + } + + kubeDNSIP := os.Getenv("KUBEDNS_IP") + if kubeDNSIP == "" { + return fmt.Errorf("KUBEDNS_IP environment variable not set") + } + + // Create and write to /scratch/etc/resolv.conf + resolvConfContent := fmt.Sprintf(`search %s.svc.cluster.local svc.cluster.local cluster.local + nameserver %s + options ndots:5`, pod.Namespace, kubeDNSIP) + + if err := os.WriteFile("/scratch/etc/resolv.conf", []byte(resolvConfContent), os.ModePerm); err != nil { + return fmt.Errorf("error writing to resolv.conf: %v", err) + } + + // Add hostname to /scratch/etc/hosts + hostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("error getting hostname: %v", err) + } + + addrs, err := net.InterfaceAddrs() + if err != nil { + return fmt.Errorf("could not get interfaces from host: %v", err) + } + + var ipAddresses []string + for _, addr := range addrs { + // Add only if the address is an IP address + if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() && ipNet.IP.To4() != nil { + ipAddresses = append(ipAddresses, ipNet.IP.String()) + } + } + ipString := strings.Join(ipAddresses, " ") + " " + hostname + + hostsContent := fmt.Sprintf("127.0.0.1 localhost\n%s \n", ipString) + + if err := os.WriteFile("/scratch/etc/hosts", []byte(hostsContent), os.ModePerm); err != nil { + return fmt.Errorf("error writing to hosts: %v", err) + } + DebugDNSInfo(resolvConfContent, hostsContent) + return nil +} + +func DebugDNSInfo(resolvConfContent string, hostsContent string) { + fmt.Printf("====================================================================\n%s\n", resolvConfContent) + fmt.Printf("====================================================================\n%s", hostsContent) + fmt.Println("====================================================================") + +} + +func handleInitContainers(pod *v1.Pod, hpkEnv bool) error { + isDebug := os.Getenv("DEBUG_MODE") == "true" + podKey := client.ObjectKeyFromObject(pod) + hpk := endpoint.HPK(pod.Annotations["workingDirectory"]) + podPath := hpk.Pod(podKey) + for _, container := range pod.Spec.InitContainers { + effectiSecurityContext := podhandler.DetermineEffectiveSecurityContext(pod, &container) + uid, gid := podhandler.DetermineEffectiveRunAsUser(effectiSecurityContext) + log.Info().Msgf("Spawning init container: %s", container.Name) + instanceName := fmt.Sprintf("%s_%s_%s", pod.GetNamespace(), pod.GetName(), container.Name) + + containerPath := podPath.Container(container.Name) + envFilePath := containerPath.EnvFilePath() + + // Environment File Handling + if fileExists(envFilePath) { + output, err := exec.Command("sh", "-c", envFilePath).CombinedOutput() + if err != nil { + return fmt.Errorf("error executing EnvFilePath: %v, output: %s", err, output) + } + envFileName := filepath.Join("/scratch", instanceName+".env") + if err := os.WriteFile(envFileName, output, 0644); err != nil { + return fmt.Errorf("error writing env file: %v", err) + } + } + + executionMode := "exec" + if container.Command == nil { + executionMode = "run" + } + + binds := make([]string, len(container.VolumeMounts)) + + // check the code from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet_pods.go#L196 + for i, mount := range container.VolumeMounts { + hostPath := filepath.Join(podPath.VolumeDir(), mount.Name) + + // subPath := mount.SubPath + // if mount.SubPathExpr != "" { + // subPath, err = helpers.ExpandContainerVolumeMounts(mount, h.podEnvVariables) + // if err != nil { + // compute.SystemPanic(err, "cannot expand env variables for container '%s' of pod '%s'", container, h.podKey) + // } + // } + + // if subPath != "" { + // if filepath.IsAbs(subPath) { + // return fmt.Errorf("error SubPath '%s' must not be an absolute path", subPath) + // } + + // subPathFile := filepath.Join(hostPath, subPath) + + // // mount the subpath + // hostPath = subPathFile + // } + + accessMode := "rw" + if mount.ReadOnly { + accessMode = "ro" + } + + binds[i] = hostPath + ":" + mount.MountPath + ":" + accessMode + } + + // Apptainer Command Construction + apptainerVerbosity := "--quiet" + if isDebug { + apptainerVerbosity = "--debug" + } + apptainerArgs := []string{ + apptainerVerbosity, executionMode, "--cleanenv", "--writable-tmpfs", "--no-mount", "home", "--unsquash", + } + if hpkEnv { + apptainerArgs = append(apptainerArgs, "--bind", "/scratch/etc/resolv.conf:/etc/resolv.conf,/scratch/etc/hosts:/etc/hosts") + } + if len(binds) > 0 { + bindArgs := &apptainerArgs[len(apptainerArgs)-1] + *bindArgs += "," + strings.Join(binds, ",") + } + if uid != 0 { + apptainerArgs = append(apptainerArgs, "--security", fmt.Sprintf("uid:%d,gid:%d", uid, uid), "--userns") + } + if gid != 0 { + apptainerArgs = append(apptainerArgs, "--security", fmt.Sprintf("gid:%d", gid), "--userns") + } + + if fileExists(envFilePath) { + apptainerArgs = append(apptainerArgs, "--env-file", filepath.Join("/scratch", instanceName+".env")) + } + + apptainerArgs = append(apptainerArgs, hpk.ImageDir()+image.ParseImageName(container.Image)) + apptainerArgs = append(apptainerArgs, container.Command...) + apptainerArgs = append(apptainerArgs, container.Args...) + + // Get the PID + pid := os.Getpid() + if err := os.WriteFile(containerPath.IDPath(), []byte(fmt.Sprintf("pid://%d", pid)), 0644); err != nil { + return fmt.Errorf("failed to create pid file") // Log the error + } + + // Execute Apptainer (Blocking) + log.Debug().Msg(fmt.Sprintf("ApptainerArgs: %v", apptainerArgs)) + cmd := exec.Command("apptainer", apptainerArgs...) + cmd.Env = os.Environ() + + // Open log file + logFile, err := os.Create(containerPath.LogsPath()) + if err != nil { + return fmt.Errorf("failed to create log file: %v", err) + } + defer logFile.Close() + + // // Redirect output to log file + cmd.Stdout = logFile + cmd.Stderr = logFile + if err := cmd.Run(); err != nil { + log.Error().Err(err).Msgf("Error executing init container: %s", container.Name) + return fmt.Errorf("init container failed: %v", err) // Abort on failure + } + if err := os.WriteFile(containerPath.ExitCodePath(), []byte(strconv.Itoa(cmd.ProcessState.ExitCode())), 0644); err != nil { + return fmt.Errorf("failed to create exitCode file") // Log the error + } + } + return nil +} + +func handleContainers(pod *v1.Pod, wg *sync.WaitGroup, hpkEnv bool) error { + isDebug := os.Getenv("DEBUG_MODE") == "true" + podKey := client.ObjectKeyFromObject(pod) + hpk := endpoint.HPK(pod.Annotations["workingDirectory"]) + podPath := hpk.Pod(podKey) + for _, container := range pod.Spec.Containers { + effectiSecurityContext := podhandler.DetermineEffectiveSecurityContext(pod, &container) + uid, gid := podhandler.DetermineEffectiveRunAsUser(effectiSecurityContext) + instanceName := fmt.Sprintf("%s_%s_%s", pod.GetNamespace(), pod.GetName(), container.Name) + + containerPath := podPath.Container(container.Name) + envFilePath := containerPath.EnvFilePath() + + // Environment File Handling + if fileExists(envFilePath) { + output, err := exec.Command("sh", "-c", envFilePath).CombinedOutput() + if err != nil { + return fmt.Errorf("error executing EnvFilePath: %v, output: %s", err, output) + } + envFileName := filepath.Join("/scratch", instanceName+".env") + if err := os.WriteFile(envFileName, output, 0644); err != nil { + return fmt.Errorf("error writing env file: %v", err) + } + } + + executionMode := "exec" + if container.Command == nil { + executionMode = "run" + } + + binds := make([]string, len(container.VolumeMounts)) + + // check the code from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet_pods.go#L196 + for i, mount := range container.VolumeMounts { + hostPath := filepath.Join(podPath.VolumeDir(), mount.Name) + + // subPath := mount.SubPath + // if mount.SubPathExpr != "" { + // subPath, err = helpers.ExpandContainerVolumeMounts(mount, h.podEnvVariables) + // if err != nil { + // compute.SystemPanic(err, "cannot expand env variables for container '%s' of pod '%s'", container, h.podKey) + // } + // } + + // if subPath != "" { + // if filepath.IsAbs(subPath) { + // return fmt.Errorf("error SubPath '%s' must not be an absolute path", subPath) + // } + + // subPathFile := filepath.Join(hostPath, subPath) + + // // mount the subpath + // hostPath = subPathFile + // } + + accessMode := "rw" + if mount.ReadOnly { + accessMode = "ro" + } + + binds[i] = hostPath + ":" + mount.MountPath + ":" + accessMode + } + + // Apptainer Command Construction + apptainerVerbosity := "--quiet" + if isDebug { + apptainerVerbosity = "--debug" + } + apptainerArgs := []string{ + apptainerVerbosity, executionMode, "--cleanenv", "--writable-tmpfs", "--no-mount", "home", "--unsquash", + } + if hpkEnv { + apptainerArgs = append(apptainerArgs, "--bind", "/scratch/etc/resolv.conf:/etc/resolv.conf,/scratch/etc/hosts:/etc/hosts") + if len(binds) > 0 { + bindArgs := &apptainerArgs[len(apptainerArgs)-1] + *bindArgs += "," + strings.Join(binds, ",") + } + } + if uid != 0 { + apptainerArgs = append(apptainerArgs, "--security", fmt.Sprintf("uid:%d,gid:%d", uid, uid), "--userns") + } + if gid != 0 { + apptainerArgs = append(apptainerArgs, "--security", fmt.Sprintf("gid:%d", gid), "--userns") + } + + if fileExists(envFilePath) { + apptainerArgs = append(apptainerArgs, "--env-file", filepath.Join("/scratch", instanceName+".env")) + } + + apptainerArgs = append(apptainerArgs, hpk.ImageDir()+image.ParseImageName(container.Image)) + apptainerArgs = append(apptainerArgs, container.Command...) + apptainerArgs = append(apptainerArgs, container.Args...) + + wg.Add(1) + go func(container v1.Container) { // Ensure container cleanup + defer wg.Done() + // Execute Apptainer in Background + log.Debug().Msg(fmt.Sprintf("ApptainerArgs: %v", apptainerArgs)) + cmd := exec.Command("apptainer", apptainerArgs...) + cmd.Env = os.Environ() + // If needed, get references to stdout and stderr + // log.Debug().Msgf("LogPath: %s", containerPath.LogsPath()) + logFile, err := os.Create(containerPath.LogsPath()) + if err != nil { + log.Error().Err(err).Msgf("Failed to create log file %s", containerPath.LogsPath()) + return + } + defer logFile.Close() + + cmd.Stdout = logFile + cmd.Stderr = logFile + log.Info().Msgf("Spawning main container: %s", container.Name) + // Start the container + if err := cmd.Start(); err != nil { + log.Error().Err(err).Msg("Failed to start Apptainer container") + } + + // Get the PID + pid := cmd.Process.Pid + if err := os.WriteFile(containerPath.IDPath(), []byte(fmt.Sprintf("pid://%d", pid)), 0644); err != nil { + log.Error().Err(err).Msg("Failed to create pid file") // Log the error + return + } + + // Handle Exit (consider moving output writing or using cmd.Wait) + if err := cmd.Wait(); err != nil { + log.Error().Err(err).Msgf("error executing container: %s, because of %v", container.Name, err) + } + + if err := os.WriteFile(containerPath.ExitCodePath(), []byte(strconv.Itoa(cmd.ProcessState.ExitCode())), 0644); err != nil { + log.Error().Err(err).Msg("Failed to create exitCode file") // Log the error + return + } + + }(container) + + } + return nil +} + +/** + +if err := scriptTemplate.Execute(&scriptFileContent, JobFields{ + Pod: h.podKey, + PauseImageFilePath: pauseImage.Filepath, + HostEnv: compute.Environment, + VirtualEnv: compute.VirtualEnvironment{ + PodDirectory: h.podDirectory.String(), + CgroupFilePath: h.podDirectory.CgroupFilePath(), + ConstructorFilePath: h.podDirectory.ConstructorFilePath(), + IPAddressPath: h.podDirectory.IPAddressPath(), + StdoutPath: h.podDirectory.StdoutPath(), + StderrPath: h.podDirectory.StderrPath(), + SysErrorFilePath: h.podDirectory.SysErrorFilePath(), + }, + InitContainers: initContainers, + Containers: containers, + ResourceRequest: resources.ResourceListToStruct(resourceRequest), + CustomFlags: customFlags, + }); err != nil { + //-- since both the template and fields are internal to the code, the evaluation should always succeed -- + compute.SystemPanic(err, "failed to evaluate sbatch template") + } +**/ + +/** +#!/bin/bash + +############################ +# Auto-Generated Script # +# Please do not edit. # +############################ + +# If any command fails, the script will immediately exit, +# and unset variables or errors in pipelines are treated as errors + +set -eum pipeline + +function debug_info() { + echo -e "\n" + echo "==============================" + echo " Compute Environment Info" + echo "==============================" + echo "* DNS: {{.HostEnv.KubeDNS}}" + echo "* PodDir: {{.VirtualEnv.PodDirectory}}" + echo "==============================" + echo -e "\n" + echo "==============================" + echo " Virtual Environment Info" + echo "==============================" + echo "* Host: $(hostname)" + echo "* IP: $(hostname -I)" + echo "* User: $(id)" + echo "==============================" + echo -e "\n" +} + +handle_dns() { + mkdir -p /scratch/etc + +# Rewire /scratch/etc/resolv.conf to point to KubeDNS +cat > /scratch/etc/resolv.conf << DNS_EOF +search {{.Pod.Namespace}}.svc.cluster.local svc.cluster.local cluster.local +nameserver {{.HostEnv.KubeDNS}} +options ndots:5 +DNS_EOF + + # Add hostname to known hosts. Required for loopback + echo -e "127.0.0.1 localhost" >> /scratch/etc/hosts + echo -e "$(hostname -I) $(hostname)" >> /scratch/etc/hosts +} + +# If not removed, Flags will be consumed by the nested Singularity and overwrite paths. +# https://docs.sylabs.io/guides/3.11/user-guide/environment_and_metadata.html +function reset_env() { + unset LD_LIBRARY_PATH + + unset SINGULARITY_COMMAND + unset SINGULARITY_CONTAINER + unset SINGULARITY_ENVIRONMENT + unset SINGULARITY_NAME + + unset APPTAINER_APPNAME + unset APPTAINER_COMMAND + unset APPTAINER_CONTAINER + unset APPTAINER_ENVIRONMENT + unset APPTAINER_NAME + + unset APPTAINER_BIND + unset SINGULARITY_BIND +} + +function cleanup() { + lastCommand=$1 + exitCode=$2 + + echo "[Virtual] Ensure all background jobs are terminated". + wait + + if [[ $exitCode -eq 0 ]]; then + echo "[Virtual] Gracefully exit the Virtual Environment. All resources will be released." + else + echo "[Virtual] **SYSTEMERROR** ${lastCommand} command filed with exit code ${exitCode}" | tee {{.VirtualEnv.SysErrorFilePath}} + fi + + exit ${exitCode} +} + +function handle_init_containers() { +{{range $index, $container := .InitContainers}} + #################### + ## New Container # + #################### + + echo "[Virtual] Spawning InitContainer: {{$container.InstanceName}}" + + {{- if $container.EnvFilePath}} + sh -c {{$container.EnvFilePath}} > /scratch/{{$container.InstanceName}}.env + {{- end}} + + # Mark the beginning of an init job (all get the shell's pid). + echo pid://$$ > {{$container.JobIDPath}} + + + $(apptainer {{ $container.ExecutionMode }} --cleanenv --writable-tmpfs --no-mount home --unsquash \ + {{- if $container.RunAsUser}} + --security uid:{{$container.RunAsUser}},gid:{{$container.RunAsUser}} --userns \ + {{- end}} + {{- if $container.RunAsGroup}} + --security gid:{{$container.RunAsGroup}} --userns \ + {{- end}} + --bind /scratch/etc/resolv.conf:/etc/resolv.conf,/scratch/etc/hosts:/etc/hosts,{{join "," $container.Binds}} \ + {{- if $container.EnvFilePath}} + --env-file /scratch/{{$container.InstanceName}}.env \ + {{- end}} + {{$container.ImageFilePath}} + {{- if $container.Command}} + {{- range $index, $cmd := $container.Command}} {{$cmd | param}} {{- end}} + {{- end -}} + {{- if $container.Args}} + {{range $index, $arg := $container.Args}} {{$arg | param}} {{- end}} + {{- end }} \ + &>> {{$container.LogsPath}}) + + # Mark the ending of an init job. + echo $? > {{$container.ExitCodePath}} +{{end}} + + echo "[Virtual] All InitContainers have been completed." + return +} + +function handle_containers() { +{{range $index, $container := .Containers}} + #################### + ## New Container # + #################### + + {{- if $container.EnvFilePath}} + sh -c {{$container.EnvFilePath}} > /scratch/{{$container.InstanceName}}.env + {{- end}} + + $(apptainer {{ $container.ExecutionMode }} --cleanenv --writable-tmpfs --no-mount home --unsquash \ + {{- if $container.RunAsUser}} + --security uid:{{$container.RunAsUser}},gid:{{$container.RunAsUser}} --userns \ + {{- end}} + {{- if $container.RunAsGroup}} + --security gid:{{$container.RunAsGroup}} --userns \ + {{- end}} + --bind /scratch/etc/resolv.conf:/etc/resolv.conf,/scratch/etc/hosts:/etc/hosts,{{join "," $container.Binds}} \ + {{- if $container.EnvFilePath}} + --env-file /scratch/{{$container.InstanceName}}.env \ + {{- end}} + {{$container.ImageFilePath}} + {{- if $container.Command}} + {{- range $index, $cmd := $container.Command}} {{$cmd | param}} {{- end}} + {{- end -}} + {{- if $container.Args}} + {{- range $index, $arg := $container.Args}} {{$arg | param}} {{- end}} + {{- end }} \ + &>> {{$container.LogsPath}}; \ + echo $? > {{$container.ExitCodePath}}) & + + pid=$! + echo pid://${pid} > {{$container.JobIDPath}} + echo "[Virtual] Container started: {{$container.InstanceName}} ${pid}" +{{end}} + + ###################### + ## Wait Containers # + ###################### + + echo "[Virtual] ... Waiting for containers to complete ..." + wait || echo "[Virtual] ... wait failed with error: $?" + echo "[Virtual] ... Containers terminated ..." +} + + + +debug_info + +echo "[Virtual] Resetting Environment ..." +reset_env + +echo "[Virtual] Announcing IP ..." +echo $(hostname -I) > {{.VirtualEnv.IPAddressPath}} + +echo "[Virtual] Setting DNS ..." +handle_dns + +echo "[Virtual] Setting Cleanup Handler ..." +trap 'cleanup "${BASH_COMMAND}" "$?"' EXIT + +{{if gt (len .InitContainers) 0 }} handle_init_containers {{end}} + +{{if gt (len .Containers) 0 }} handle_containers {{end}} +**/ diff --git a/cmd/pause/main_test.go b/cmd/pause/main_test.go new file mode 100644 index 0000000..f6ddb1f --- /dev/null +++ b/cmd/pause/main_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "sync" + "testing" + + "github.com/carv-ics-forth/hpk/compute/endpoint" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Mock execFunc for testing +func mockExecFunc(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + return cmd +} + +// Test for successful init container execution +func TestHandleInitContainers_Success(t *testing.T) { + + // Create a test Pod + annotations := make(map[string]string) + annotations["workingDirectory"] = "/home/malvag" + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-init-pod", + Namespace: "default", + Annotations: annotations, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + {Name: "test-init-container", Image: "busybox", Command: []string{"sh", "-c", "echo hello from init container"}}, + }, + }, + } + podKey := client.ObjectKeyFromObject(pod) + hpk := endpoint.HPK(pod.Annotations["workingDirectory"]) + podPath := hpk.Pod(podKey) + logPath := podPath.Container("test-init-container").LogsPath() + exitCodeFilePath := podPath.Container("test-init-container").ExitCodePath() + + if err := os.MkdirAll(string(podPath), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + if err := os.MkdirAll(string(podPath.ControlFileDir()), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + if err := os.MkdirAll(string(podPath.JobDir()), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + if err := os.MkdirAll(string(podPath.LogDir()), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + + if err := handleInitContainers(pod, false); err != nil { + t.Errorf("handleInitContainers failed unexpectedly: %v", err) + } + // Verify log file contents (adjust the path as needed based on your implementation) + logData, err := os.ReadFile(logPath) + if err != nil { + t.Errorf("Error reading log file: %v", err) + } + + expectedOutput := "hello from init container\n" + if string(logData) != expectedOutput { + t.Errorf("Unexpected log output. Got: %v, Expected: %v", string(logData), expectedOutput) + } + exitData, err := os.ReadFile(exitCodeFilePath) + if err != nil { + t.Errorf("Error reading exitCode file: %v", err) + } + + if string(exitData) != fmt.Sprint(0) { + t.Errorf("Unexpected exitCode. Got: %v, Expected: %v", string(exitData), 0) + } +} + +// Test for successful init container execution +func TestHandleContainers_Success(t *testing.T) { + + // Create a test Pod + annotations := make(map[string]string) + annotations["workingDirectory"] = "/home/malvag" + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-main-pod", + Namespace: "default", + Annotations: annotations, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + {Name: "test-main-container", Image: "busybox", Command: []string{"sh", "-c", "echo hello from main container"}}, + }, + }, + } + podKey := client.ObjectKeyFromObject(pod) + hpk := endpoint.HPK(pod.Annotations["workingDirectory"]) + podPath := hpk.Pod(podKey) + logPath := podPath.Container("test-main-container").LogsPath() + exitCodeFilePath := podPath.Container("test-main-container").ExitCodePath() + + if err := os.MkdirAll(string(podPath), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + if err := os.MkdirAll(string(podPath.ControlFileDir()), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + if err := os.MkdirAll(string(podPath.JobDir()), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + if err := os.MkdirAll(string(podPath.LogDir()), 0750); err != nil { + t.Errorf("create pod directory failed unexpectedly: %v", err) + } + + var wg sync.WaitGroup + if err := handleContainers(pod, &wg, false); err != nil { + t.Errorf("handleContainers failed unexpectedly: %v", err) + } + + wg.Wait() + // Verify log file contents (adjust the path as needed based on your implementation) + logData, err := os.ReadFile(logPath) + if err != nil { + t.Errorf("Error reading log file: %v", err) + } + + expectedOutput := "hello from main container\n" + if string(logData) != expectedOutput { + t.Errorf("Unexpected log output. Got: %v, Expected: %v", string(logData), expectedOutput) + } + + exitData, err := os.ReadFile(exitCodeFilePath) + if err != nil { + t.Errorf("Error reading exitCode file: %v", err) + } + + if string(exitData) != fmt.Sprint(0) { + t.Errorf("Unexpected exitCode. Got: %v, Expected: %v", string(exitData), 0) + } +} diff --git a/compute/image/constants.go b/compute/image/constants.go index 8cef6f3..eef4ba7 100644 --- a/compute/image/constants.go +++ b/compute/image/constants.go @@ -22,4 +22,5 @@ func (t Transport) Wrap(imageName string) string { return string(t) + imageName } -const PauseImage = "icsforth/pause:apptainer" +// const PauseImage = "icsforth/pause:apptainer" +const PauseImage = "malvag/pause:apptainer" diff --git a/compute/podhandler/container.go b/compute/podhandler/container.go index 3d09129..8dee6ed 100644 --- a/compute/podhandler/container.go +++ b/compute/podhandler/container.go @@ -35,7 +35,7 @@ import ( // buildContainer replicates the behavior of // https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kuberuntime/kuberuntime_container.go -func (h *podHandler) buildContainer(container *corev1.Container, containerStatus *corev1.ContainerStatus) (Container, error) { +func (h *PodHandler) buildContainer(container *corev1.Container, containerStatus *corev1.ContainerStatus) (Container, error) { /*--------------------------------------------------- * Determine the effective security context *---------------------------------------------------*/ diff --git a/compute/podhandler/container_test.go b/compute/podhandler/container_test.go index ec908b5..229ca7d 100644 --- a/compute/podhandler/container_test.go +++ b/compute/podhandler/container_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -func Test_podHandler_buildContainer(t *testing.T) { +func Test_PodHandler_buildContainer(t *testing.T) { /* var container corev1.Container @@ -39,7 +39,7 @@ func Test_podHandler_buildContainer(t *testing.T) { var pod corev1.Pod var containerStatus corev1.ContainerStatus - h := &podHandler{ + h := &PodHandler{ Pod: &pod, podEnvVariables: []corev1.EnvVar{}, podDirectory: "", diff --git a/compute/podhandler/envvars.go b/compute/podhandler/envvars.go index 98699a6..a613074 100644 --- a/compute/podhandler/envvars.go +++ b/compute/podhandler/envvars.go @@ -22,6 +22,7 @@ import ( "github.com/carv-ics-forth/hpk/compute" corev1 "k8s.io/api/core/v1" + // discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/compute/podhandler/podHandler.go b/compute/podhandler/podhandler.go similarity index 92% rename from compute/podhandler/podHandler.go rename to compute/podhandler/podhandler.go index e169c8b..2234695 100644 --- a/compute/podhandler/podHandler.go +++ b/compute/podhandler/podhandler.go @@ -17,6 +17,7 @@ package podhandler import ( "bytes" "context" + "fmt" "io/fs" "os" "path/filepath" @@ -200,7 +201,7 @@ remove_pod: return true } -type podHandler struct { +type PodHandler struct { *corev1.Pod podKey client.ObjectKey @@ -218,7 +219,7 @@ func CreatePod(ctx context.Context, pod *corev1.Pod, watcher filenotify.FileWatc podKey := client.ObjectKeyFromObject(pod) logger := compute.DefaultLogger.WithValues("pod", podKey) - h := podHandler{ + h := PodHandler{ Pod: pod, podKey: podKey, podDirectory: compute.HPK.Pod(podKey), @@ -357,6 +358,28 @@ func CreatePod(ctx context.Context, pod *corev1.Pod, watcher filenotify.FileWatc scriptFileContent := bytes.Buffer{} + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + + // Set annotations from HostEnvironment + pod.Annotations["kubeMasterHost"] = compute.Environment.KubeMasterHost + pod.Annotations["containerRegistry"] = compute.Environment.ContainerRegistry + pod.Annotations["apptainerBin"] = compute.Environment.ApptainerBin + pod.Annotations["enableCgroupV2"] = fmt.Sprintf("%t", compute.Environment.EnableCgroupV2) + pod.Annotations["workingDirectory"] = compute.Environment.WorkingDirectory + pod.Annotations["kubeDNS"] = compute.Environment.KubeDNS + + // Set annotations from VirtualEnvironment + pod.Annotations["cgroupFilePath"] = h.podDirectory.CgroupFilePath() + pod.Annotations["constructorFilePath"] = h.podDirectory.ConstructorFilePath() + pod.Annotations["ipAddressPath"] = h.podDirectory.IPAddressPath() + pod.Annotations["stdoutPath"] = h.podDirectory.StdoutPath() + pod.Annotations["stderrPath"] = h.podDirectory.StderrPath() + pod.Annotations["sysErrorFilePath"] = h.podDirectory.SysErrorFilePath() + + pod.Annotations["PauseImage"] = image.PauseImage + if err := scriptTemplate.Execute(&scriptFileContent, JobFields{ Pod: h.podKey, PauseImageFilePath: pauseImage.Filepath, diff --git a/compute/podhandler/templates.go b/compute/podhandler/templates.go index 3a67418..f606d03 100644 --- a/compute/podhandler/templates.go +++ b/compute/podhandler/templates.go @@ -313,15 +313,28 @@ trap 'echo [HOST] Deleting workdir ${workdir}; rm -rf ${workdir}' EXIT # --network-args "portmap=8080:80/tcp" # --container is needed to start a separate /dev/sh +#exec {{$.HostEnv.ApptainerBin}} exec --containall --net --fakeroot --scratch /scratch --workdir ${workdir} \ +#{{- if .HostEnv.EnableCgroupV2}} +#--apply-cgroups {{.VirtualEnv.CgroupFilePath}} \ +#{{- end}} +#--env PARENT=${PPID} \ +#--bind $HOME,/tmp \ +#--hostname {{.Pod.Name}} \ +#{{$.PauseImageFilePath}} sh -ci {{.VirtualEnv.ConstructorFilePath}} || +#echo "[HOST] **SYSTEMERROR** apptainer exited with code $?" | tee {{.VirtualEnv.SysErrorFilePath}} + +export APPTAINERENV_KUBEDNS_IP={{.HostEnv.KubeDNS}} + exec {{$.HostEnv.ApptainerBin}} exec --containall --net --fakeroot --scratch /scratch --workdir ${workdir} \ {{- if .HostEnv.EnableCgroupV2}} --apply-cgroups {{.VirtualEnv.CgroupFilePath}} \ {{- end}} --env PARENT=${PPID} \ ---bind $HOME,/tmp \ +--bind $HOME/.k8sfs/kubernetes:/k8s-data \ +--bind $HOME,/tmp \ --hostname {{.Pod.Name}} \ -{{$.PauseImageFilePath}} sh -ci {{.VirtualEnv.ConstructorFilePath}} || -echo "[HOST] **SYSTEMERROR** apptainer exited with code $?" | tee {{.VirtualEnv.SysErrorFilePath}} +{{$.PauseImageFilePath}} /usr/local/bin/hpk-pause -namespace {{.Pod.Namespace}} -pod {{.Pod.Name}} || +echo "[HOST] **SYSTEMERROR** hpk-pause exited with code $?" | tee {{.VirtualEnv.SysErrorFilePath}} #### END SECTION: Host Environment #### ` diff --git a/compute/podhandler/templates_test.go b/compute/podhandler/templates_test.go index c01bd2e..eb15834 100644 --- a/compute/podhandler/templates_test.go +++ b/compute/podhandler/templates_test.go @@ -21,7 +21,7 @@ import ( "testing" "github.com/carv-ics-forth/hpk/compute" - "github.com/carv-ics-forth/hpk/compute/podhandler" + PodHandler "github.com/carv-ics-forth/hpk/compute/podhandler" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/types" ) @@ -112,11 +112,11 @@ func TestConstructorSyntax(t *testing.T) { tests := []struct { name string - fields podhandler.JobFields + fields PodHandler.JobFields }{ { name: "noenv", - fields: podhandler.JobFields{ + fields: PodHandler.JobFields{ HostEnv: compute.HostEnvironment{ ApptainerBin: "apptainer", KubeDNS: "6.6.6.6", @@ -132,7 +132,7 @@ func TestConstructorSyntax(t *testing.T) { StderrPath: podDir.StderrPath(), SysErrorFilePath: "", }, - InitContainers: []podhandler.Container{ + InitContainers: []PodHandler.Container{ { InstanceName: "init0", RunAsUser: 0, @@ -163,7 +163,7 @@ func TestConstructorSyntax(t *testing.T) { }, }, - Containers: []podhandler.Container{ + Containers: []PodHandler.Container{ { InstanceName: "lala", RunAsUser: 0, @@ -207,7 +207,7 @@ func TestConstructorSyntax(t *testing.T) { }, } - submitTpl, err := podhandler.ParseTemplate(podhandler.PauseScriptTemplate) + submitTpl, err := PodHandler.ParseTemplate(PodHandler.PauseScriptTemplate) if err != nil { t.Fatal(err) } @@ -234,7 +234,7 @@ func TestConstructorSyntax(t *testing.T) { log.Fatal(err) } - if err := podhandler.ValidateScript(f.Name()); err != nil { + if err := PodHandler.ValidateScript(f.Name()); err != nil { log.Fatal(err) } diff --git a/compute/podhandler/volumes.go b/compute/podhandler/volumes.go index bcec834..57addad 100644 --- a/compute/podhandler/volumes.go +++ b/compute/podhandler/volumes.go @@ -54,7 +54,7 @@ var NotFoundBackoff = wait.Backoff{ // mountVolumeSource prepares the volumes into the local pod directory. // Critical errors related to the HPK fail directly. // Misconfigurations (like wrong hostpaths), are returned as errors -func (h *podHandler) mountVolumeSource(ctx context.Context, vol corev1.Volume) error { +func (h *PodHandler) mountVolumeSource(ctx context.Context, vol corev1.Volume) error { switch { case vol.VolumeSource.EmptyDir != nil: /*--------------------------------------------------- @@ -228,7 +228,7 @@ func (h *podHandler) mountVolumeSource(ctx context.Context, vol corev1.Volume) e } } -func (h *podHandler) DownwardAPIVolumeSource(ctx context.Context, vol corev1.Volume) { +func (h *PodHandler) DownwardAPIVolumeSource(ctx context.Context, vol corev1.Volume) { downApiDir := filepath.Join(h.podDirectory.VolumeDir(), vol.Name) if err := os.MkdirAll(downApiDir, endpoint.PodGlobalDirectoryPermissions); err != nil { @@ -250,7 +250,7 @@ func (h *podHandler) DownwardAPIVolumeSource(ctx context.Context, vol corev1.Vol } } -func (h *podHandler) PersistentVolumeClaimSource(ctx context.Context, vol corev1.Volume) { +func (h *PodHandler) PersistentVolumeClaimSource(ctx context.Context, vol corev1.Volume) { /*--------------------------------------------------- * Get the Referenced PVC from Volume *---------------------------------------------------*/ diff --git a/deploy/images/pause-apptainer-agent/docker-entrypoint.sh b/deploy/images/pause-apptainer-agent/docker-entrypoint.sh new file mode 100755 index 0000000..5fc4448 --- /dev/null +++ b/deploy/images/pause-apptainer-agent/docker-entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec "$@" diff --git a/deploy/images/pause-apptainer-agent/pause.apptainer.Dockerfile b/deploy/images/pause-apptainer-agent/pause.apptainer.Dockerfile new file mode 100644 index 0000000..d0e907a --- /dev/null +++ b/deploy/images/pause-apptainer-agent/pause.apptainer.Dockerfile @@ -0,0 +1,33 @@ +FROM golang:latest AS builder + +WORKDIR /app + +# Copy your HPK-pause source code +COPY . . +# Build the HPK-pause binary +RUN make hpk-pause + +# --- Minimal final image --- + +FROM ubuntu:latest + +ENV VERSION 1.1.9 + +RUN apt-get update && apt-get install -y wget + +RUN wget https://github.com/apptainer/apptainer/releases/download/v${VERSION}/apptainer_${VERSION}_amd64.deb + +RUN apt-get install -y ./apptainer_${VERSION}_amd64.deb && rm ./apptainer_${VERSION}_amd64.deb + +# (Install other dependencies) +RUN apt-get install -y iptables iproute2 fakeroot fakeroot-ng gosu patchelf nano vim + +# Copy the compiled pause binary from the builder stage +COPY --from=builder /app/bin/hpk-pause /usr/local/bin +COPY --from=builder /app/deploy/images/pause-apptainer-agent/docker-entrypoint.sh /usr/local/bin/ + +RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat +USER root +WORKDIR /root + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docs/admin-guide.md b/docs/admin-guide.md index caa712d..41fe152 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -1,17 +1,21 @@ # Admin Guide + In this tutorial, we'll walk you through the setup process to get HPK up and operational. + > Tested on Ubuntu 20.04, CentOS 7. + ## Requirements -| Variable | Version | -|------------------------------|--------------| -| APPTAINER_VERSION | 1.1.4 | -| FLANNEL_VERSION | 0.20.2 | -| FLANNEL_CNI_PLUGIN_VERSION | 1.1.2 | -| KUBERNETES_VERSION | 1.24.8 | -| HELM_VERSION | 3.10.3 | +| Variable | Version | +| -------------------------- | ------- | +| APPTAINER_VERSION | 1.1.4 | +| FLANNEL_VERSION | 0.20.2 | +| FLANNEL_CNI_PLUGIN_VERSION | 1.1.2 | +| KUBERNETES_VERSION | 1.24.8 | +| HELM_VERSION | 3.10.3 | Set environment variables: + ```bash APPTAINER_VERSION=1.1.4 FLANNEL_VERSION=0.20.2 @@ -25,6 +29,7 @@ ETCD_ADDRESS=`cat ${SLURM_CONFIG} | grep SlurmctldHost | awk -F '[()]' '{print $ ``` Install wget utility: + ```bash if [[ "$(. /etc/os-release; echo $ID)" == "ubuntu" ]]; then apt-get update @@ -33,12 +38,15 @@ else yum install -y wget fi ``` -### Install [etcd](https://etcd.io/) + +### Install [etcd](https://etcd.io/) + > On one host We use etcd, a key-value store that will be used by flannel later. Setup etcd service and fire it up: + ```bash if [[ "$HOST_ADDRESS" == "$ETCD_ADDRESS" ]]; then if [[ "$(. /etc/os-release; echo $ID)" == "ubuntu" ]]; then @@ -61,7 +69,9 @@ fi ``` ### Install [Apptainer](https://apptainer.org/) + > On all hosts + ```bash if [[ "$(. /etc/os-release; echo $ID)" == "ubuntu" ]]; then wget -q https://github.com/apptainer/apptainer/releases/download/v${APPTAINER_VERSION}/apptainer_${APPTAINER_VERSION}_amd64.deb @@ -81,12 +91,14 @@ else fi ``` -### Install [Flannel](https://github.com/flannel-io/flannel) +### Install [Flannel](https://github.com/flannel-io/flannel) + > On all hosts Flannel runs a small, single binary agent called flanneld on each host, and is responsible for allocating a subnet lease to each host out of a larger, preconfigured address space. Flannel uses either the Kubernetes API or etcd directly to store the network configuration, the allocated subnets, and any auxiliary data (such as the host's public IP). Packets are forwarded using one of several backend mechanisms including VXLAN and various cloud integrations. To install: + ```bash if [[ "$(. /etc/os-release; echo $ID)" == "ubuntu" ]]; then apt-get install -y nscd # https://github.com/flannel-io/flannel/issues/1512 @@ -99,6 +111,7 @@ rm -f flanneld-amd64 ``` Now setup the flannel service and fire it up: + ```bash cat >/etc/systemd/system/flanneld.service < On all hosts First we download and install Flannel binary + ```bash wget -q https://github.com/flannel-io/cni-plugin/releases/download/v${FLANNEL_CNI_PLUGIN_VERSION}/flannel-amd64 chmod +x flannel-amd64 cp flannel-amd64 /usr/libexec/apptainer/cni/flannel rm -f flannel-amd64 ``` + Then we configure Apptainer to use Flannel as a CNI plug-in for fakeroot runs + ```bash cat > /etc/apptainer/network/40_fakeroot.conflist < In case there is a problem using these CNI plugins as a regular user you can additionally setup apptainer with the following: + ```bash cat >>/etc/apptainer/apptainer.conf < @@ -160,15 +178,20 @@ allow net groups = allow net networks = bridge, flannel EOF ``` + ### Install Utilities + - Kubectl utility with the same version as the Kubernetes + ```bash wget -q https://dl.k8s.io/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl chmod +x kubectl cp kubectl /usr/local/bin/kubectl rm -f kubectl ``` + - Helm utility + ```bash wget -q https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz --strip-components=1 linux-amd64/helm @@ -179,6 +202,7 @@ rm -f helm helm-v${HELM_VERSION}-linux-amd64.tar.gz ## HPK Installation & Setup Back to the head node, as the local user: + ```sh git clone https://github.com/CARV-ICS-FORTH/HPK.git cd HPK @@ -191,6 +215,7 @@ mv hpk-kubelet bin/ ``` Run each of the following in a separate window: + ```sh make run-kubemaster make run-kubelet @@ -201,19 +226,22 @@ Running the above commands, respectively: ![After deploying HPK](images/run-kubelet.png) - And you are all set: + ```sh export KUBE_PATH=~/.k8sfs/kubernetes/ export KUBECONFIG=${KUBE_PATH}/admin.conf kubectl get nodes ``` -![After getting-nodes](images/get-nodes.png) +![After getting-nodes](images/get-nodes.png) ## Test + To test that everything is running correctly: + ```bash make test ``` -![](images/make-test.png) \ No newline at end of file + +![Testing suite makefile recipe example image](images/make-test.png) diff --git a/examples/apps/analytics-spark/shell-demo.yaml b/examples/apps/analytics-spark/shell-demo.yaml new file mode 100644 index 0000000..05eeb36 --- /dev/null +++ b/examples/apps/analytics-spark/shell-demo.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + name: shell-demo +spec: + volumes: + - name: shared-data + emptyDir: {} + containers: + - name: nginx + image: nginx # Assuming you still want the Nginx webserver + volumeMounts: + - name: shared-data + mountPath: /usr/share/nginx/html + hostNetwork: true + dnsPolicy: Default + initContainers: + - name: dns-checker + image: tutum/dnsutils:latest # Or a specific version + command: ["/bin/sh", "-c"] + args: + - | + nslookup kubernetes.default + nslookup google.com + nslookup archive.ubuntu.com + volumeMounts: + - name: shared-data + mountPath: /usr/share/nginx/html + - name: check-resolv + image: alpine:latest # Or a specific version like ubuntu:22.04 + command: ["/bin/sh", "-c"] + args: + - ping 139.91.157.1 \ No newline at end of file diff --git a/examples/apps/analytics-spark/tpcds.sh b/examples/apps/analytics-spark/tpcds.sh index a6a5ed8..69bd324 100755 --- a/examples/apps/analytics-spark/tpcds.sh +++ b/examples/apps/analytics-spark/tpcds.sh @@ -13,7 +13,7 @@ else fi # run TPCDS benchmark -cd HPK/examples/apps/analytics-spark +#cd HPK/examples/apps/analytics-spark # Pod name and namespace DATA_GENERATION_DRIVER_POD_NAME="tpcds-benchmark-data-generation-1g-driver" @@ -74,4 +74,4 @@ while true; do done echo "TCPDS Benchmark executed succesfully." popd -# ./uninstall.sh \ No newline at end of file +# ./uninstall.sh diff --git a/go.mod b/go.mod index 1d1b4aa..bdaa5b7 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/virtual-kubelet/virtual-kubelet v1.7.0 - golang.org/x/sys v0.6.0 + golang.org/x/sys v0.12.0 golang.org/x/time v0.2.0 k8s.io/api v0.25.4 k8s.io/apimachinery v0.25.4 @@ -61,7 +61,8 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -76,6 +77,7 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/rs/zerolog v1.31.0 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.2.1 // indirect diff --git a/go.sum b/go.sum index 7881115..0f42e3d 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2 h1:tjT4Jp4gxECvsJcYpAMtW2I3YqzBTPuB67OejxXs86s= github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -116,6 +117,7 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -235,10 +237,14 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f h1:B0OD7nYl2FPQEVrw8g2uyc1lGEzNbvrKh7fspGZcbvY= github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f/go.mod h1:aEt7p9Rvh67BYApmZwNDPpgircTO2kgdmDUoF/1QmwA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -304,6 +310,9 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -504,6 +513,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= diff --git a/provider/provider.go b/provider/provider.go index e2991b5..2e09482 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -26,7 +26,7 @@ import ( "github.com/carv-ics-forth/hpk/compute/endpoint" "github.com/carv-ics-forth/hpk/compute/events" - "github.com/carv-ics-forth/hpk/compute/podhandler" + PodHandler "github.com/carv-ics-forth/hpk/compute/podhandler" "github.com/carv-ics-forth/hpk/compute/runtime" "github.com/carv-ics-forth/hpk/compute/slurm" "github.com/carv-ics-forth/hpk/pkg/container" @@ -203,7 +203,7 @@ func (v *VirtualK8S) CreatePod(ctx context.Context, pod *corev1.Pod) error { go func() { // acknowledge the creation request and do the creation in the background. // if the creation fails, the pod should be marked as failed and returned to the provider. - podhandler.CreatePod(ctx, pod, v.fileWatcher) + PodHandler.CreatePod(ctx, pod, v.fileWatcher) v.updatedPod(pod) }() @@ -231,7 +231,7 @@ func (v *VirtualK8S) UpdatePod(ctx context.Context, pod *corev1.Pod) error { /*--------------------------------------------------- * Ensure that received pod is newer than the local *---------------------------------------------------*/ - localPod, err := podhandler.LoadPodFromKey(podKey) + localPod, err := PodHandler.LoadPodFromKey(podKey) if err != nil { return errdefs.NotFoundf("object not found") } @@ -266,7 +266,7 @@ func (v *VirtualK8S) UpdatePod(ctx context.Context, pod *corev1.Pod) error { } /*-- Update the local status of Pod --*/ - if err := podhandler.SavePodToFile(ctx, pod); err != nil { + if err := PodHandler.SavePodToFile(ctx, pod); err != nil { compute.SystemPanic(err, "failed to set job id for pod '%s'", podKey) } @@ -282,7 +282,7 @@ func (v *VirtualK8S) DeletePod(ctx context.Context, pod *corev1.Pod) error { logger.Info("[K8s] -> DeletePod") - if !podhandler.DeletePod(podKey, v.fileWatcher) { + if !PodHandler.DeletePod(podKey, v.fileWatcher) { logger.Info("[K8s] <- DeletePod (POD NOT FOUND)") return errdefs.NotFoundf("object not found") @@ -302,7 +302,7 @@ func (v *VirtualK8S) GetPod(ctx context.Context, namespace, name string) (*corev logger.Info("[K8s] -> GetPod") - pod, err := podhandler.LoadPodFromKey(podKey) + pod, err := PodHandler.LoadPodFromKey(podKey) if err != nil { logger.Info("[K8s] <- GetPod (POD NOT FOUND)") @@ -327,7 +327,7 @@ func (v *VirtualK8S) GetPodStatus(ctx context.Context, namespace, name string) ( logger.Info("[K8s] -> GetPodStatus") - pod, err := podhandler.LoadPodFromKey(podKey) + pod, err := PodHandler.LoadPodFromKey(podKey) if err != nil { logger.Info("[K8s] <- GetPodStatus (POD NOT FOUND)") @@ -408,8 +408,8 @@ func (v *VirtualK8S) NotifyPods(ctx context.Context, f func(*corev1.Pod)) { }) go eh.Listen(ctx, events.PodControl{ - UpdateStatus: podhandler.UpdateStatusFromRuntime, - LoadFromDisk: podhandler.LoadPodFromKey, + UpdateStatus: PodHandler.UpdateStatusFromRuntime, + LoadFromDisk: PodHandler.LoadPodFromKey, NotifyVirtualKubelet: func(pod *corev1.Pod) { if pod == nil { panic("this should not happen")