diff --git a/go.mod b/go.mod index 6d5c9ab37f..9e7f01d48c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,13 @@ go 1.22.0 toolchain go1.22.5 -replace github.com/edgelesssys/contrast/node-installer => ./node-installer +replace ( + github.com/edgelesssys/contrast/node-installer => ./node-installer + // The upstream package has some stepping issues with Genoa: + // https://github.com/google/go-sev-guest/issues/115 + // https://github.com/google/go-sev-guest/issues/103 + github.com/google/go-sev-guest => github.com/edgelesssys/go-sev-guest v0.0.0-20240705062330-4bd4b2483aa0 +) require ( filippo.io/keygen v0.0.0-20230306160926-5201437acf8e @@ -64,7 +70,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/pborman/uuid v1.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect diff --git a/go.sum b/go.sum index 23dee87ba3..36227abe4e 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/edgelesssys/go-sev-guest v0.0.0-20240705062330-4bd4b2483aa0 h1:d9uufoq2x0o81qC4GOau1NaAbZ/KkiVnw8mpMSEqS80= +github.com/edgelesssys/go-sev-guest v0.0.0-20240705062330-4bd4b2483aa0/go.mod h1:tqyacT8AaJv6ZWLtsFK5HKJ+d6b4L5iuAmRMWCkIupA= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= @@ -41,8 +43,6 @@ github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwM github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k= -github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -50,7 +50,6 @@ github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ= github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -96,8 +95,6 @@ github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index c060b7cb57..08ecb424d5 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -17,12 +17,15 @@ import ( ) // ContrastRuntimeClass creates a new RuntimeClassConfig. -func ContrastRuntimeClass() *RuntimeClassConfig { +func ContrastRuntimeClass(platform platforms.Platform) *RuntimeClassConfig { r := RuntimeClass(runtimeHandler). WithHandler(runtimeHandler). WithLabels(map[string]string{"addonmanager.kubernetes.io/mode": "Reconcile"}). - WithOverhead(Overhead(corev1.ResourceList{"memory": resource.MustParse("1152Mi")})). - WithScheduling(Scheduling(map[string]string{"kubernetes.azure.com/kata-cc-isolation": "true"})) + WithOverhead(Overhead(corev1.ResourceList{"memory": resource.MustParse("1152Mi")})) + + if platform == platforms.AKSCloudHypervisorSNP { + r.WithScheduling(Scheduling(map[string]string{"kubernetes.azure.com/kata-cc-isolation": "true"})) + } return &RuntimeClassConfig{r} } @@ -40,7 +43,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle switch platform { case platforms.AKSCloudHypervisorSNP: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest" - case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" default: return nil, fmt.Errorf("unsupported platform %q", platform) diff --git a/internal/kuberesource/sets.go b/internal/kuberesource/sets.go index d696484898..d6e76d0ba0 100644 --- a/internal/kuberesource/sets.go +++ b/internal/kuberesource/sets.go @@ -31,7 +31,7 @@ func CoordinatorBundle() []any { func Runtime(platform platforms.Platform) ([]any, error) { ns := "" - runtimeClass := ContrastRuntimeClass().RuntimeClassApplyConfiguration + runtimeClass := ContrastRuntimeClass(platform).RuntimeClassApplyConfiguration nodeInstaller, err := NodeInstaller(ns, platform) if err != nil { return nil, fmt.Errorf("creating node installer: %w", err) diff --git a/node-installer/internal/constants/configuration-qemu-snp.toml b/node-installer/internal/constants/configuration-qemu-snp.toml new file mode 100644 index 0000000000..f6e82e54c7 --- /dev/null +++ b/node-installer/internal/constants/configuration-qemu-snp.toml @@ -0,0 +1,61 @@ +# Minimized list, inactive options removed. +# upstream source: https://github.com/kata-containers/kata-containers/blob/dac07239f5c7e3deefbdefa8d02bbc427487042e/src/runtime/config/configuration-qemu-snp.toml.in +[hypervisor.qemu] +path = "/usr/bin/qemu-system-x86_64" +kernel = "/opt/kata/share/kata-containers/vmlinuz-confidential.container" +initrd = "/opt/kata/share/kata-containers/kata-containers-initrd-confidential.img" +machine_type = "q35" +rootfs_type="erofs" +confidential_guest = true +sev_snp_guest = true +snp_certs_path = "/opt/snp/cert_chain.cert" +enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "default_vcpus", "default_memory"] +valid_hypervisor_paths = ["/usr/bin/qemu-system-x86_64"] +kernel_params = "" +firmware = "/usr/share/ovmf/OVMF.fd" +firmware_volume = "" +machine_accelerators="" +cpu_features="pmu=off" +default_vcpus = 1 +default_maxvcpus = 0 +default_bridges = 1 +default_memory = 2048 +default_maxmemory = 0 +disable_block_device_use = false +shared_fs = "none" +virtio_fs_daemon = "/opt/kata/libexec/virtiofsd" +valid_virtio_fs_daemon_paths = ["/opt/kata/libexec/virtiofsd"] +virtio_fs_cache_size = 0 +virtio_fs_queue_size = 1024 +virtio_fs_extra_args = ["--thread-pool-size=1", "--announce-submounts"] +virtio_fs_cache = "auto" +block_device_driver = "virtio-scsi" +block_device_aio = "io_uring" +enable_iothreads = false +enable_vhost_user_store = false +vhost_user_store_path = "/var/run/kata-containers/vhost-user" +valid_vhost_user_store_paths = ["/var/run/kata-containers/vhost-user"] +vhost_user_reconnect_timeout_sec = 0 +file_mem_backend = "" +valid_file_mem_backends = [""] +pflashes = [] +disable_image_nvdimm = true +valid_entropy_sources = ["/dev/urandom","/dev/random",""] +disable_selinux=false +disable_guest_selinux=true + +[agent.kata] +kernel_modules=[] +dial_timeout = 90 + +[runtime] +internetworking_model="tcfilter" +disable_guest_seccomp=true +sandbox_cgroup_only=false +static_sandbox_resource_mgmt=true +sandbox_bind_mounts=[] +vfio_mode="guest-kernel" +disable_guest_empty_dir=false +experimental=[] +create_container_timeout = 60 +dan_conf = "/run/kata-containers/dans" diff --git a/node-installer/internal/constants/constants.go b/node-installer/internal/constants/constants.go index db97eca812..8863a85298 100644 --- a/node-installer/internal/constants/constants.go +++ b/node-installer/internal/constants/constants.go @@ -26,6 +26,12 @@ var ( //go:embed configuration-qemu-tdx.toml kataBareMetalQEMUTDXBaseConfig string + // kataBareMetalQEMUSNPBaseConfig is the configuration file for the Kata runtime on bare-metal SNP + // with QEMU. + // + //go:embed configuration-qemu-snp.toml + kataBareMetalQEMUSNPBaseConfig string + // containerdBaseConfig is the base configuration file for containerd // //go:embed containerd-config.toml @@ -66,6 +72,27 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, debug bool) config.Runtime["enable_debug"] = true } return &config, nil + case platforms.K3sQEMUSNP: + if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) + } + config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "bin", "qemu-system-x86_64") + config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "share", "OVMF.fd") + config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img") + config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel") + delete(config.Hypervisor["qemu"], "initrd") + config.Hypervisor["qemu"]["block_device_aio"] = "threads" + config.Hypervisor["qemu"]["shared_fs"] = "virtio-9p" + config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "bin", "qemu-system-x86_64")} + config.Hypervisor["qemu"]["rootfs_type"] = "erofs" + if debug { + config.Hypervisor["qemu"]["enable_debug"] = true + config.Hypervisor["qemu"]["kernel_params"] = " agent.log=debug initcall_debug" + config.Agent["kata"]["enable_debug"] = true + config.Agent["kata"]["debug_console_enabled"] = true + config.Runtime["enable_debug"] = true + } + return &config, nil default: return nil, fmt.Errorf("unsupported platform: %s", platform) } @@ -99,6 +126,10 @@ func ContainerdRuntimeConfigFragment(baseDir, snapshotter string, platform platf cfg.Options = map[string]any{ "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-tdx.toml"), } + case platforms.K3sQEMUSNP: + cfg.Options = map[string]any{ + "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-snp.toml"), + } default: return nil, fmt.Errorf("unsupported platform: %s", platform) } diff --git a/node-installer/node-installer.go b/node-installer/node-installer.go index c273552fe8..d3e94f1c3b 100644 --- a/node-installer/node-installer.go +++ b/node-installer/node-installer.go @@ -4,6 +4,8 @@ package main import ( + "bufio" + "bytes" "context" "encoding/json" "flag" @@ -69,7 +71,7 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, if err := os.MkdirAll(binDir, os.ModePerm); err != nil { return fmt.Errorf("creating runtime bin directory: %w", err) } - if err := os.MkdirAll(filepath.Join(hostMount, runtimeBase, "share"), os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Join(hostMount, runtimeBase, "share/qemu"), os.ModePerm); err != nil { return fmt.Errorf("creating runtime share directory: %w", err) } if err := os.MkdirAll(filepath.Join(hostMount, runtimeBase, "etc"), os.ModePerm); err != nil { @@ -114,6 +116,9 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, case platforms.K3sQEMUTDX: kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml") containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml") + case platforms.K3sQEMUSNP: + kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-snp.toml") + containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "k3s", "agent", "etc", "containerd", "config.toml.tmpl") case platforms.RKE2QEMUTDX: kataConfigPath = filepath.Join(kataConfigPath, "configuration-qemu-tdx.toml") containerdConfigPath = filepath.Join(hostMount, "var", "lib", "rancher", "rke2", "agent", "etc", "containerd", "config.toml") @@ -131,7 +136,7 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, if err := patchContainerdConfig(config.RuntimeHandlerName, runtimeBase, containerdConfigPath, platform); err != nil { return fmt.Errorf("patching containerd configuration: %w", err) } - case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX: // K3s or RKE2: We need to extend the configuration template, which, in it's un-templated form, is non-TOML. // Therefore just write the TOML configuration fragment ourselves and append it to the template file. // This assumes that the user does not yet have a runtime with the same name configured himself, @@ -151,7 +156,7 @@ func run(ctx context.Context, fetcher assetFetcher, platform platforms.Platform, switch platform { case platforms.AKSCloudHypervisorSNP: return restartHostContainerd(containerdConfigPath, "containerd") - case platforms.K3sQEMUTDX: + case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP: if hostServiceExists("k3s") { return restartHostContainerd(containerdConfigPath, "k3s") } else if hostServiceExists("k3s-agent") { @@ -241,8 +246,21 @@ func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath str return fmt.Errorf("reading containerd config template: %w", err) } + // Don't add the runtime section if it already exists. + runtimeSection := fmt.Sprintf("[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.%s]", runtimeName) + if bytes.Contains(existingConfig, []byte(runtimeSection)) { + fmt.Printf("Runtime section %q already exists\n", runtimeSection) + return nil + } + + // PluginFragment contains just the `Plugins` property used to configure containerd. + type PluginFragment struct { + // Plugins provides plugin specific configuration for the initialization of a plugin + Plugins map[string]any `toml:"plugins"` + } + // Extend a scratchpad config with the new plugin configuration. (including the new contrast-cc runtime) - var newConfigFragment config.ContainerdConfig + var newConfigFragment PluginFragment runtimes := ensureMapPath(&newConfigFragment.Plugins, constants.CRIFQDN, "containerd", "runtimes") containerdRuntimeConfig, err := constants.ContainerdRuntimeConfigFragment(basePath, "no-snapshotter", platform) if err != nil { @@ -250,8 +268,7 @@ func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath str } runtimes[runtimeName] = containerdRuntimeConfig - // We purposely don't marshal the full config, as we only want to append the plugin section. - rawNewPluginConfig, err := toml.Marshal(newConfigFragment.Plugins) + rawNewPluginConfig, err := toml.Marshal(newConfigFragment) if err != nil { return fmt.Errorf("marshaling containerd runtime config: %w", err) } @@ -259,8 +276,58 @@ func patchContainerdConfigTemplate(runtimeName, basePath, configTemplatePath str // First append the existing config template by a newline, so that if it ends without a newline, // the new config fragment isn't appended to the last line.. newRawConfig := append(existingConfig, []byte("\n")...) - // ..then append the new config fragment - newRawConfig = append(newRawConfig, rawNewPluginConfig...) + + // The marshalled config is shaped like a tree with the important bits at + // the leaves and lots of empty parent nodes (except for the link the one + // child): + // A > B > C > D > E=foo + // | > F > G=bar + // | > H=baz + // Nodes that don't contain any keys, but only contain a links to children + // are marshalled as empty sections: + // ```toml + // [A] # <- empty section + // [A.B] # <- empty section + // [A.B.C] # <- empty section + // [A.B.C.D] # <- non-empty section + // E = "foo" + // [A.B.C.D.F] # <- non-empty section + // G = "bar" + // H = "baz" + // ``` + // We want to avoid appending empty sections (i.e. sections with only a + // section header, but no keys) to the file because there's a chance that + // they already exist in the template and creating duplicate sections is + // illegal. + // On the other hand, omitting intermediate empty sections is always legal + // in TOML, so there's no risk in omitting empty sections. + // + // We iterate over the marshalled config line by line. If we encounter a + // section header, we don't immediately append it to the file, but buffer + // it in `pendingHeaderSection`. If the next line is also a section header, + // we discard the value in `pendingHeaderSection` and fill it with the new + // section header. If we encounter a non-section header line, we flush the + // value of `pendingHeaderSection` to the file, before adding the + // non-section header line. + // + // TODO(freax13): One day, go-toml might add an option to omit empty + // sections: https://github.com/pelletier/go-toml/issues/957 + var pendingHeaderSection []byte + scanner := bufio.NewScanner(bytes.NewReader(rawNewPluginConfig)) + for scanner.Scan() { + // Is the line a section header? + if strings.HasPrefix(scanner.Text(), "[") { + pendingHeaderSection = scanner.Bytes() + continue + } + if len(pendingHeaderSection) != 0 { + newRawConfig = append(newRawConfig, pendingHeaderSection...) + newRawConfig = append(newRawConfig, []byte("\n")...) + pendingHeaderSection = []byte{} + } + newRawConfig = append(newRawConfig, scanner.Bytes()...) + newRawConfig = append(newRawConfig, []byte("\n")...) + } return os.WriteFile(configTemplatePath, newRawConfig, os.ModePerm) } diff --git a/node-installer/node-installer_test.go b/node-installer/node-installer_test.go index 16096fe15e..76ce5dda2c 100644 --- a/node-installer/node-installer_test.go +++ b/node-installer/node-installer_test.go @@ -18,9 +18,19 @@ import ( var ( //go:embed testdata/expected-aks-clh-snp.toml expectedConfAKSCLHSNP []byte - //go:embed testdata/expected-bare-metal-qemu-tdx.toml expectedConfBareMetalQEMUTDX []byte + //go:embed testdata/expected-bare-metal-qemu-snp.toml + expectedConfBareMetalQEMUSNP []byte + + //go:embed testdata/input-bare-metal-qemu-tdx.toml.tmpl + inputConfTmplBareMetalQEMUTDX []byte + //go:embed testdata/expected-bare-metal-qemu-tdx.toml.tmpl + expectedConfTmplBareMetalQEMUTDX []byte + //go:embed testdata/input-bare-metal-qemu-snp.toml.tmpl + inputConfTmplBareMetalQEMUSNP []byte + //go:embed testdata/expected-bare-metal-qemu-snp.toml.tmpl + expectedConfTmplBareMetalQEMUSNP []byte ) func TestPatchContainerdConfig(t *testing.T) { @@ -37,6 +47,10 @@ func TestPatchContainerdConfig(t *testing.T) { platform: platforms.K3sQEMUTDX, expected: expectedConfBareMetalQEMUTDX, }, + "BareMetalQEMUSNP": { + platform: platforms.K3sQEMUSNP, + expected: expectedConfBareMetalQEMUSNP, + }, "Unknown": { platform: platforms.Unknown, wantErr: true, @@ -68,3 +82,59 @@ func TestPatchContainerdConfig(t *testing.T) { }) } } + +func TestPatchContainerdConfigTemplate(t *testing.T) { + testCases := map[string]struct { + platform platforms.Platform + input []byte + expected []byte + }{ + "BareMetalQEMUTDX": { + platform: platforms.K3sQEMUTDX, + input: inputConfTmplBareMetalQEMUTDX, + expected: expectedConfTmplBareMetalQEMUTDX, + }, + "BareMetalQEMUSNP": { + platform: platforms.K3sQEMUSNP, + input: inputConfTmplBareMetalQEMUSNP, + expected: expectedConfTmplBareMetalQEMUSNP, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + tmpDir, err := os.MkdirTemp("", "patch-containerd-config-test") + require.NoError(err) + t.Cleanup(func() { _ = os.RemoveAll(tmpDir) }) + + // Unlike patchContainerdConfig, patchContainerdConfigTemplate + // requires the file to exist already. Create one. + configTemplatePath := filepath.Join(tmpDir, "config.toml.tmpl") + err = os.WriteFile(configTemplatePath, tc.input, os.ModePerm) + require.NoError(err) + + // Testing patching a config template. + + err = patchContainerdConfigTemplate("my-runtime", "/opt/edgeless/my-runtime", + configTemplatePath, tc.platform) + require.NoError(err) + + configData, err := os.ReadFile(configTemplatePath) + require.NoError(err) + assert.Equal(string(tc.expected), string(configData)) + + // Test that patching the same template twice doesn't change it. + + err = patchContainerdConfigTemplate("my-runtime", "/opt/edgeless/my-runtime", + configTemplatePath, tc.platform) + require.NoError(err) + + configData, err = os.ReadFile(configTemplatePath) + require.NoError(err) + assert.Equal(string(tc.expected), string(configData)) + }) + } +} diff --git a/node-installer/platforms/platforms.go b/node-installer/platforms/platforms.go index 1f9f3a745d..92966870c0 100644 --- a/node-installer/platforms/platforms.go +++ b/node-installer/platforms/platforms.go @@ -20,13 +20,15 @@ const ( AKSCloudHypervisorSNP // K3sQEMUTDX represents a deployment with QEMU on bare-metal TDX K3s. K3sQEMUTDX + // K3sQEMUSNP represents a deployment with QEMU on bare-metal SNP K3s. + K3sQEMUSNP // RKE2QEMUTDX represents a deployment with QEMU on bare-metal TDX RKE2. RKE2QEMUTDX ) // All returns a list of all available platforms. func All() []Platform { - return []Platform{AKSCloudHypervisorSNP, K3sQEMUTDX, RKE2QEMUTDX} + return []Platform{AKSCloudHypervisorSNP, K3sQEMUTDX, K3sQEMUSNP, RKE2QEMUTDX} } // AllStrings returns a list of all available platforms as strings. @@ -45,6 +47,8 @@ func (p Platform) String() string { return "AKS-CLH-SNP" case K3sQEMUTDX: return "K3s-QEMU-TDX" + case K3sQEMUSNP: + return "K3s-QEMU-SNP" case RKE2QEMUTDX: return "RKE2-QEMU-TDX" default: @@ -59,6 +63,8 @@ func FromString(s string) (Platform, error) { return AKSCloudHypervisorSNP, nil case "k3s-qemu-tdx": return K3sQEMUTDX, nil + case "k3s-qemu-snp": + return K3sQEMUSNP, nil case "rke2-qemu-tdx": return RKE2QEMUTDX, nil default: diff --git a/node-installer/testdata/expected-bare-metal-qemu-snp.toml b/node-installer/testdata/expected-bare-metal-qemu-snp.toml new file mode 100644 index 0000000000..33e4ad5624 --- /dev/null +++ b/node-installer/testdata/expected-bare-metal-qemu-snp.toml @@ -0,0 +1,79 @@ +version = 2 +root = '' +state = '' +temp = '' +plugin_dir = '' +disabled_plugins = [] +required_plugins = [] +oom_score = 0 +imports = [] + +[metrics] +address = '0.0.0.0:10257' + +[plugins] +[plugins.'io.containerd.grpc.v1.cri'] +sandbox_image = 'mcr.microsoft.com/oss/kubernetes/pause:3.6' + +[plugins.'io.containerd.grpc.v1.cri'.cni] +bin_dir = '/opt/cni/bin' +conf_dir = '/etc/cni/net.d' +conf_template = '/etc/containerd/kubenet_template.conf' + +[plugins.'io.containerd.grpc.v1.cri'.containerd] +default_runtime_name = 'runc' +disable_snapshot_annotations = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata] +runtime_type = 'io.containerd.kata.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc] +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true +runtime_type = 'io.containerd.kata-cc.v2' +snapshotter = 'tardev' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.kata-cc.options] +ConfigPath = '/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli] +runtime_type = 'io.containerd.runc.v1' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.katacli.options] +BinaryName = '/usr/bin/kata-runtime' +CriuPath = '' +IoGid = 0 +IoUid = 0 +NoNewKeyring = false +NoPivotRoot = false +Root = '' +ShimCgroup = '' +SystemdCgroup = false + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] +runtime_type = 'io.containerd.contrast-cc.v2' +runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] +ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-qemu-snp.toml' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted] +runtime_type = 'io.containerd.runc.v2' + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.untrusted.options] +BinaryName = '/usr/bin/runc' + +[plugins.'io.containerd.grpc.v1.cri'.registry] +config_path = '/etc/containerd/certs.d' + +[plugins.'io.containerd.grpc.v1.cri'.registry.headers] +X-Meta-Source-Client = ['azure/aks'] diff --git a/node-installer/testdata/expected-bare-metal-qemu-snp.toml.tmpl b/node-installer/testdata/expected-bare-metal-qemu-snp.toml.tmpl new file mode 100644 index 0000000000..349d39770e --- /dev/null +++ b/node-installer/testdata/expected-bare-metal-qemu-snp.toml.tmpl @@ -0,0 +1,10 @@ +{{ template "base" . }} + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] +runtime_type = 'io.containerd.contrast-cc.v2' +runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] +ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-qemu-snp.toml' diff --git a/node-installer/testdata/expected-bare-metal-qemu-tdx.toml.tmpl b/node-installer/testdata/expected-bare-metal-qemu-tdx.toml.tmpl new file mode 100644 index 0000000000..182cc742e3 --- /dev/null +++ b/node-installer/testdata/expected-bare-metal-qemu-tdx.toml.tmpl @@ -0,0 +1,10 @@ +{{ template "base" . }} + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime] +runtime_type = 'io.containerd.contrast-cc.v2' +runtime_path = '/opt/edgeless/my-runtime/bin/containerd-shim-contrast-cc-v2' +pod_annotations = ['io.katacontainers.*'] +privileged_without_host_devices = true + +[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.my-runtime.options] +ConfigPath = '/opt/edgeless/my-runtime/etc/configuration-qemu-tdx.toml' diff --git a/node-installer/testdata/input-bare-metal-qemu-snp.toml.tmpl b/node-installer/testdata/input-bare-metal-qemu-snp.toml.tmpl new file mode 100644 index 0000000000..99435df5ec --- /dev/null +++ b/node-installer/testdata/input-bare-metal-qemu-snp.toml.tmpl @@ -0,0 +1 @@ +{{ template "base" . }} diff --git a/node-installer/testdata/input-bare-metal-qemu-tdx.toml.tmpl b/node-installer/testdata/input-bare-metal-qemu-tdx.toml.tmpl new file mode 100644 index 0000000000..99435df5ec --- /dev/null +++ b/node-installer/testdata/input-bare-metal-qemu-tdx.toml.tmpl @@ -0,0 +1 @@ +{{ template "base" . }} diff --git a/packages/by-name/OVMF-SNP/package.nix b/packages/by-name/OVMF-SNP/package.nix new file mode 100644 index 0000000000..ccb8abe070 --- /dev/null +++ b/packages/by-name/OVMF-SNP/package.nix @@ -0,0 +1,41 @@ +# Copyright 2024 Edgeless Systems GmbH +# SPDX-License-Identifier: AGPL-3.0-only + +{ + edk2, + fetchFromGitHub, + nasm, + acpica-tools, +}: + +edk2.mkDerivation "OvmfPkg/AmdSev/AmdSevX64.dsc" rec { + name = "OVMF-SNP"; + src = fetchFromGitHub { + owner = "amdese"; + repo = "ovmf"; + # https://github.com/AMDESE/ovmf/tree/apic-mmio-fix4 + rev = "64b3116ed087f8cb026201e56e42efe751e2cf7d"; + fetchSubmodules = true; + hash = "sha256-nb4p01Y+M5a3EEJb9692hcFkU7HgpbG1rZa60T+I8N4="; + }; + postPatch = '' + touch OvmfPkg/AmdSev/Grub/grub.efi + ''; + # Disable making all warnings errors. Nix's GCC is fairly new, so it spews a + # few more warnings, but that shouldn't prevent us from building OVMF. + postConfigure = '' + sed -i "s/-Werror//g" Conf/tools_def.txt + ''; + + nativeBuildInputs = [ + nasm + acpica-tools + ]; + + hardeningDisable = [ + "format" + "stackprotector" + "pic" + "fortify" + ]; +} diff --git a/packages/by-name/contrast-node-installer/package.nix b/packages/by-name/contrast-node-installer/package.nix index 257ed47bcd..b2d29ddc70 100644 --- a/packages/by-name/contrast-node-installer/package.nix +++ b/packages/by-name/contrast-node-installer/package.nix @@ -24,6 +24,7 @@ buildGoModule { (path.append root "go.mod") (path.append root "go.sum") (fileset.fileFilter (file: hasSuffix ".toml" file.name) root) + (fileset.fileFilter (file: hasSuffix ".toml.tmpl" file.name) root) (fileset.fileFilter (file: hasSuffix ".go" file.name) root) ]; }; diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 44fa6d6409..71dd1acd02 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -83,7 +83,7 @@ buildGoModule rec { }; proxyVendor = true; - vendorHash = "sha256-Tl1caSm7BfM5QkI61wsvFxmKT7V0izu47wGrvUkV3jE="; + vendorHash = "sha256-1gsPupNX4L1vmmxecAnzYnkvRCtwvkI72hNLEyQE9Gk="; nativeBuildInputs = [ installShellFiles ]; diff --git a/packages/by-name/kata/contrast-node-installer-image/package.nix b/packages/by-name/kata/contrast-node-installer-image/package.nix index 9cdeea5919..7af9ccdf50 100644 --- a/packages/by-name/kata/contrast-node-installer-image/package.nix +++ b/packages/by-name/kata/contrast-node-installer-image/package.nix @@ -62,6 +62,18 @@ let url = "file:///opt/edgeless/bin/kata-runtime"; path = "/opt/edgeless/${runtime-handler}/bin/kata-runtime"; } + { + url = "file:///opt/edgeless/share/qemu/kvmvapic.bin"; + path = "/opt/edgeless/${runtime-handler}/share/qemu/kvmvapic.bin"; + } + { + url = "file:///opt/edgeless/share/qemu/linuxboot_dma.bin"; + path = "/opt/edgeless/${runtime-handler}/share/qemu/linuxboot_dma.bin"; + } + { + url = "file:///opt/edgeless/share/qemu/efi-virtio.rom"; + path = "/opt/edgeless/${runtime-handler}/share/qemu/efi-virtio.rom"; + } ]; runtimeHandlerName = runtime-handler; inherit (kata.runtime-class-files) debugRuntime; @@ -99,6 +111,18 @@ let source = kata.runtime-class-files.qemu-bin; destination = "/opt/edgeless/bin/qemu-system-x86_64"; } + { + source = "${kata.runtime-class-files.qemu-share}/kvmvapic.bin"; + destination = "/opt/edgeless/share/qemu/kvmvapic.bin"; + } + { + source = "${kata.runtime-class-files.qemu-share}/linuxboot_dma.bin"; + destination = "/opt/edgeless/share/qemu/linuxboot_dma.bin"; + } + { + source = "${kata.runtime-class-files.qemu-share}/efi-virtio.rom"; + destination = "/opt/edgeless/share/qemu/efi-virtio.rom"; + } ]; }; diff --git a/packages/by-name/kata/genpolicy/package.nix b/packages/by-name/kata/genpolicy/package.nix index 124c532ad0..89382b7053 100644 --- a/packages/by-name/kata/genpolicy/package.nix +++ b/packages/by-name/kata/genpolicy/package.nix @@ -51,7 +51,7 @@ rustPlatform.buildRustPackage rec { settings = fetchurl { name = "${pname}-${version}-settings"; url = "https://raw.githubusercontent.com/kata-containers/kata-containers/${version}/src/tools/genpolicy/genpolicy-settings.json"; - hash = "sha256-Rlm1BOo0/yNHBf17p2Mk7ta6VbaGcrgezCk8mraFPtU="; + hash = "sha256-4uBxU71wwvS2vMVxSizTBmy+C+VXIeAHgcrATgaqgD4="; downloadToTemp = true; recursiveHash = true; postFetch = "install -D $downloadedFile $out/genpolicy-settings.json"; @@ -60,7 +60,7 @@ rustPlatform.buildRustPackage rec { rules = fetchurl { name = "${pname}-${version}-rules"; url = "https://raw.githubusercontent.com/kata-containers/kata-containers/${version}/src/tools/genpolicy/rules.rego"; - hash = "sha256-J4WIgEgCzm3vEji9f/0kF+gLdE8ziio4PAyRWUJjqZk="; + hash = "sha256-AAO0bsM1pcsafR6YHbqH9NbPMFPQty9o+jLSUmYfScs="; downloadToTemp = true; recursiveHash = true; postFetch = "install -D $downloadedFile $out/genpolicy-rules.rego"; diff --git a/packages/by-name/kata/kata-kernel-uvm/package.nix b/packages/by-name/kata/kata-kernel-uvm/package.nix index b157119427..030241f189 100644 --- a/packages/by-name/kata/kata-kernel-uvm/package.nix +++ b/packages/by-name/kata/kata-kernel-uvm/package.nix @@ -20,9 +20,8 @@ let # We don't use an initrd. postPatch = '' - cat <<- EOF > kata/share/kata-containers/config-6.7-132-confidential - CONFIG_INITRAMFS_SOURCE="" - EOF + substituteInPlace kata/share/kata-containers/config-6.7-132-confidential \ + --replace-fail 'CONFIG_INITRAMFS_SOURCE="initramfs.cpio.gz"' 'CONFIG_INITRAMFS_SOURCE=""' ''; dontBuild = true; diff --git a/packages/by-name/kata/runtime-class-files/package.nix b/packages/by-name/kata/runtime-class-files/package.nix index 6e99d164ef..cd60e7f4c5 100644 --- a/packages/by-name/kata/runtime-class-files/package.nix +++ b/packages/by-name/kata/runtime-class-files/package.nix @@ -4,7 +4,7 @@ { stdenvNoCC, kata, - OVMF, + OVMF-SNP, debugRuntime ? false, qemu-static, }: @@ -14,8 +14,9 @@ let kernel = "${kata.kata-kernel-uvm}/bzImage"; qemu-bin = "${qemu-static}/bin/qemu-system-x86_64"; + qemu-share = "${qemu-static}/share/qemu"; - ovmf = "${OVMF.fd}/FV/OVMF.fd"; + ovmf = "${OVMF-SNP}/FV/OVMF.fd"; containerd-shim-contrast-cc-v2 = "${kata.kata-runtime}/bin/containerd-shim-kata-v2"; @@ -31,7 +32,7 @@ stdenvNoCC.mkDerivation { # TODO(msanft): perform the actual launch digest calculation. buildPhase = '' mkdir -p $out - sha256sum ${image} ${kernel} ${qemu-bin} ${containerd-shim-contrast-cc-v2} ${ovmf} | sha256sum | cut -d " " -f 1 > $out/launch-digest.hex + sha256sum ${image} ${kernel} ${qemu-bin} ${qemu-share}/kvmvapic.bin ${qemu-share}/linuxboot_dma.bin ${qemu-share}/efi-virtio.rom ${containerd-shim-contrast-cc-v2} ${ovmf} | sha256sum | cut -d " " -f 1 > $out/launch-digest.hex printf "contrast-cc-%s" "$(cat $out/launch-digest.hex | head -c 32)" > $out/runtime-handler ''; @@ -40,6 +41,7 @@ stdenvNoCC.mkDerivation { kernel image qemu-bin + qemu-share containerd-shim-contrast-cc-v2 ovmf kata-runtime