From 1309bc5d5fb16996de06151712fdcda4c783c01a Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:31:31 +0200 Subject: [PATCH 1/5] kuberesource: move to internal --- e2e/internal/contrasttest/contrasttest.go | 2 +- e2e/openssl/openssl_test.go | 2 +- e2e/servicemesh/servicemesh_test.go | 2 +- {e2e/internal => internal}/kuberesource/constants.go | 0 {e2e/internal => internal}/kuberesource/lookup.go | 0 {e2e/internal => internal}/kuberesource/mutators.go | 0 {e2e/internal => internal}/kuberesource/parts.go | 0 {e2e/internal => internal}/kuberesource/parts_test.go | 0 {e2e/internal => internal}/kuberesource/resourcegen/main.go | 2 +- {e2e/internal => internal}/kuberesource/sets.go | 0 {e2e/internal => internal}/kuberesource/wrappers.go | 0 {e2e/internal => internal}/kuberesource/writer.go | 4 ++-- packages/by-name/contrast/package.nix | 6 +++--- 13 files changed, 9 insertions(+), 9 deletions(-) rename {e2e/internal => internal}/kuberesource/constants.go (100%) rename {e2e/internal => internal}/kuberesource/lookup.go (100%) rename {e2e/internal => internal}/kuberesource/mutators.go (100%) rename {e2e/internal => internal}/kuberesource/parts.go (100%) rename {e2e/internal => internal}/kuberesource/parts_test.go (100%) rename {e2e/internal => internal}/kuberesource/resourcegen/main.go (97%) rename {e2e/internal => internal}/kuberesource/sets.go (100%) rename {e2e/internal => internal}/kuberesource/wrappers.go (100%) rename {e2e/internal => internal}/kuberesource/writer.go (93%) diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index fb4d503b05..7a649cd571 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -19,8 +19,8 @@ import ( "github.com/edgelesssys/contrast/cli/cmd" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" - "github.com/edgelesssys/contrast/e2e/internal/kuberesource" "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/stretchr/testify/require" ) diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index c094a2d183..18ebc64da4 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -18,7 +18,7 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" - "github.com/edgelesssys/contrast/e2e/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/stretchr/testify/require" ) diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go index 2db09d08bd..a7f8087113 100644 --- a/e2e/servicemesh/servicemesh_test.go +++ b/e2e/servicemesh/servicemesh_test.go @@ -20,7 +20,7 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" - "github.com/edgelesssys/contrast/e2e/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/stretchr/testify/require" ) diff --git a/e2e/internal/kuberesource/constants.go b/internal/kuberesource/constants.go similarity index 100% rename from e2e/internal/kuberesource/constants.go rename to internal/kuberesource/constants.go diff --git a/e2e/internal/kuberesource/lookup.go b/internal/kuberesource/lookup.go similarity index 100% rename from e2e/internal/kuberesource/lookup.go rename to internal/kuberesource/lookup.go diff --git a/e2e/internal/kuberesource/mutators.go b/internal/kuberesource/mutators.go similarity index 100% rename from e2e/internal/kuberesource/mutators.go rename to internal/kuberesource/mutators.go diff --git a/e2e/internal/kuberesource/parts.go b/internal/kuberesource/parts.go similarity index 100% rename from e2e/internal/kuberesource/parts.go rename to internal/kuberesource/parts.go diff --git a/e2e/internal/kuberesource/parts_test.go b/internal/kuberesource/parts_test.go similarity index 100% rename from e2e/internal/kuberesource/parts_test.go rename to internal/kuberesource/parts_test.go diff --git a/e2e/internal/kuberesource/resourcegen/main.go b/internal/kuberesource/resourcegen/main.go similarity index 97% rename from e2e/internal/kuberesource/resourcegen/main.go rename to internal/kuberesource/resourcegen/main.go index 097ed0ec20..b17989fa70 100644 --- a/e2e/internal/kuberesource/resourcegen/main.go +++ b/internal/kuberesource/resourcegen/main.go @@ -9,7 +9,7 @@ import ( "log" "os" - "github.com/edgelesssys/contrast/e2e/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/kuberesource" ) func main() { diff --git a/e2e/internal/kuberesource/sets.go b/internal/kuberesource/sets.go similarity index 100% rename from e2e/internal/kuberesource/sets.go rename to internal/kuberesource/sets.go diff --git a/e2e/internal/kuberesource/wrappers.go b/internal/kuberesource/wrappers.go similarity index 100% rename from e2e/internal/kuberesource/wrappers.go rename to internal/kuberesource/wrappers.go diff --git a/e2e/internal/kuberesource/writer.go b/internal/kuberesource/writer.go similarity index 93% rename from e2e/internal/kuberesource/writer.go rename to internal/kuberesource/writer.go index 8904c223fd..813f6ff538 100644 --- a/e2e/internal/kuberesource/writer.go +++ b/internal/kuberesource/writer.go @@ -6,12 +6,12 @@ package kuberesource import ( "bytes" - "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" ) -// EncodeResources encodes a list of resources into a single YAML document. +// EncodeResources encodes a list of apply configurations into a single YAML document. func EncodeResources(resources ...any) ([]byte, error) { unstructuredResources, err := ResourcesToUnstructured(resources) if err != nil { diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 28c1cbcf4b..e4748f5f5b 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -19,7 +19,7 @@ let ldflags = [ "-s" "-X github.com/edgelesssys/contrast/internal/manifest.trustedMeasurement=${launchDigest}" - "-X github.com/edgelesssys/contrast/e2e/internal/kuberesource.runtimeHandler=${runtimeHandler}" + "-X github.com/edgelesssys/contrast/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; @@ -62,7 +62,7 @@ buildGoModule rec { proxyVendor = true; vendorHash = "sha256-i+7DhygotCNhczpaZlI9O7enKVOW7smauOKcGQhOtzI="; - subPackages = packageOutputs ++ [ "e2e/internal/kuberesource/resourcegen" ]; + subPackages = packageOutputs ++ [ "internal/kuberesource/resourcegen" ]; prePatch = '' install -D ${lib.getExe genpolicy} cli/cmd/assets/genpolicy @@ -77,7 +77,7 @@ buildGoModule rec { "-X main.version=v${version}" "-X github.com/edgelesssys/contrast/internal/manifest.trustedMeasurement=${launchDigest}" "-X github.com/edgelesssys/contrast/cli/cmd.runtimeHandler=${runtimeHandler}" - "-X github.com/edgelesssys/contrast/e2e/internal/kuberesource.runtimeHandler=${runtimeHandler}" + "-X github.com/edgelesssys/contrast/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; preCheck = '' From 2642f9c8662e3afbe83b096da3f006f416873f3b Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:29:24 +0200 Subject: [PATCH 2/5] kuberesource: implement unmarshaling of applyconfigurations --- internal/kuberesource/reader.go | 32 +++++++++++++++++++++ internal/kuberesource/reader_writer_test.go | 32 +++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 internal/kuberesource/reader.go create mode 100644 internal/kuberesource/reader_writer_test.go diff --git a/internal/kuberesource/reader.go b/internal/kuberesource/reader.go new file mode 100644 index 0000000000..3b08a5e5df --- /dev/null +++ b/internal/kuberesource/reader.go @@ -0,0 +1,32 @@ +// Copyright 2024 Edgeless Systems GmbH +// SPDX-License-Identifier: AGPL-3.0-only + +package kuberesource + +import ( + "fmt" + + "github.com/edgelesssys/contrast/internal/kubeapi" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/applyconfigurations" +) + +// UnmarshalApplyConfigurations unmarshals a YAML document into a list of ApplyConfigurations. +func UnmarshalApplyConfigurations(data []byte) ([]any, error) { + objs, err := kubeapi.UnmarshalUnstructuredK8SResource(data) + if err != nil { + return nil, err + } + var result []any + for _, obj := range objs { + applyConfig := applyconfigurations.ForKind(obj.GetObjectKind().GroupVersionKind()) + if applyConfig == nil { + return nil, fmt.Errorf("unmarshalling: unsupported resource type %s for %q", obj.GroupVersionKind().String(), obj.GetName()) + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), applyConfig); err != nil { + return nil, err + } + result = append(result, applyConfig) + } + return result, nil +} diff --git a/internal/kuberesource/reader_writer_test.go b/internal/kuberesource/reader_writer_test.go new file mode 100644 index 0000000000..e75c911464 --- /dev/null +++ b/internal/kuberesource/reader_writer_test.go @@ -0,0 +1,32 @@ +// Copyright 2024 Edgeless Systems GmbH +// SPDX-License-Identifier: AGPL-3.0-only + +package kuberesource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncodeDecode(t *testing.T) { + require := require.New(t) + + fixture := `apiVersion: v1 +kind: Pod +metadata: + name: foo +spec: + containers: + - image: image + name: bar +` + + resources, err := UnmarshalApplyConfigurations([]byte(fixture)) + require.NoError(err) + + got, err := EncodeResources(resources...) + require.NoError(err) + + require.Equal(fixture, string(got)) +} From 40d63ee979e92ddd54f6b90005b411ee36918eea Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:29:53 +0200 Subject: [PATCH 3/5] kuberesource: allow mutation of nested pod specs --- internal/kuberesource/mutators.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/internal/kuberesource/mutators.go b/internal/kuberesource/mutators.go index 5e2f84b8c1..a078f0c3ba 100644 --- a/internal/kuberesource/mutators.go +++ b/internal/kuberesource/mutators.go @@ -7,6 +7,7 @@ import ( "errors" applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1" + applybatchv1 "k8s.io/client-go/applyconfigurations/batch/v1" applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" ) @@ -123,3 +124,29 @@ func AddLogging(resources []any, level string) []any { } return resources } + +// MapPodSpec applies a function to a PodSpec in a Kubernetes resource. +func MapPodSpec(resource any, f func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration) any { + if resource == nil { + return nil + } + switch r := resource.(type) { + case *applybatchv1.CronJobApplyConfiguration: + r.Spec.JobTemplate.Spec.Template.Spec = f(r.Spec.JobTemplate.Spec.Template.Spec) + case *applyappsv1.DaemonSetApplyConfiguration: + r.Spec.Template.Spec = f(r.Spec.Template.Spec) + case *applyappsv1.DeploymentApplyConfiguration: + r.Spec.Template.Spec = f(r.Spec.Template.Spec) + case *applybatchv1.JobApplyConfiguration: + r.Spec.Template.Spec = f(r.Spec.Template.Spec) + case *applycorev1.PodApplyConfiguration: + r.Spec = f(r.Spec) + case *applyappsv1.ReplicaSetApplyConfiguration: + r.Spec.Template.Spec = f(r.Spec.Template.Spec) + case *applycorev1.ReplicationControllerApplyConfiguration: + r.Spec.Template.Spec = f(r.Spec.Template.Spec) + case *applyappsv1.StatefulSetApplyConfiguration: + r.Spec.Template.Spec = f(r.Spec.Template.Spec) + } + return resource +} From 65e714459aff1cca5f2c7e71683d6c9e0d91c154 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:30:13 +0200 Subject: [PATCH 4/5] generate: automatic replacement of runtimeClassName --- cli/cmd/generate.go | 53 +++++++++++++++++++++++++++ packages/by-name/contrast/package.nix | 1 + 2 files changed, 54 insertions(+) diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index 4cb16c0a7e..7c55cad1ab 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -24,7 +24,10 @@ import ( "strings" "github.com/edgelesssys/contrast/internal/embedbin" + "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" + applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" + "github.com/spf13/cobra" ) @@ -82,6 +85,9 @@ func runGenerate(cmd *cobra.Command, args []string) error { return err } + if err := patchTargets(paths, log); err != nil { + return fmt.Errorf("failed to patch targets: %w", err) + } if err := generatePolicies(cmd.Context(), flags.policyPath, flags.settingsPath, paths, log); err != nil { return fmt.Errorf("failed to generate policies: %w", err) } @@ -231,6 +237,53 @@ func generatePolicies(ctx context.Context, regoRulesPath, policySettingsPath str return nil } +func patchTargets(paths []string, logger *slog.Logger) error { + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + kubeObjs, err := kuberesource.UnmarshalApplyConfigurations(data) + if err != nil { + return fmt.Errorf("failed to unmarshal %s: %w", path, err) + } + + var changed bool + replaceRuntimeClassName := runtimeClassNamePatcher(&changed) + for i := range kubeObjs { + kubeObjs[i] = kuberesource.MapPodSpec(kubeObjs[i], replaceRuntimeClassName) + } + + if !changed { + logger.Debug("No changes needed for yaml file", "path", path) + continue + } + logger.Debug("Updating resources in yaml file", "path", path) + resource, err := kuberesource.EncodeResources(kubeObjs...) + if err != nil { + return err + } + if err := os.WriteFile(path, resource, os.ModePerm); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + } + return nil +} + +func runtimeClassNamePatcher(modified *bool) func(*applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { + return func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { + if spec.RuntimeClassName == nil || *spec.RuntimeClassName == runtimeHandler { + return spec + } + + if strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") || *spec.RuntimeClassName == "kata-cc-isolation" { + *modified = true + spec.RuntimeClassName = &runtimeHandler + } + return spec + } +} + func addWorkloadOwnerKeyToManifest(manifst *manifest.Manifest, keyPath string) error { keyData, err := os.ReadFile(keyPath) if err != nil { diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index e4748f5f5b..2a8b9e4c4a 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -19,6 +19,7 @@ let ldflags = [ "-s" "-X github.com/edgelesssys/contrast/internal/manifest.trustedMeasurement=${launchDigest}" + "-X github.com/edgelesssys/contrast/cli/cmd.runtimeHandler=${runtimeHandler}" "-X github.com/edgelesssys/contrast/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; From f39973a50cfb53877ceb85d0effc4400d1712955 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:35:55 +0200 Subject: [PATCH 5/5] update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7af04500f0..0b9ed8fda5 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( k8s.io/apimachinery v0.30.0 k8s.io/client-go v0.30.0 k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -66,5 +67,4 @@ require ( k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect )