From 5df8826c3b5e2c1d8c8f782fa616b19efd83330c Mon Sep 17 00:00:00 2001 From: Gennady Azarenkov Date: Mon, 6 Jan 2025 20:08:26 +0200 Subject: [PATCH] Configure other than Backstage container to mount PVC volume to (#582) * Add containers annotation to default PVC Signed-off-by: gazarenkov * use PVC for dynamic-plugins volume in RHDH default config Signed-off-by: gazarenkov * regenerate rhdh default configmap Signed-off-by: gazarenkov * rollback rhdh config Signed-off-by: gazarenkov * docs refinement Signed-off-by: gazarenkov * docs refinement Signed-off-by: gazarenkov --------- Signed-off-by: gazarenkov --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- api/v1alpha2/zz_generated.deepcopy.go | 2 +- api/v1alpha3/zz_generated.deepcopy.go | 2 +- ...kstage-operator.clusterserviceversion.yaml | 2 +- .../rhdh-default-config_v1_configmap.yaml | 2 +- .../rhdh/default-config/deployment.yaml | 2 +- docs/configuration.md | 58 +++++++++++++------ pkg/model/deployment.go | 22 +++++++ pkg/model/pvcs.go | 22 +++++-- pkg/model/pvcs_test.go | 25 ++++++++ pkg/model/runtime.go | 1 + pkg/model/testdata/multi-pvc-containers.yaml | 44 ++++++++++++++ .../testdata/multicontainer-deployment.yaml | 24 ++++++++ pkg/utils/pod-mutator.go | 2 +- pkg/utils/utils.go | 20 +++++++ pkg/utils/utils_test.go | 19 ++++++ 16 files changed, 219 insertions(+), 30 deletions(-) create mode 100644 pkg/model/testdata/multi-pvc-containers.yaml create mode 100644 pkg/model/testdata/multicontainer-deployment.yaml diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9b7acdd3..ca4d7a34 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5,7 +5,7 @@ package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index c3ecd6b2..1357c08b 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -5,7 +5,7 @@ package v1alpha2 import ( - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index b315b789..6613a889 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -5,7 +5,7 @@ package v1alpha3 import ( - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml index 1f3a115d..58b5e431 100644 --- a/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml +++ b/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml @@ -39,7 +39,7 @@ metadata: categories: Developer Tools certified: "true" containerImage: registry-proxy.engineering.redhat.com/rh-osbs/rhdh-rhdh-rhel9-operator:1.3 - createdAt: "2024-12-13T14:46:16Z" + createdAt: "2024-12-19T17:20:07Z" description: Red Hat Developer Hub is a Red Hat supported version of Backstage. It comes with pre-built plug-ins and configuration settings, supports use of an external database, and can help streamline the process of setting up a self-managed diff --git a/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml b/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml index 41284e85..3e247e6d 100644 --- a/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml +++ b/bundle/rhdh/manifests/rhdh-default-config_v1_configmap.yaml @@ -210,7 +210,7 @@ data: type: RuntimeDefault capabilities: drop: - - ALL + - ALL env: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins diff --git a/config/profile/rhdh/default-config/deployment.yaml b/config/profile/rhdh/default-config/deployment.yaml index 608e6221..c9c54d2c 100644 --- a/config/profile/rhdh/default-config/deployment.yaml +++ b/config/profile/rhdh/default-config/deployment.yaml @@ -55,7 +55,7 @@ spec: type: RuntimeDefault capabilities: drop: - - ALL + - ALL env: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins diff --git a/docs/configuration.md b/docs/configuration.md index b729b671..fcc3a07d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,21 +12,21 @@ The Default Configuration defines the structure of all Backstage instances withi ### Default Configuration Files -| Key/File Name | Object Kind | Object Name | Mandatory | Multi| Version | Notes | -|-----------------------------|------------------------------|-------------------------------------|--------------|-----|---------|----------------------------------------------------------| -| deployment.yaml | appsv1.Deployment | backstage- | Yes | No | >=0.1.x | Backstage deployment | -| service.yaml | corev1.Service | backstage- | Yes | No | >=0.1.x | Backstage Service | -| db-statefulset.yaml | appsv1.StatefulSet | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL StatefulSet | -| db-service.yaml | corev1.Service | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL Service | -| db-secret.yaml | corev1.Secret | backstage-psql-secret- | For local DB | No | >=0.1.x | Secret to connect Backstage to PGSQL | -| route.yaml | openshift.Route | backstage- | No (for OCP) | No | >=0.1.x | Route exposing Backstage service | -| app-config.yaml | corev1.ConfigMap | backstage-appconfig- | No | No | >=0.2.x | Backstage app-config.yaml | -| configmap-files.yaml | corev1.ConfigMap | backstage-files- | No | No | >=0.2.x | Backstage config file inclusions from configMap | -| configmap-envs.yaml | corev1.ConfigMap | backstage-envs- | No | No | >=0.2.x | Backstage environment variables from ConfigMap | -| secret-files.yaml | corev1.Secret | backstage-files- | No | No | >=0.2.x | Backstage config file inclusions from Secret | -| secret-envs.yaml | corev1.Secret | backstage-envs- | No | No | >=0.2.x | Backstage environment variables from Secret | -| dynamic-plugins.yaml | corev1.ConfigMap | backstage-dynamic-plugins- | No | No | >=0.2.x | Dynamic plugins configuration | -| pvcs.yaml | corev1.PersistentVolumeClaim | backstage-<cr-name>-<pvc-name> | No | Yes | >=0.4.x | List of PVC objects to be mounted to Backstage container | +| Key/File Name | Object Kind | Object Name | Mandatory | Multi| Version | Notes | +|----------------------|------------------------------|--------------------------------------------|--------------|-----|---------|-------------------------------------------------| +| deployment.yaml | appsv1.Deployment | backstage- | Yes | No | >=0.1.x | Backstage deployment | +| service.yaml | corev1.Service | backstage- | Yes | No | >=0.1.x | Backstage Service | +| db-statefulset.yaml | appsv1.StatefulSet | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL StatefulSet | +| db-service.yaml | corev1.Service | backstage-psql- | For local DB | No | >=0.1.x | PostgreSQL Service | +| db-secret.yaml | corev1.Secret | backstage-psql-secret- | For local DB | No | >=0.1.x | Secret to connect Backstage to PGSQL | +| route.yaml | openshift.Route | backstage- | No (for OCP) | No | >=0.1.x | Route exposing Backstage service | +| app-config.yaml | corev1.ConfigMap | backstage-appconfig- | No | No | >=0.2.x | Backstage app-config.yaml | +| configmap-files.yaml | corev1.ConfigMap | backstage-files- | No | No | >=0.2.x | Backstage config file inclusions from configMap | +| configmap-envs.yaml | corev1.ConfigMap | backstage-envs- | No | No | >=0.2.x | Backstage environment variables from ConfigMap | +| secret-files.yaml | corev1.Secret | backstage-files- | No | No | >=0.2.x | Backstage config file inclusions from Secret | +| secret-envs.yaml | corev1.Secret | backstage-envs- | No | No | >=0.2.x | Backstage environment variables from Secret | +| dynamic-plugins.yaml | corev1.ConfigMap | backstage-dynamic-plugins- | No | No | >=0.2.x | Dynamic plugins configuration | +| pvcs.yaml | corev1.PersistentVolumeClaim | backstage-<cr-name>-<pvc-name> | No | Yes | >=0.4.x | List of PVC objects to be mounted to containers | **Meanings of "Mandatory" Column:** - **Yes** - Must be configured; deployment will fail otherwise. @@ -40,10 +40,11 @@ You can see examples of default configurations as part of the [Operator Profiles Some objects, such as: app-config, configmap-files, secret-files, dynamic-plugins, pvcs, are mounted to the Backstage Container as files or directories. Default mount path is Container's WorkingDir, if not defined it falls to "/opt/app-root/src". -#### Object annotation for mounting a volume to a specific path +#### Object annotation for mounting a PVC volume to a specific path -Using **rhdh.redhat.com/mount-path** annotation it is possible to define the directory where **PersistentVolumeClaim** object will be mounted to Backstage Container. +Use **rhdh.redhat.com/mount-path** annotation to configure mount path for **PersistentVolumeClaim** volume. +_**pvcs.yaml**_ ```yaml apiVersion: v1 kind: PersistentVolumeClaim @@ -56,6 +57,29 @@ metadata: In the example above the PVC called **myclaim** will be mounted to **/mount/path/from/annotation** directory +#### Object annotation for mounting a PVC volume to specific container(s) + +Use **rhdh.redhat.com/containers** annotation to configure containers where **PersistentVolumeClaim** volume will be mounted. + +Options: + +* No or empty annotation means the volume will be mounted to the Backstage container only +* \* (asterisk) means the volume will be mounted to all the containers +* Otherwise, container names separated by commas will be used + +_**pvcs.yaml**_ +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: myclaim + annotations: + rhdh.redhat.com/containers: "init-dynamic-plugins,backstage-backend" +... +``` +In the example above the PVC called **myclaim** will be mounted to **init-dynamic-plugins** and **backstage-backend** containers + + ### Metadata Generation For Backstage to function consistently at runtime, certain metadata values need to be predictable. Therefore, the Operator generates values according to the following rules. Any value for these fields specified in either Default or Raw Configuration will be replaced by the generated values. diff --git a/pkg/model/deployment.go b/pkg/model/deployment.go index d1632f7d..ca8ec40b 100644 --- a/pkg/model/deployment.go +++ b/pkg/model/deployment.go @@ -134,6 +134,28 @@ func (b *BackstageDeployment) container() *corev1.Container { return &b.deployment.Spec.Template.Spec.Containers[BackstageContainerIndex(b.deployment)] } +func (b *BackstageDeployment) containerByName(name string) *corev1.Container { + for i, c := range b.deployment.Spec.Template.Spec.Containers { + if c.Name == name { + return &b.deployment.Spec.Template.Spec.Containers[i] + } + } + for i, c := range b.deployment.Spec.Template.Spec.InitContainers { + if c.Name == name { + return &b.deployment.Spec.Template.Spec.InitContainers[i] + } + } + return nil +} + +func (b *BackstageDeployment) allContainers() []corev1.Container { + containers := []corev1.Container{} + spec := b.deployment.Spec.Template.Spec + containers = append(containers, spec.InitContainers...) + containers = append(containers, spec.Containers...) + return containers +} + func (b *BackstageDeployment) podSpec() *corev1.PodSpec { return &b.deployment.Spec.Template.Spec } diff --git a/pkg/model/pvcs.go b/pkg/model/pvcs.go index 43ba5313..72a3619b 100644 --- a/pkg/model/pvcs.go +++ b/pkg/model/pvcs.go @@ -39,7 +39,7 @@ func addPvcsFromSpec(spec bsv1.BackstageSpec, model *BackstageModel) { subPath = utils.ToRFC1123Label(pvcSpec.Name) } - addPvc(model.backstageDeployment, pvcSpec.Name, mountPath, subPath) + addPvc(model.backstageDeployment, pvcSpec.Name, mountPath, subPath, nil) } } @@ -84,8 +84,9 @@ func (b *BackstagePvcs) updateAndValidate(m *BackstageModel, _ bsv1.Backstage) e mountPath = filepath.Join(m.backstageDeployment.defaultMountPath(), volName) subPath = volName } - addPvc(m.backstageDeployment, pvc.Name, mountPath, subPath) + containers := utils.FilterContainers(m.backstageDeployment.allContainers(), pvc.GetAnnotations()[ContainersAnnotation]) + addPvc(m.backstageDeployment, pvc.Name, mountPath, subPath, containers) } return nil } @@ -99,7 +100,7 @@ func (b *BackstagePvcs) setMetaInfo(backstage bsv1.Backstage, scheme *runtime.Sc } } -func addPvc(bsd *BackstageDeployment, pvcName, mountPath, subPath string) { +func addPvc(bsd *BackstageDeployment, pvcName, mountPath, subPath string, affectedContainers []corev1.Container) { volName := utils.ToRFC1123Label(pvcName) volSrc := corev1.VolumeSource{ @@ -110,7 +111,16 @@ func addPvc(bsd *BackstageDeployment, pvcName, mountPath, subPath string) { bsd.deployment.Spec.Template.Spec.Volumes = append(bsd.deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: volName, VolumeSource: volSrc}) - bsd.container().VolumeMounts = append(bsd.container().VolumeMounts, - corev1.VolumeMount{Name: volName, MountPath: mountPath, SubPath: subPath}) - + if affectedContainers == nil { + // if nothing specified mount to the Backstage container only + bsd.container().VolumeMounts = append(bsd.container().VolumeMounts, + corev1.VolumeMount{Name: volName, MountPath: mountPath, SubPath: subPath}) + } else { + // else mount to the affectedContainers + for _, c := range affectedContainers { + update := bsd.containerByName(c.Name) + update.VolumeMounts = append(update.VolumeMounts, + corev1.VolumeMount{Name: volName, MountPath: mountPath, SubPath: subPath}) + } + } } diff --git a/pkg/model/pvcs_test.go b/pkg/model/pvcs_test.go index 5d8975c0..c682c217 100644 --- a/pkg/model/pvcs_test.go +++ b/pkg/model/pvcs_test.go @@ -62,6 +62,31 @@ func TestDefaultPvcs(t *testing.T) { } +func TestMultiContainersPvc(t *testing.T) { + bs := bsv1.Backstage{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + }, + } + + testObj := createBackstageTest(bs).withDefaultConfig(true).addToDefaultConfig("deployment.yaml", "multicontainer-deployment.yaml").addToDefaultConfig("pvcs.yaml", "multi-pvc-containers.yaml") + model, err := InitObjects(context.TODO(), bs, testObj.externalConfig, true, testObj.scheme) + assert.NoError(t, err) + assert.NotNil(t, model) + assert.Equal(t, 4, len(model.backstageDeployment.allContainers())) + + assert.Equal(t, 3, len(model.backstageDeployment.podSpec().Volumes)) + // myclaim1(default), myclaim2(listed), myclaim3(*) + assert.Equal(t, 3, len(model.backstageDeployment.containerByName("backstage-backend").VolumeMounts)) + // myclaim2(listed), myclaim3(*) + assert.Equal(t, 2, len(model.backstageDeployment.containerByName("install-dynamic-plugins").VolumeMounts)) + // myclaim3(*) + assert.Equal(t, 1, len(model.backstageDeployment.containerByName("another-container").VolumeMounts)) + // myclaim3(*) + assert.Equal(t, 1, len(model.backstageDeployment.containerByName("another-init-container").VolumeMounts)) + +} + func TestSpecifiedPvcs(t *testing.T) { bs := bsv1.Backstage{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/model/runtime.go b/pkg/model/runtime.go index acd56857..5489f68d 100644 --- a/pkg/model/runtime.go +++ b/pkg/model/runtime.go @@ -25,6 +25,7 @@ import ( const BackstageAppLabel = "rhdh.redhat.com/app" const ConfiguredNameAnnotation = "rhdh.redhat.com/configured-name" const DefaultMountPathAnnotation = "rhdh.redhat.com/mount-path" +const ContainersAnnotation = "rhdh.redhat.com/containers" // Backstage configuration scaffolding with empty BackstageObjects. // There are all possible objects for configuration diff --git a/pkg/model/testdata/multi-pvc-containers.yaml b/pkg/model/testdata/multi-pvc-containers.yaml new file mode 100644 index 00000000..99e7aaed --- /dev/null +++ b/pkg/model/testdata/multi-pvc-containers.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: myclaim1 +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 8Gi + storageClassName: slow +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: myclaim2 + annotations: + rhdh.redhat.com/mount-path: /mount/path/from/annotation + rhdh.redhat.com/containers: "backstage-backend,install-dynamic-plugins" +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 8Gi + storageClassName: slow +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: myclaim3 + annotations: + rhdh.redhat.com/mount-path: /mount/path/from/annotation2 + rhdh.redhat.com/containers: "*" +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 8Gi + storageClassName: slow \ No newline at end of file diff --git a/pkg/model/testdata/multicontainer-deployment.yaml b/pkg/model/testdata/multicontainer-deployment.yaml new file mode 100644 index 00000000..042c0c7a --- /dev/null +++ b/pkg/model/testdata/multicontainer-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: # placeholder for 'backstage-' +spec: + replicas: 1 + selector: + matchLabels: + rhdh.redhat.com/app: # placeholder for 'backstage-' + template: + metadata: + labels: + rhdh.redhat.com/app: # placeholder for 'backstage-' + spec: + initContainers: + - image: 'quay.io/rhdh/rhdh-hub-rhel9:next' + name: install-dynamic-plugins + - image: 'quay.io/rhdh/rhdh-hub-rhel9:next' + name: another-init-container + containers: + - name: backstage-backend + image: quay.io/rhdh/rhdh-hub-rhel9:next + - name: another-container + image: quay.io/rhdh/rhdh-hub-rhel9:next diff --git a/pkg/utils/pod-mutator.go b/pkg/utils/pod-mutator.go index 79300953..fddde456 100644 --- a/pkg/utils/pod-mutator.go +++ b/pkg/utils/pod-mutator.go @@ -29,7 +29,7 @@ type PodMutator struct { // mountPath - mount path, default one or as it specified in BackstageCR.spec.Application.AppConfig|ExtraFiles // fileName - file name which fits one of the object's key, otherwise error will be returned. // withSubPath - if true will be mounted file-by-file with subpath, otherwise will be mounted as directory to specified path -// data - key:value pairs from the object. should be specified if fileName specified +// dataKeys - keys for ConfigMap/Secret data func MountFilesFrom(podSpec *corev1.PodSpec, container *corev1.Container, kind ObjectKind, objectName, mountPath, fileName string, withSubPath bool, dataKeys []string) { volName := GenerateVolumeNameFromCmOrSecret(objectName) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 0f39787d..bddd0234 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -14,6 +14,8 @@ import ( "strconv" "strings" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime" @@ -212,3 +214,21 @@ func BoolEnvVar(envvar string, def bool) bool { } return def } + +func FilterContainers(allContainers []corev1.Container, filter string) []corev1.Container { + if filter == "*" { + return allContainers + } else if filter == "" { + return nil + } + + filtered := []corev1.Container{} + for _, c := range allContainers { + for _, cname := range strings.Split(filter, ",") { + if c.Name == strings.TrimSpace(cname) { + filtered = append(filtered, c) + } + } + } + return filtered +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index bfbd4195..65d48301 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -140,3 +140,22 @@ func TestBoolEnvVar(t *testing.T) { t.Setenv("MyVar", "anything") assert.True(t, BoolEnvVar("anything", true)) } + +func TestFilterContainers(t *testing.T) { + + containers := []corev1.Container{{Name: "c1"}, {Name: "c2"}, {Name: "c3"}} + + cs := FilterContainers(containers, "") + assert.Nil(t, cs) + + cs = FilterContainers(containers, "*") + assert.Equal(t, 3, len(cs)) + + cs = FilterContainers(containers, "c123") + assert.Equal(t, 0, len(cs)) + + cs = FilterContainers(containers, "c1,c2") + assert.Equal(t, 2, len(cs)) + assert.Equal(t, "c1", cs[0].Name) + +}