diff --git a/Makefile b/Makefile index d11c38f83..43c1f467b 100644 --- a/Makefile +++ b/Makefile @@ -170,9 +170,19 @@ generate: bootstrap code-gen crds mocks verify-generate: bootstrap generate git diff --exit-code +.PHONY: apply-ra +apply-ra: bootstrap + kubectl apply -f ./charts/radix-operator/templates/radixapplication.yaml + +.PHONY: apply-rd +apply-rd: bootstrap + kubectl apply -f ./charts/radix-operator/templates/radixdeployment.yaml + HAS_GOLANGCI_LINT := $(shell command -v golangci-lint;) HAS_MOCKGEN := $(shell command -v mockgen;) HAS_CONTROLLER_GEN := $(shell command -v controller-gen;) +HAS_YQ := $(shell command -v yq;) +HAS_KUBECTL := $(shell command -v kubectl;) .PHONY: bootstrap bootstrap: vendor @@ -185,3 +195,9 @@ endif ifndef HAS_CONTROLLER_GEN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.2 endif +ifndef HAS_YQ + go install github.com/mikefarah/yq/v4@latest +endif +ifndef HAS_KUBECTL + go install k8s.io/kubernetes/cmd/kubectl@latest +endif diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index f4dfb025a..79eb1440e 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator -version: 1.48.4 -appVersion: 1.68.4 +version: 1.49.0 +appVersion: 1.69.0 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/charts/radix-operator/templates/radixapplication.yaml b/charts/radix-operator/templates/radixapplication.yaml index 87b3c10ba..71ca3113a 100644 --- a/charts/radix-operator/templates/radixapplication.yaml +++ b/charts/radix-operator/templates/radixapplication.yaml @@ -1369,68 +1369,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: AzureFile settings for Azure File CSI - driver - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) - which will be set as owner of the mounted - volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in - the external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) - which will be set as owner of the mounted - volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - Immediate - WaitForFirstConsumer @@ -1438,7 +1390,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage - FUSE CSI driver + FUSE CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -1473,7 +1425,6 @@ spec: Storage FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -1481,6 +1432,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. + Applicable when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -1492,6 +1447,13 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is + mandatory when using a workload identity. + It is optional when using Access Key, if it + is not defined, it will be configured in a + secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -1534,6 +1496,14 @@ spec: minimum: 1 type: integer type: object + subscriptionId: + description: SubscriptionId of a storage account. + Applicable when using a workload identity. + type: string + tenantId: + description: TenantId of a storage account. + Applicable when using a workload identity. + type: string uid: description: UID defines the user ID (number) which will be set as owner of the mounted @@ -1544,11 +1514,17 @@ spec: Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported. Default false. This must be turned on when HNS enabled account is mounted. type: boolean + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Storage will be acquired + using Azure Workload Identity instead of using + a ClientID and Secret. + type: boolean required: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -1567,8 +1543,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string name: description: |- @@ -1584,33 +1560,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. enum: - - blob - azure-blob - - azure-file - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string required: - name @@ -2435,7 +2409,8 @@ spec: - name x-kubernetes-list-type: map public: - description: Deprecated, use publicPort instead. + description: 'Deprecated: For backwards compatibility Public + is still supported, new code should use PublicPort instead' type: boolean publicPort: description: |- @@ -2607,65 +2582,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: AzureFile settings for Azure File CSI driver - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - Immediate - WaitForFirstConsumer @@ -2673,7 +2603,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -2707,7 +2637,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -2715,6 +2644,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -2726,6 +2659,12 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -2766,6 +2705,14 @@ spec: minimum: 1 type: integer type: object + subscriptionId: + description: SubscriptionId of a storage account. + Applicable when using a workload identity. + type: string + tenantId: + description: TenantId of a storage account. Applicable + when using a workload identity. + type: string uid: description: UID defines the user ID (number) which will be set as owner of the mounted volume. @@ -2775,11 +2722,17 @@ spec: Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported. Default false. This must be turned on when HNS enabled account is mounted. type: boolean + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Storage will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -2798,8 +2751,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string name: description: |- @@ -2815,33 +2768,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. enum: - - blob - azure-blob - - azure-file - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string required: - name @@ -3528,68 +3479,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: AzureFile settings for Azure File CSI - driver - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) - which will be set as owner of the mounted - volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in - the external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) - which will be set as owner of the mounted - volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - Immediate - WaitForFirstConsumer @@ -3597,7 +3500,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage - FUSE CSI driver + FUSE CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -3632,7 +3535,6 @@ spec: Storage FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -3640,6 +3542,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. + Applicable when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -3651,6 +3557,13 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is + mandatory when using a workload identity. + It is optional when using Access Key, if it + is not defined, it will be configured in a + secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -3693,6 +3606,14 @@ spec: minimum: 1 type: integer type: object + subscriptionId: + description: SubscriptionId of a storage account. + Applicable when using a workload identity. + type: string + tenantId: + description: TenantId of a storage account. + Applicable when using a workload identity. + type: string uid: description: UID defines the user ID (number) which will be set as owner of the mounted @@ -3703,11 +3624,17 @@ spec: Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported. Default false. This must be turned on when HNS enabled account is mounted. type: boolean + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Storage will be acquired + using Azure Workload Identity instead of using + a ClientID and Secret. + type: boolean required: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -3726,8 +3653,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string name: description: |- @@ -3743,33 +3670,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. enum: - - blob - azure-blob - - azure-file - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string required: - name @@ -4147,65 +4072,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: AzureFile settings for Azure File CSI driver - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - Immediate - WaitForFirstConsumer @@ -4213,7 +4093,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -4247,7 +4127,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -4255,6 +4134,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -4266,6 +4149,12 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -4306,6 +4195,14 @@ spec: minimum: 1 type: integer type: object + subscriptionId: + description: SubscriptionId of a storage account. + Applicable when using a workload identity. + type: string + tenantId: + description: TenantId of a storage account. Applicable + when using a workload identity. + type: string uid: description: UID defines the user ID (number) which will be set as owner of the mounted volume. @@ -4315,11 +4212,17 @@ spec: Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported. Default false. This must be turned on when HNS enabled account is mounted. type: boolean + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Storage will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -4338,8 +4241,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string name: description: |- @@ -4355,33 +4258,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. enum: - - blob - azure-blob - - azure-file - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string required: - name diff --git a/charts/radix-operator/templates/radixdeployment.yaml b/charts/radix-operator/templates/radixdeployment.yaml index 6948d6723..66e88c05a 100644 --- a/charts/radix-operator/templates/radixdeployment.yaml +++ b/charts/radix-operator/templates/radixdeployment.yaml @@ -1171,65 +1171,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: AzureFile settings for Azure File CSI driver - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - Immediate - WaitForFirstConsumer @@ -1237,7 +1192,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -1271,7 +1226,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -1279,6 +1233,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -1290,6 +1248,12 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -1330,6 +1294,14 @@ spec: minimum: 1 type: integer type: object + subscriptionId: + description: SubscriptionId of a storage account. + Applicable when using a workload identity. + type: string + tenantId: + description: TenantId of a storage account. Applicable + when using a workload identity. + type: string uid: description: UID defines the user ID (number) which will be set as owner of the mounted volume. @@ -1339,11 +1311,17 @@ spec: Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported. Default false. This must be turned on when HNS enabled account is mounted. type: boolean + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Storage will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -1362,8 +1340,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string name: description: |- @@ -1379,33 +1357,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. enum: - - blob - azure-blob - - azure-file - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string required: - name @@ -1818,65 +1794,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: AzureFile settings for Azure File CSI driver - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. enum: - Immediate - WaitForFirstConsumer @@ -1884,7 +1815,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -1918,7 +1849,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -1926,6 +1856,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -1937,6 +1871,12 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -1977,6 +1917,14 @@ spec: minimum: 1 type: integer type: object + subscriptionId: + description: SubscriptionId of a storage account. + Applicable when using a workload identity. + type: string + tenantId: + description: TenantId of a storage account. Applicable + when using a workload identity. + type: string uid: description: UID defines the user ID (number) which will be set as owner of the mounted volume. @@ -1986,11 +1934,17 @@ spec: Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported. Default false. This must be turned on when HNS enabled account is mounted. type: boolean + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Storage will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -2009,8 +1963,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string name: description: |- @@ -2026,33 +1980,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. enum: - - blob - azure-blob - - azure-file - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. type: string required: - name diff --git a/go.mod b/go.mod index ebe0da8b0..be25e4890 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.5 require ( dario.cat/mergo v1.0.1 github.com/cert-manager/cert-manager v1.15.4 - github.com/equinor/radix-common v1.9.4 + github.com/equinor/radix-common v1.9.5 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index cdb22974e..f7fc93e93 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/equinor/radix-common v1.9.4 h1:ErSnB2tqlRwaQuQdaA0qzsReDtHDgubcvqRO098ncEw= -github.com/equinor/radix-common v1.9.4/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= +github.com/equinor/radix-common v1.9.5 h1:p1xldkYUoavwIMguoxxOyVkOXLPA6K8qMsgzeztQtQw= +github.com/equinor/radix-common v1.9.5/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= diff --git a/json-schema/radixapplication.json b/json-schema/radixapplication.json index e9c63893c..7dc66a163 100644 --- a/json-schema/radixapplication.json +++ b/json-schema/radixapplication.json @@ -1335,7 +1335,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -1344,53 +1344,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -1399,7 +1354,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -1432,7 +1387,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -1441,6 +1395,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -1452,6 +1410,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -1492,6 +1454,14 @@ }, "type": "object" }, + "subscriptionId": { + "description": "SubscriptionId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, + "tenantId": { + "description": "TenantId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "uid": { "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" @@ -1499,6 +1469,10 @@ "useAdls": { "description": "Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported.\nDefault false. This must be turned on when HNS enabled account is mounted.", "type": "boolean" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Storage will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -1507,7 +1481,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -1533,7 +1507,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -1548,29 +1522,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" } }, @@ -2431,7 +2403,7 @@ "x-kubernetes-list-type": "map" }, "public": { - "description": "Deprecated, use publicPort instead.", + "description": "Deprecated: For backwards compatibility Public is still supported, new code should use PublicPort instead", "type": "boolean" }, "publicPort": { @@ -2603,7 +2575,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -2612,53 +2584,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -2667,7 +2594,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -2700,7 +2627,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -2709,6 +2635,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -2720,6 +2650,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -2760,6 +2694,14 @@ }, "type": "object" }, + "subscriptionId": { + "description": "SubscriptionId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, + "tenantId": { + "description": "TenantId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "uid": { "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" @@ -2767,6 +2709,10 @@ "useAdls": { "description": "Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported.\nDefault false. This must be turned on when HNS enabled account is mounted.", "type": "boolean" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Storage will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -2775,7 +2721,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -2801,7 +2747,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -2816,29 +2762,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" } }, @@ -3558,7 +3502,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -3567,53 +3511,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -3622,7 +3521,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -3655,7 +3554,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -3664,6 +3562,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -3675,6 +3577,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -3715,6 +3621,14 @@ }, "type": "object" }, + "subscriptionId": { + "description": "SubscriptionId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, + "tenantId": { + "description": "TenantId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "uid": { "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" @@ -3722,6 +3636,10 @@ "useAdls": { "description": "Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported.\nDefault false. This must be turned on when HNS enabled account is mounted.", "type": "boolean" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Storage will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -3730,7 +3648,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -3756,7 +3674,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -3771,29 +3689,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" } }, @@ -4182,7 +4098,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -4191,53 +4107,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -4246,7 +4117,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -4279,7 +4150,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -4288,6 +4158,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -4299,6 +4173,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -4339,6 +4217,14 @@ }, "type": "object" }, + "subscriptionId": { + "description": "SubscriptionId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, + "tenantId": { + "description": "TenantId of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "uid": { "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" @@ -4346,6 +4232,10 @@ "useAdls": { "description": "Enables blobfuse to access Azure DataLake storage account. When set to false, blobfuse will access Azure Block Blob storage account, hierarchical file system is not supported.\nDefault false. This must be turned on when HNS enabled account is mounted.", "type": "boolean" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Storage will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -4354,7 +4244,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -4380,7 +4270,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -4395,29 +4285,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" } }, diff --git a/pipeline-runner/internal/jobs/build/acr.go b/pipeline-runner/internal/jobs/build/acr.go index fc7db5a06..fc716cee5 100644 --- a/pipeline-runner/internal/jobs/build/acr.go +++ b/pipeline-runner/internal/jobs/build/acr.go @@ -34,7 +34,7 @@ func NewACR() JobsBuilder { type acr struct{} -func (c *acr) BuildJobs(useBuildCache bool, pipelineArgs model.PipelineArguments, cloneURL, gitCommitHash, gitTags string, componentImages []pipeline.BuildComponentImage, buildSecrets []string) []batchv1.Job { +func (c *acr) BuildJobs(_ bool, pipelineArgs model.PipelineArguments, cloneURL, gitCommitHash, gitTags string, componentImages []pipeline.BuildComponentImage, buildSecrets []string) []batchv1.Job { props := &acrKubeJobProps{ pipelineArgs: pipelineArgs, componentImages: componentImages, diff --git a/pipeline-runner/internal/test/utils.go b/pipeline-runner/internal/test/utils.go index 085456e62..9ab61bd05 100644 --- a/pipeline-runner/internal/test/utils.go +++ b/pipeline-runner/internal/test/utils.go @@ -3,12 +3,12 @@ package test import ( "context" - "github.com/equinor/radix-operator/pipeline-runner/internal/hash" "github.com/equinor/radix-operator/pipeline-runner/model" pipelineDefaults "github.com/equinor/radix-operator/pipeline-runner/model/defaults" "github.com/equinor/radix-operator/pkg/apis/defaults" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pipeline-runner/steps/applyconfig/step.go b/pipeline-runner/steps/applyconfig/step.go index d1562e8d5..b2ba6c84e 100644 --- a/pipeline-runner/steps/applyconfig/step.go +++ b/pipeline-runner/steps/applyconfig/step.go @@ -21,6 +21,7 @@ import ( validate "github.com/equinor/radix-operator/pkg/apis/radixvalidators" operatorutils "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/equinor/radix-operator/pkg/apis/utils/git" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/rs/zerolog/log" corev1 "k8s.io/api/core/v1" kubeerrors "k8s.io/apimachinery/pkg/api/errors" @@ -401,7 +402,7 @@ func isRadixConfigNewOrModifiedSinceDeployment(ctx context.Context, rd *radixv1. if len(currentRdConfigHash) == 0 { return true, nil } - hashEqual, err := internal.CompareRadixApplicationHash(currentRdConfigHash, ra) + hashEqual, err := hash.CompareRadixApplicationHash(currentRdConfigHash, ra) if !hashEqual && err == nil { log.Ctx(ctx).Info().Msgf("RadixApplication updated since last deployment to environment %s", rd.Spec.Environment) } @@ -416,7 +417,7 @@ func isBuildSecretNewOrModifiedSinceDeployment(ctx context.Context, rd *radixv1. if len(targetHash) == 0 { return true, nil } - hashEqual, err := internal.CompareBuildSecretHash(targetHash, buildSecret) + hashEqual, err := hash.CompareBuildSecretHash(targetHash, buildSecret) if !hashEqual && err == nil { log.Ctx(ctx).Info().Msgf("Build secrets updated since last deployment to environment %s", rd.Spec.Environment) } diff --git a/pipeline-runner/steps/deploy/step.go b/pipeline-runner/steps/deploy/step.go index 4f30d9104..435249097 100644 --- a/pipeline-runner/steps/deploy/step.go +++ b/pipeline-runner/steps/deploy/step.go @@ -9,6 +9,7 @@ import ( "github.com/equinor/radix-operator/pipeline-runner/steps/internal" "github.com/equinor/radix-operator/pkg/apis/pipeline" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/rs/zerolog/log" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -76,12 +77,12 @@ func (cli *DeployStepImplementation) deployToEnv(ctx context.Context, appName, e commitID = pipelineInfo.PipelineArguments.CommitID // Commit ID specified by job arguments } - radixApplicationHash, err := internal.CreateRadixApplicationHash(pipelineInfo.RadixApplication) + radixApplicationHash, err := hash.CreateRadixApplicationHash(pipelineInfo.RadixApplication) if err != nil { return err } - buildSecretHash, err := internal.CreateBuildSecretHash(pipelineInfo.BuildSecret) + buildSecretHash, err := hash.CreateBuildSecretHash(pipelineInfo.BuildSecret) if err != nil { return err } diff --git a/pipeline-runner/steps/deployconfig/step_test.go b/pipeline-runner/steps/deployconfig/step_test.go index 8f0adfa87..dfa7d38e4 100644 --- a/pipeline-runner/steps/deployconfig/step_test.go +++ b/pipeline-runner/steps/deployconfig/step_test.go @@ -11,10 +11,10 @@ import ( "github.com/equinor/radix-operator/pipeline-runner/internal/watcher" "github.com/equinor/radix-operator/pipeline-runner/model" "github.com/equinor/radix-operator/pipeline-runner/steps/deployconfig" - "github.com/equinor/radix-operator/pipeline-runner/steps/internal" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" radixfake "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" "github.com/golang/mock/gomock" kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" @@ -683,8 +683,8 @@ func (s *deployConfigTestSuite) createRadixDeployments(deploymentBuildersProps [ } func (s *deployConfigTestSuite) buildRadixDeployments(deploymentBuildersProps []radixDeploymentBuildersProps, ra *radixv1.RadixApplication, sourceEnvMap map[string]radixv1.RadixDeployment) []radixv1.RadixDeployment { - radixConfigHash, _ := internal.CreateRadixApplicationHash(ra) - buildSecretHash, _ := internal.CreateBuildSecretHash(nil) + radixConfigHash, _ := hash.CreateRadixApplicationHash(ra) + buildSecretHash, _ := hash.CreateBuildSecretHash(nil) var rdList []radixv1.RadixDeployment for _, rdProps := range deploymentBuildersProps { if sourceRd, ok := sourceEnvMap[rdProps.envName]; ok { diff --git a/pipeline-runner/steps/internal/deployment_test.go b/pipeline-runner/steps/internal/deployment_test.go index cedc82162..bf07767c1 100644 --- a/pipeline-runner/steps/internal/deployment_test.go +++ b/pipeline-runner/steps/internal/deployment_test.go @@ -50,7 +50,7 @@ func TestConstructForTargetEnvironment_PicksTheCorrectEnvironmentConfig(t *testi }). WithVolumeMounts([]radixv1.RadixVolumeMount{ { - Type: radixv1.MountTypeBlob, + Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Container: "some-container", Path: "some-path", }, diff --git a/pipeline-runner/steps/internal/hash.go b/pipeline-runner/steps/internal/hash.go deleted file mode 100644 index 071d0b05e..000000000 --- a/pipeline-runner/steps/internal/hash.go +++ /dev/null @@ -1,43 +0,0 @@ -package internal - -import ( - "github.com/equinor/radix-operator/pipeline-runner/internal/hash" - radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - corev1 "k8s.io/api/core/v1" -) - -// Constants used to generate hash for RadixApplication and BuildSecret if they are nil. Do not change. -const ( - magicValueForNilRadixApplication = "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7" - magicValueForNilBuildSecretData = "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj" -) - -func CreateRadixApplicationHash(ra *radixv1.RadixApplication) (string, error) { - return hash.ToHashString(hash.SHA256, getRadixApplicationOrMagicValue(ra)) -} - -func CompareRadixApplicationHash(targetHash string, ra *radixv1.RadixApplication) (bool, error) { - return hash.CompareWithHashString(getRadixApplicationOrMagicValue(ra), targetHash) -} - -func CreateBuildSecretHash(secret *corev1.Secret) (string, error) { - return hash.ToHashString(hash.SHA256, getBuildSecretOrMagicValue(secret)) -} - -func CompareBuildSecretHash(targetHash string, secret *corev1.Secret) (bool, error) { - return hash.CompareWithHashString(getBuildSecretOrMagicValue(secret), targetHash) -} - -func getRadixApplicationOrMagicValue(ra *radixv1.RadixApplication) any { - if ra == nil { - return magicValueForNilRadixApplication - } - return ra.Spec -} - -func getBuildSecretOrMagicValue(secret *corev1.Secret) any { - if secret == nil || len(secret.Data) == 0 { - return magicValueForNilBuildSecretData - } - return secret.Data -} diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index 04bf41d0c..b02b8c4b6 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -15,6 +15,7 @@ import ( operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/equinor/radix-operator/pkg/apis/utils/annotations" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" + "github.com/equinor/radix-operator/pkg/apis/volumemount" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -27,7 +28,7 @@ const ( jobPayloadVolumeName = "job-payload" ) -func (s *syncer) reconcileKubeJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, rd *radixv1.RadixDeployment, jobComponent *radixv1.RadixDeployJobComponent, existingJobs []*batchv1.Job) error { +func (s *syncer) reconcileKubeJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, rd *radixv1.RadixDeployment, jobComponent *radixv1.RadixDeployJobComponent, existingJobs []*batchv1.Job, volumes []corev1.Volume) error { batchJobKubeJobs := slice.FindAll(existingJobs, isKubeJobForBatchJob(batchJob)) if isBatchJobStopRequested(batchJob) { @@ -51,7 +52,7 @@ func (s *syncer) reconcileKubeJob(ctx context.Context, batchJob *radixv1.RadixBa return err } - job, err := s.buildJob(ctx, batchJob, jobComponent, rd) + job, err := s.buildJob(ctx, batchJob, jobComponent, rd, volumes) if err != nil { return err } @@ -92,7 +93,7 @@ func (s *syncer) deleteJobs(ctx context.Context, jobsToDelete []*batchv1.Job) er return nil } -func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment) (*batchv1.Job, error) { +func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment, volumes []corev1.Volume) (*batchv1.Job, error) { jobLabels := s.batchJobIdentifierLabel(batchJob.Name, rd.Spec.AppName) podLabels := radixlabels.Merge( jobLabels, @@ -100,11 +101,6 @@ func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, ) podAnnotations := annotations.ForClusterAutoscalerSafeToEvict(false) - volumes, err := s.getVolumes(ctx, rd.GetNamespace(), rd.Spec.Environment, batchJob, jobComponent, rd.Name) - if err != nil { - return nil, err - } - kubeJobName := getKubeJobName(s.radixBatch.GetName(), batchJob.Name) containers, err := s.getContainers(ctx, rd, jobComponent, batchJob, kubeJobName) if err != nil { @@ -135,6 +131,7 @@ func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, } serviceAccountSpec := deployment.NewServiceAccountSpec(rd, jobComponent) + volumes = s.appendPayloadSecretVolumes(batchJob, jobComponent, volumes) job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -167,7 +164,6 @@ func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, TTLSecondsAfterFinished: pointers.Ptr(int32(86400)), // delete completed job after 24 hours }, } - return job, nil } @@ -179,27 +175,21 @@ func (s *syncer) getJobPodImagePullSecrets(rd *radixv1.RadixDeployment) []corev1 return imagePullSecrets } -func (s *syncer) getVolumes(ctx context.Context, namespace, environment string, batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.Volume, error) { - volumes, err := deployment.GetVolumes(ctx, s.kubeClient, s.kubeUtil, namespace, environment, radixJobComponent, radixDeploymentName) - if err != nil { - return nil, err - } - - if radixJobComponent.Payload != nil && batchJob.PayloadSecretRef != nil { - volumes = append(volumes, corev1.Volume{ - Name: jobPayloadVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: batchJob.PayloadSecretRef.Name, - Items: []corev1.KeyToPath{ - {Key: batchJob.PayloadSecretRef.Key, Path: "payload"}, - }, +func (s *syncer) appendPayloadSecretVolumes(batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, volumes []corev1.Volume) []corev1.Volume { + if radixJobComponent.Payload == nil || batchJob.PayloadSecretRef == nil { + return volumes + } + return append(volumes, corev1.Volume{ + Name: jobPayloadVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: batchJob.PayloadSecretRef.Name, + Items: []corev1.KeyToPath{ + {Key: batchJob.PayloadSecretRef.Key, Path: "payload"}, }, }, - }) - } - - return volumes, nil + }, + }) } func (s *syncer) getContainers(ctx context.Context, rd *radixv1.RadixDeployment, jobComponent *radixv1.RadixDeployJobComponent, batchJob *radixv1.RadixBatchJob, kubeJobName string) ([]corev1.Container, error) { @@ -276,7 +266,7 @@ func getContainerPorts(radixJobComponent *radixv1.RadixDeployJobComponent) []cor } func (s *syncer) getContainerVolumeMounts(batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.VolumeMount, error) { - volumeMounts, err := deployment.GetRadixDeployComponentVolumeMounts(radixJobComponent, radixDeploymentName) + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixJobComponent, radixDeploymentName) if err != nil { return nil, err } diff --git a/pkg/apis/batch/syncer.go b/pkg/apis/batch/syncer.go index 5febda8eb..b0ad97382 100644 --- a/pkg/apis/batch/syncer.go +++ b/pkg/apis/batch/syncer.go @@ -10,6 +10,7 @@ import ( "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" + "github.com/equinor/radix-operator/pkg/apis/volumemount" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" "k8s.io/apimachinery/pkg/api/errors" @@ -86,22 +87,35 @@ func (s *syncer) reconcile(ctx context.Context) error { return err } - existingJobs, err := s.kubeUtil.ListJobsWithSelector(ctx, s.radixBatch.GetNamespace(), s.batchIdentifierLabel().String()) + namespace := s.radixBatch.GetNamespace() + existingJobs, err := s.kubeUtil.ListJobsWithSelector(ctx, namespace, s.batchIdentifierLabel().String()) if err != nil { return err } - existingServices, err := s.kubeUtil.ListServicesWithSelector(ctx, s.radixBatch.GetNamespace(), s.batchIdentifierLabel().String()) + existingServices, err := s.kubeUtil.ListServicesWithSelector(ctx, namespace, s.batchIdentifierLabel().String()) if err != nil { return err } + existingVolumes, err := volumemount.GetExistingJobAuxComponentVolumes(ctx, s.kubeUtil, namespace, jobComponent.GetName()) + if err != nil { + return err + } + desiredVolumes, err := volumemount.GetVolumes(ctx, s.kubeUtil, namespace, jobComponent, rd.Name, existingVolumes) + if err != nil { + return err + } + actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(ctx, s.kubeUtil.KubeClient(), rd, namespace, jobComponent, desiredVolumes) + if err != nil { + return fmt.Errorf("failed to create or update csi azure volume resources: %w", err) + } for i, batchJob := range s.radixBatch.Spec.Jobs { if err := s.reconcileService(ctx, &batchJob, rd, jobComponent, existingServices); err != nil { return fmt.Errorf("batchjob %s: failed to reconcile service: %w", batchJob.Name, err) } - if err := s.reconcileKubeJob(ctx, &batchJob, rd, jobComponent, existingJobs); err != nil { + if err := s.reconcileKubeJob(ctx, &batchJob, rd, jobComponent, existingJobs, actualVolumes); err != nil { return fmt.Errorf("batchjob %s: failed to reconcile kubejob: %w", batchJob.Name, err) } diff --git a/pkg/apis/batch/syncer_test.go b/pkg/apis/batch/syncer_test.go index f20e33fb1..7cf3e92f2 100644 --- a/pkg/apis/batch/syncer_test.go +++ b/pkg/apis/batch/syncer_test.go @@ -1069,8 +1069,6 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts() { Name: componentName, VolumeMounts: []radixv1.RadixVolumeMount{ {Name: "azureblob2name", Path: "/azureblob2path", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Protocol: radixv1.BlobFuse2ProtocolFuse2, Container: "azureblob2container"}}, - {Name: "azurenfsname", Path: "/azurenfspath", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Protocol: radixv1.BlobFuse2ProtocolNfs, Container: "azurenfscontainer"}}, - {Name: "azurefilename", Path: "/azurefilepath", AzureFile: &radixv1.RadixAzureFileVolumeMount{Share: "azurefilecontainer"}}, }, }, }, @@ -1086,14 +1084,10 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts() { jobs, _ := s.kubeClient.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, jobName) })[0] - s.Require().Len(job.Spec.Template.Spec.Volumes, 3) - s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 3) + s.Require().Len(job.Spec.Template.Spec.Volumes, 1) + s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 1) s.Equal(job.Spec.Template.Spec.Volumes[0].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) - s.Equal(job.Spec.Template.Spec.Volumes[1].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name) - s.Equal(job.Spec.Template.Spec.Volumes[2].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[2].Name) s.Equal("/azureblob2path", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) - s.Equal("/azurenfspath", job.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath) - s.Equal("/azurefilepath", job.Spec.Template.Spec.Containers[0].VolumeMounts[2].MountPath) } func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { @@ -1117,9 +1111,7 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { { Name: componentName, VolumeMounts: []radixv1.RadixVolumeMount{ - {Type: "blob", Name: "blobname", Container: "blobcontainer", Path: "/blobpath"}, {Type: "azure-blob", Name: "azureblobname", Storage: "azureblobcontainer", Path: "/azureblobpath"}, - {Type: "azure-file", Name: "azurefilename", Storage: "azurefilecontainer", Path: "/azurefilepath"}, }, }, }, @@ -1135,14 +1127,10 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { jobs, _ := s.kubeClient.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, jobName) })[0] - s.Require().Len(job.Spec.Template.Spec.Volumes, 3) - s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 3) + s.Require().Len(job.Spec.Template.Spec.Volumes, 1) + s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 1) s.Equal(job.Spec.Template.Spec.Volumes[0].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) - s.Equal(job.Spec.Template.Spec.Volumes[1].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name) - s.Equal(job.Spec.Template.Spec.Volumes[2].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[2].Name) - s.Equal("/blobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) - s.Equal("/azureblobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath) - s.Equal("/azurefilepath", job.Spec.Template.Spec.Containers[0].VolumeMounts[2].MountPath) + s.Equal("/azureblobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) } func (s *syncerTestSuite) Test_JobWithAzureSecretRefs() { diff --git a/pkg/apis/defaults/jobscheduler.go b/pkg/apis/defaults/jobscheduler.go index b9cd09f6f..dd4fb7da1 100644 --- a/pkg/apis/defaults/jobscheduler.go +++ b/pkg/apis/defaults/jobscheduler.go @@ -1,4 +1,11 @@ package defaults +import "fmt" + const RadixJobSchedulerPortName = "scheduler-port" const RadixJobTimeLimitSeconds = 43200 // 12 hours + +// GetJobAuxKubeDeployName Get the aux kube deployment name for a job component +func GetJobAuxKubeDeployName(jobName string) string { + return fmt.Sprintf("%s-aux", jobName) +} diff --git a/pkg/apis/defaults/k8s/k8s.go b/pkg/apis/defaults/k8s/k8s.go index 72a3bab71..7c69f9822 100644 --- a/pkg/apis/defaults/k8s/k8s.go +++ b/pkg/apis/defaults/k8s/k8s.go @@ -1,9 +1,10 @@ package k8s const ( - KindClusterRole = "ClusterRole" - KindClusterRoleBinding = "ClusterRoleBinding" - KindRole = "Role" - KindRoleBinding = "RoleBinding" - KindIngress = "Ingress" + KindClusterRole = "ClusterRole" + KindClusterRoleBinding = "ClusterRoleBinding" + KindRole = "Role" + KindRoleBinding = "RoleBinding" + KindIngress = "Ingress" + KindPersistentVolumeClaim = "PersistentVolumeClaim" ) diff --git a/pkg/apis/defaults/pod.go b/pkg/apis/defaults/pod.go index 93a369c71..cddb421d2 100644 --- a/pkg/apis/defaults/pod.go +++ b/pkg/apis/defaults/pod.go @@ -1,6 +1,10 @@ package defaults const ( + // DefaultReplicas Hold the default replicas for the deployment if nothing is stated in the radix config + DefaultReplicas int32 = 1 + // DefaultNodeSelectorArchitecture Hold the default architecture for the deployment if nothing is stated in the radix config DefaultNodeSelectorArchitecture = "amd64" - DefaultNodeSelectorOS = "linux" + // DefaultNodeSelectorOS Hold the default OS for the deployment if nothing is stated in the radix config + DefaultNodeSelectorOS = "linux" ) diff --git a/pkg/apis/defaults/secrets.go b/pkg/apis/defaults/secrets.go index 47255b7e2..2240fc6e6 100644 --- a/pkg/apis/defaults/secrets.go +++ b/pkg/apis/defaults/secrets.go @@ -70,6 +70,12 @@ const ( // TLSSecretName Secret name for the Radix wildcard TLS cert TLSSecretName = "radix-wildcard-tls-cert" + + // SecretDefaultData default secret content + SecretDefaultData = "xx" + + // SecretUsedBySecretStoreDriverLabel Label used to mark secrets used by secret store driver + SecretUsedBySecretStoreDriverLabel = "secrets-store.csi.k8s.io/used" ) // GetBlobFuseCredsSecretName Helper method diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go new file mode 100644 index 000000000..56f807db4 --- /dev/null +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -0,0 +1,221 @@ +package deployment + +import ( + "context" + "strings" + "testing" + + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/volumemount" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { + const ( + appName = "app" + namespace = "some-namespace" + environment = "some-env" + componentName1 = "component1" + componentNameLong = "max-long-component-name-0123456789012345678901234" + ) + type expectedVolumeProps struct { + expectedVolumeNamePrefix string + expectedVolumeMountPath string + expectedNodePublishSecretRefName string + expectedVolumeAttributePrefixes map[string]string + } + scenarios := []struct { + name string + deployComponentBuilders []utils.DeployCommonComponentBuilder + componentName string + azureKeyVaults []v1.RadixAzureKeyVault + expectedVolumeProps []expectedVolumeProps + radixVolumeMounts []v1.RadixVolumeMount + }{ + { + name: "No Azure Key volumes as no RadixAzureKeyVault-s", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{}, + expectedVolumeProps: []expectedVolumeProps{}, + }, + { + name: "No Azure Key volumes as no secret names in secret object", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, + }, + { + name: "One Azure Key volume for one secret objects secret name", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{{ + Name: "kv1", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }}, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv1-", + }, + }, + }, + }, + { + name: "Multiple Azure Key volumes for each RadixAzureKeyVault", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{ + { + Name: "kv1", + Path: utils.StringPtr("/mnt/customPath"), + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }, + { + Name: "kv2", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, + }, + }, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + expectedVolumeMountPath: "/mnt/customPath", + expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv1-", + }, + }, + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", + expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv2-", + }, + }, + }, + }, + { + name: "Volume name should be trimmed when exceeding 63 chars", + componentName: componentNameLong, + azureKeyVaults: []v1.RadixAzureKeyVault{{ + Name: "kv1", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }}, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "max-long-component-name-0123456789012345678901234-az-keyv", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + expectedNodePublishSecretRefName: "max-long-component-name-0123456789012345678901234-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "max-long-component-name-0123456789012345678901234-az-keyvault-kv1-", + }, + }, + }, + }, + } + t.Run("CSI Azure Key vault volumes", func(t *testing.T) { + t.Parallel() + for _, scenario := range scenarios { + t.Logf("Test case %s", scenario.name) + rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + var builders []utils.DeployComponentBuilder + builders = append(builders, utils.NewDeployComponentBuilder(). + WithName(scenario.componentName). + WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) + return builders + }) + deployment := getDeployment(t, rdBuilder) + radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) + for _, azureKeyVault := range scenario.azureKeyVaults { + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + if err != nil { + t.Log(err.Error()) + } else { + t.Logf("created secret provider class %s", spc.Name) + } + } + volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, deployment.radixDeployment.GetName(), nil) + require.NoError(t, err, "failed to get volumes") + assert.Len(t, volumes, len(scenario.expectedVolumeProps)) + if len(scenario.expectedVolumeProps) == 0 { + continue + } + + for i := 0; i < len(volumes); i++ { + volume := volumes[i] + assert.Less(t, len(volume.Name), 64, "volume name is too long") + assert.NotNil(t, volume.CSI, "CSI should ne not nil") + assert.NotEmpty(t, volume.CSI.VolumeAttributes[volumemount.CsiVolumeSourceVolumeAttributeSecretProviderClass], "VolumeAttributes should not be empty") + assert.NotNil(t, volume.CSI.NodePublishSecretRef, "NodePublishSecretRef should not be nil") + assert.Equal(t, volumemount.CsiVolumeSourceDriverSecretStore, volume.CSI.Driver, "Volume driver should be %s, but it is %s", volumemount.CsiVolumeSourceDriverSecretStore, volume.CSI.Driver) + + volumeProp := scenario.expectedVolumeProps[i] + for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { + spcValue, exists := volume.CSI.VolumeAttributes[attrKey] + assert.True(t, exists) + assert.True(t, strings.HasPrefix(spcValue, attrValue), "SecretProviderClass name should have a prefix %s, but it has name %s", attrValue, spcValue) + } + assert.True(t, strings.HasPrefix(volume.Name, volumeProp.expectedVolumeNamePrefix), "Volume name should have prefix %s, but it is %s", volumeProp.expectedVolumeNamePrefix, volume.Name) + assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name, "NodePublishSecretRef name should be %s, but it is %s", volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) + } + } + }) + + t.Run("CSI Azure Key vault volume mounts", func(t *testing.T) { + //t.Parallel() + for _, scenario := range scenarios { + rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + var builders []utils.DeployComponentBuilder + builders = append(builders, utils.NewDeployComponentBuilder(). + WithName(scenario.componentName). + WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) + return builders + }) + deployment := getDeployment(t, rdBuilder) + radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) + for _, azureKeyVault := range scenario.azureKeyVaults { + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + if err != nil { + t.Log(err.Error()) + } else { + t.Logf("created secret provider class %s", spc.Name) + } + } + + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixDeployComponent, deployment.radixDeployment.GetName()) + require.Nil(t, err) + + assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) + if len(scenario.expectedVolumeProps) == 0 { + continue + } + + for i := 0; i < len(volumeMounts); i++ { + volumeMount := volumeMounts[i] + volumeProp := scenario.expectedVolumeProps[i] + assert.Less(t, len(volumeMount.Name), 64, "volumemount name is too long") + assert.True(t, strings.HasPrefix(volumeMount.Name, volumeProp.expectedVolumeNamePrefix), + "VolumeMount name should have prefix %s, but it is %s", volumeProp.expectedVolumeNamePrefix, volumeMount.Name) + assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath, "VolumeMount mount path should be %s, but it is %s", volumeProp.expectedVolumeMountPath, volumeMount.MountPath) + assert.True(t, volumeMount.ReadOnly, "VolumeMount should be read only") + } + } + }) +} + +func getDeployment(t *testing.T, deploymentBuilder utils.DeploymentBuilder) *Deployment { + tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, _, certClient := SetupTest(t) + rd, err := ApplyDeploymentWithSync(tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, certClient, + deploymentBuilder) + require.NoError(t, err) + return &Deployment{radixclient: radixClient, kubeutil: kubeUtil, radixDeployment: rd, config: &testConfig} +} + +func getRdBuilderWithComponentBuilders(appName string, environment string, componentBuilders func() []utils.DeployComponentBuilder) utils.DeploymentBuilder { + return utils.ARadixDeployment(). + WithAppName(appName). + WithEnvironment(environment). + WithComponents(componentBuilders()...) +} diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index e571532f7..b7a842170 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/equinor/radix-operator/pkg/apis/utils/resources" "sort" "strings" "time" @@ -15,10 +14,12 @@ import ( "github.com/equinor/radix-operator/pkg/apis/config" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/ingress" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" "github.com/equinor/radix-operator/pkg/apis/metrics" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/volumemount" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/rs/zerolog/log" @@ -31,9 +32,7 @@ import ( ) const ( - // DefaultReplicas Hold the default replicas for the deployment if nothing is stated in the radix config - DefaultReplicas int32 = 1 - prometheusInstanceLabel = "LABEL_PROMETHEUS_INSTANCE" + prometheusInstanceLabel = "LABEL_PROMETHEUS_INSTANCE" ) // DeploymentSyncer defines interface for syncing a RadixDeployment @@ -234,7 +233,7 @@ func (deploy *Deployment) syncDeployment(ctx context.Context) error { } for _, jobComponent := range deploy.radixDeployment.Spec.Jobs { ctx := log.Ctx(ctx).With().Str("jobComponent", jobComponent.Name).Logger().WithContext(ctx) - jobSchedulerComponent := newJobSchedulerComponent(&jobComponent, deploy.radixDeployment) + jobSchedulerComponent := internal.NewJobSchedulerComponent(&jobComponent, deploy.radixDeployment) if err := deploy.syncDeploymentForRadixComponent(ctx, jobSchedulerComponent); err != nil { errs = append(errs, err) } @@ -476,14 +475,6 @@ func getLabelSelectorForComponent(component v1.RadixCommonDeployComponent) strin return fmt.Sprintf("%s=%s", kube.RadixComponentLabel, component.GetName()) } -func getLabelSelectorForBlobVolumeMountSecret(component v1.RadixCommonDeployComponent) string { - return fmt.Sprintf("%s=%s, %s=%s", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlob)) -} - -func getLabelSelectorForCsiAzureVolumeMountSecret(component v1.RadixCommonDeployComponent) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure), string(v1.MountTypeBlobFuse2NfsCsiAzure), string(v1.MountTypeAzureFileCsiAzure)) -} - func (deploy *Deployment) maintainHistoryLimit(ctx context.Context, deploymentHistoryLimit int) { if deploymentHistoryLimit <= 0 { return @@ -541,7 +532,7 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c return fmt.Errorf("failed to create service account: %w", err) } - err = deploy.reconcileDeployment(ctx, component) + err = deploy.reconcileDeployComponent(ctx, component) if err != nil { return fmt.Errorf("failed to create deployment: %w", err) } @@ -595,11 +586,15 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c } } + if err = volumemount.GarbageCollectCsiAzureVolumeResourcesForDeployComponent(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace()); err != nil { + return fmt.Errorf("failed to garbage collect persistent volumes or persistent volume claims: %w", err) + } + return nil } -func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, deployComponent, desiredDeployment) +func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, jobKubeDeploymentName string, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (*appsv1.Deployment, *appsv1.Deployment, error) { + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, namespace, jobKubeDeploymentName) if err != nil { return nil, nil, err } @@ -608,12 +603,9 @@ func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, de desiredJobAuxDeployment.Spec.Template.Labels = deploy.getJobAuxDeploymentPodLabels(deployComponent) desiredJobAuxDeployment.Spec.Template.Spec.ServiceAccountName = (&radixComponentServiceAccountSpec{component: deployComponent}).ServiceAccountName() desiredJobAuxDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForJobAPIAuxComponent() - // Copy volumes and volume mounts from desired deployment to job aux deployment - desiredJobAuxDeployment.Spec.Template.Spec.Volumes = desiredDeployment.Spec.Template.Spec.Volumes - desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts - // Remove volumes and volume mounts from job scheduler deployment - desiredDeployment.Spec.Template.Spec.Volumes = nil - desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = nil + // Move volumes and volume mounts from desired deployment to job aux deployment + desiredJobAuxDeployment.Spec.Template.Spec.Volumes = volumes + desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts syncRadixRestartEnvironmentVariable(deployComponent, desiredJobAuxDeployment) return currentJobAuxDeployment, desiredJobAuxDeployment, nil @@ -632,19 +624,14 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom } } -func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, desiredDeployment.Namespace, getJobAuxObjectName(desiredDeployment.Name)) +func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, namespace, jobKubeDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { + jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobKubeDeploymentName) + currentJobAuxDeployment, err := deploy.kubeutil.KubeClient().AppsV1().Deployments(namespace).Get(ctx, jobAuxKubeDeploymentName, metav1.GetOptions{}) if err != nil { if k8sErrors.IsNotFound(err) { - return nil, deploy.createJobAuxDeployment(deployComponent), nil + return nil, deploy.createJobAuxDeployment(jobKubeDeploymentName, jobAuxKubeDeploymentName), nil } return nil, nil, err } - desiredJobAuxDeployment := currentJobAuxDeployment.DeepCopy() - desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)) - return currentJobAuxDeployment, desiredJobAuxDeployment, nil -} - -func getJobAuxObjectName(jobName string) string { - return fmt.Sprintf("%s-aux", jobName) + return currentJobAuxDeployment, currentJobAuxDeployment.DeepCopy(), nil } diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index ac4b301a5..b2e1d28cd 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -1,4 +1,6 @@ // file deepcode ignore HardcodedPassword/test: unit tests +// +//nolint:staticcheck package deployment import ( @@ -130,7 +132,6 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { outdatedSecret := "outdatedSecret" remainingSecret := "remainingSecret" addingSecret := "addingSecret" - blobVolumeName := "blob_volume_1" blobCsiAzureVolumeName := "blobCsiAzure_volume_1" if componentsExist { @@ -211,12 +212,6 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { WithPort("http", 3000). WithPublicPort("http"). WithVolumeMounts( - radixv1.RadixVolumeMount{ - Type: radixv1.MountTypeBlob, - Name: blobVolumeName, - Container: "some-container", - Path: "some-path", - }, radixv1.RadixVolumeMount{ Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: blobCsiAzureVolumeName, @@ -310,7 +305,7 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { assert.True(t, deploymentByNameExists(componentNameRadixQuote, deployments), "radixquote deployment not there") spec := getDeploymentByName(componentNameRadixQuote, deployments).Spec - assert.Equal(t, DefaultReplicas, *spec.Replicas, "number of replicas was unexpected") + assert.Equal(t, defaults.DefaultReplicas, *spec.Replicas, "number of replicas was unexpected") assert.True(t, envVariableByNameExistOnDeployment(defaults.ContainerRegistryEnvironmentVariable, componentNameRadixQuote, deployments)) assert.True(t, envVariableByNameExistOnDeployment(defaults.RadixDNSZoneEnvironmentVariable, componentNameRadixQuote, deployments)) assert.True(t, envVariableByNameExistOnDeployment(defaults.ClusternameEnvironmentVariable, componentNameRadixQuote, deployments)) @@ -330,8 +325,8 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { assert.True(t, envVariableByNameExistOnDeployment(addingSecret, componentNameRadixQuote, deployments)) } - volumesExist := len(spec.Template.Spec.Volumes) > 1 - volumeMountsExist := len(spec.Template.Spec.Containers[0].VolumeMounts) > 1 + volumesExist := len(spec.Template.Spec.Volumes) > 0 + volumeMountsExist := len(spec.Template.Spec.Containers[0].VolumeMounts) > 0 if !componentsExist { assert.True(t, volumesExist, "expected existing volumes") assert.True(t, volumeMountsExist, "expected existing volume mounts") @@ -394,20 +389,18 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { secrets, _ := kubeclient.CoreV1().Secrets(envNamespace).List(context.Background(), metav1.ListOptions{}) if !componentsExist { - assert.Equal(t, 3, len(secrets.Items), "Number of secrets was not according to spec") + assert.Equal(t, 2, len(secrets.Items), "Number of secrets was not according to spec") } else { assert.Equal(t, 1, len(secrets.Items), "Number of secrets was not according to spec") } componentSecretName := utils.GetComponentSecretName(componentNameRadixQuote) assert.True(t, secretByNameExists(componentSecretName, secrets), "Component secret is not as expected") - blobFuseSecretExists := secretByNameExists(defaults.GetBlobFuseCredsSecretName(componentNameRadixQuote, blobVolumeName), secrets) blobCsiAzureFuseSecretExists := secretByNameExists(defaults.GetCsiAzureVolumeMountCredsSecretName(componentNameRadixQuote, blobCsiAzureVolumeName), secrets) if !componentsExist { - assert.True(t, blobFuseSecretExists, "expected Blobfuse volume mount secret") assert.True(t, blobCsiAzureFuseSecretExists, "expected blob CSI Azure volume mount secret") } else { - assert.False(t, blobFuseSecretExists, "unexpected volume mount secrets") + assert.False(t, blobCsiAzureFuseSecretExists, "unexpected volume mount secrets") } }) @@ -528,7 +521,6 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { outdatedSecret := "outdatedSecret" remainingSecret := "remainingSecret" addingSecret := "addingSecret" - blobVolumeName := "blob_volume_1" blobCsiAzureVolumeName := "blobCsiAzure_volume_1" payloadPath := "payloadpath" if jobsExist { @@ -587,12 +579,6 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { "cpu": "501m", }). WithVolumeMounts( - radixv1.RadixVolumeMount{ - Type: radixv1.MountTypeBlob, - Name: blobVolumeName, - Container: "some-container", - Path: "some-path", - }, radixv1.RadixVolumeMount{ Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: blobCsiAzureVolumeName, @@ -723,7 +709,7 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { secrets, _ := kubeclient.CoreV1().Secrets(envNamespace).List(context.Background(), metav1.ListOptions{}) if !jobsExist { - assert.Equal(t, 3, len(secrets.Items), "Number of secrets was not according to spec") + assert.Equal(t, 2, len(secrets.Items), "Number of secrets was not according to spec") } else { assert.Equal(t, 1, len(secrets.Items), "Number of secrets was not according to spec") } @@ -731,13 +717,11 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { jobSecretName := utils.GetComponentSecretName(jobName) assert.True(t, secretByNameExists(jobSecretName, secrets), "Job secret is not as expected") - blobFuseSecretExists := secretByNameExists(defaults.GetBlobFuseCredsSecretName(jobName, blobVolumeName), secrets) blobCsiAzureFuseSecretExists := secretByNameExists(defaults.GetCsiAzureVolumeMountCredsSecretName(jobName, blobCsiAzureVolumeName), secrets) if !jobsExist { - assert.True(t, blobFuseSecretExists, "expected Blobfuse volume mount secret") assert.True(t, blobCsiAzureFuseSecretExists, "expected blob CSI Azure volume mount secret") } else { - assert.False(t, blobFuseSecretExists, "unexpected volume mount secrets") + assert.False(t, blobCsiAzureFuseSecretExists, "unexpected volume mount secrets") } }) @@ -2335,7 +2319,6 @@ func TestObjectSynced_PublicToNonPublic_HandlesChange(t *testing.T) { assert.Equal(t, 0, len(ingresses.Items), "No component should be public") } -//nolint:staticcheck func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { tu, client, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t) defer TeardownTest() @@ -2348,7 +2331,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -2368,7 +2350,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -2392,7 +2373,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -2409,7 +2389,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -3858,9 +3837,7 @@ func Test_ComponentSynced_VolumeAndMounts(t *testing.T) { utils.NewDeployComponentBuilder(). WithName(compName). WithVolumeMounts( - radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlob, Name: "blob", Container: "blobcontainer", Path: "blobpath"}, radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blobcsi", Storage: "blobcsistorage", Path: "blobcsipath"}, - radixv1.RadixVolumeMount{Type: radixv1.MountTypeAzureFileCsiAzure, Name: "filecsi", Storage: "filecsistorage", Path: "filecsipath"}, ), ), ) @@ -3869,8 +3846,8 @@ func Test_ComponentSynced_VolumeAndMounts(t *testing.T) { envNamespace := utils.GetEnvironmentNamespace(appName, environment) deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.Background(), compName, metav1.GetOptions{}) require.NotNil(t, deployment) - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 3, "incorrect number of volumes") - assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 3, "incorrect number of volumemounts") + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 1, "incorrect number of volumes") + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 1, "incorrect number of volumemounts") } func Test_JobSynced_VolumeAndMounts(t *testing.T) { @@ -3891,9 +3868,7 @@ func Test_JobSynced_VolumeAndMounts(t *testing.T) { utils.NewDeployJobComponentBuilder(). WithName(jobName). WithVolumeMounts( - radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlob, Name: "blob", Container: "blobcontainer", Path: "blobpath"}, radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blobcsi", Storage: "blobcsistorage", Path: "blobcsipath"}, - radixv1.RadixVolumeMount{Type: radixv1.MountTypeAzureFileCsiAzure, Name: "filecsi", Storage: "filecsistorage", Path: "filecsipath"}, ), ), ) @@ -3903,7 +3878,7 @@ func Test_JobSynced_VolumeAndMounts(t *testing.T) { deploymentList, _ := client.AppsV1().Deployments(envNamespace).List(context.Background(), metav1.ListOptions{LabelSelector: radixlabels.ForJobAuxObject(jobName, kube.RadixJobTypeManagerAux).String()}) require.Len(t, deploymentList.Items, 1) deployment := deploymentList.Items[0] - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 3, "incorrect number of volumes") + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 1, "incorrect number of volumes") assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 0, "incorrect number of volumemounts") } diff --git a/pkg/apis/deployment/environmentvariables.go b/pkg/apis/deployment/environmentvariables.go index 7a1fbc7d7..75b2120be 100644 --- a/pkg/apis/deployment/environmentvariables.go +++ b/pkg/apis/deployment/environmentvariables.go @@ -8,6 +8,7 @@ import ( radixmaps "github.com/equinor/radix-common/utils/maps" "github.com/equinor/radix-operator/pkg/apis/defaults" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" @@ -111,7 +112,7 @@ func getEnvironmentVariables(ctx context.Context, appName string, envVarsSource envVars = appendDefaultEnvVars(ctx, envVars, envVarsSource, currentEnvironment, namespace, appName, deployComponent) - if !isDeployComponentJobSchedulerDeployment(deployComponent) { // JobScheduler does not need env-vars for secrets and secret-refs + if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { // JobScheduler does not need env-vars for secrets and secret-refs envVars = append(envVars, utils.GetEnvVarsFromSecrets(deployComponent.GetName(), deployComponent.GetSecrets())...) envVars = append(envVars, utils.GetEnvVarsFromAzureKeyVaultSecretRefs(radixDeployment.GetName(), deployComponent.GetName(), deployComponent.GetSecretRefs())...) } diff --git a/pkg/apis/deployment/externaldns.go b/pkg/apis/deployment/externaldns.go index 0ff3326b5..9aa2dc762 100644 --- a/pkg/apis/deployment/externaldns.go +++ b/pkg/apis/deployment/externaldns.go @@ -81,8 +81,7 @@ func (deploy *Deployment) garbageCollectExternalDnsSecretsNoLongerInSpec(ctx con if slice.Any(externalDnsAliases, func(rded radixv1.RadixDeployExternalDNS) bool { return rded.FQDN == fqdn }) { continue } - - if err := deploy.deleteSecret(ctx, &secret); err != nil { + if err := deploy.kubeutil.DeleteSecret(ctx, deploy.radixDeployment.Namespace, secret.Name); err != nil { return nil } } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 3b7e329ce..840831349 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -6,13 +6,14 @@ import ( commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/securitycontext" "github.com/equinor/radix-operator/pkg/apis/utils" radixannotations "github.com/equinor/radix-operator/pkg/apis/utils/annotations" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" - "github.com/equinor/radix-operator/pkg/apis/utils/resources" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -22,42 +23,40 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) error { - currentDeployment, desiredDeployment, err := deploy.getCurrentAndDesiredDeployment(ctx, deployComponent) +func (deploy *Deployment) reconcileDeployComponent(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) error { + namespace := deploy.radixDeployment.Namespace + currentDeployment, desiredDeployment, err := deploy.getCurrentAndDesiredDeployment(ctx, namespace, deployComponent) if err != nil { return err } // If component has manual override or HorizontalScaling is nil then delete hpa if exists before updating deployment if deployComponent.GetReplicasOverride() != nil || deployComponent.GetHorizontalScaling() == nil { - err = deploy.deleteScaledObjectIfExists(ctx, deployComponent.GetName()) - if err != nil { + if err = deploy.deleteScaledObjectIfExists(ctx, deployComponent.GetName()); err != nil { return err } - - err = deploy.deleteTargetAuthenticationIfExists(ctx, deployComponent.GetName()) - if err != nil { + if err = deploy.deleteTargetAuthenticationIfExists(ctx, deployComponent.GetName()); err != nil { return err } } - - err = deploy.createOrUpdateCsiAzureVolumeResources(ctx, desiredDeployment) + actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment.Spec.Template.Spec.Volumes) if err != nil { return err } - err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment) - if err != nil { + desiredDeployment.Spec.Template.Spec.Volumes = actualVolumes + desiredVolumeMounts := desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts + if err = deploy.handleJobAuxDeployment(ctx, namespace, deployComponent, desiredDeployment, actualVolumes, desiredVolumeMounts); err != nil { return err } - - return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentDeployment, desiredDeployment) + return deploy.kubeutil.ApplyDeployment(ctx, namespace, currentDeployment, desiredDeployment) } -func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { - if !isDeployComponentJobSchedulerDeployment(deployComponent) { +func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, namespace string, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) error { + if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil } - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, desiredDeployment) + jobKubeDeploymentName := desiredDeployment.GetName() + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, jobKubeDeploymentName, volumes, volumeMounts) if err != nil { return err } @@ -66,28 +65,27 @@ func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComp selector := labels.Set(desiredJobAuxDeployment.Spec.Selector.MatchLabels).AsSelector() if currentJobAuxDeployment != nil && !selector.Matches(labels.Set(currentJobAuxDeployment.Spec.Template.Labels)) { log.Ctx(ctx).Info().Msgf("Deleting outdated deployment (label selector does not match) %s", currentJobAuxDeployment.GetName()) - err = deploy.kubeutil.DeleteDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment.Name) - if err != nil { + if err = deploy.kubeutil.DeleteDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment.Name); err != nil { return err } - currentJobAuxDeployment, desiredJobAuxDeployment, err = deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, desiredDeployment) + currentJobAuxDeployment, desiredJobAuxDeployment, err = deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, jobKubeDeploymentName, volumes, volumeMounts) if err != nil { return err } } + // Remove volumes and volume mounts from job scheduler deployment, they are set to aux deployment + desiredDeployment.Spec.Template.Spec.Volumes = nil + desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = nil return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment, desiredJobAuxDeployment) } -func (deploy *Deployment) getCurrentAndDesiredDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) (*appsv1.Deployment, *appsv1.Deployment, error) { - namespace := deploy.radixDeployment.Namespace - +func (deploy *Deployment) getCurrentAndDesiredDeployment(ctx context.Context, namespace string, deployComponent v1.RadixCommonDeployComponent) (*appsv1.Deployment, *appsv1.Deployment, error) { currentDeployment, desiredDeployment, err := deploy.getDesiredDeployment(ctx, namespace, deployComponent) if err != nil { return nil, nil, err } - return currentDeployment, desiredDeployment, err } @@ -121,7 +119,7 @@ func (deploy *Deployment) getDesiredCreatedDeploymentConfig(ctx context.Context, desiredDeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, Spec: appsv1.DeploymentSpec{ - Replicas: pointers.Ptr(DefaultReplicas), + Replicas: pointers.Ptr(defaults.DefaultReplicas), Selector: &metav1.LabelSelector{MatchLabels: make(map[string]string)}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, @@ -133,9 +131,8 @@ func (deploy *Deployment) getDesiredCreatedDeploymentConfig(ctx context.Context, err := deploy.setDesiredDeploymentProperties(ctx, deployComponent, desiredDeployment) return desiredDeployment, err } -func (deploy *Deployment) createJobAuxDeployment(deployComponent v1.RadixCommonDeployComponent) *appsv1.Deployment { - jobName := deployComponent.GetName() - jobAuxDeploymentName := getJobAuxObjectName(jobName) + +func (deploy *Deployment) createJobAuxDeployment(jobName, jobAuxDeploymentName string) *appsv1.Deployment { desiredDeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: jobAuxDeploymentName, @@ -151,7 +148,7 @@ func (deploy *Deployment) createJobAuxDeployment(deployComponent v1.RadixCommonD Spec: corev1.PodSpec{Containers: []corev1.Container{ { Name: jobAuxDeploymentName, - Resources: resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)), + Resources: getJobAuxResources(), }}, }, }, @@ -192,18 +189,18 @@ func (deploy *Deployment) getDesiredUpdatedDeploymentConfig(ctx context.Context, func (deploy *Deployment) getDeploymentPodLabels(deployComponent v1.RadixCommonDeployComponent) map[string]string { commitID := getDeployComponentCommitId(deployComponent) - labels := radixlabels.Merge( + lbs := radixlabels.Merge( radixlabels.ForApplicationName(deploy.radixDeployment.Spec.AppName), radixlabels.ForComponentName(deployComponent.GetName()), radixlabels.ForCommitId(commitID), radixlabels.ForPodWithRadixIdentity(deployComponent.GetIdentity()), ) - if isDeployComponentJobSchedulerDeployment(deployComponent) { - labels = radixlabels.Merge(labels, radixlabels.ForPodIsJobScheduler()) + if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { + lbs = radixlabels.Merge(lbs, radixlabels.ForPodIsJobScheduler()) } - return labels + return lbs } func (deploy *Deployment) getJobAuxDeploymentPodLabels(deployComponent v1.RadixCommonDeployComponent) map[string]string { @@ -293,7 +290,11 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de desiredDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForDeployComponent(ctx, deployComponent, appName, componentName) desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetDeploymentPodSpecTolerations(deployComponent.GetNode()) - volumes, err := deploy.GetVolumesForComponent(ctx, deployComponent) + existingVolumes, err := deploy.getDeployComponentExistingVolumes(ctx, deployComponent, desiredDeployment) + if err != nil { + return err + } + volumes, err := volumemount.GetVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent, deploy.radixDeployment.GetName(), existingVolumes) if err != nil { return err } @@ -309,7 +310,7 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de return err } - volumeMounts, err := GetRadixDeployComponentVolumeMounts(deployComponent, deploy.radixDeployment.GetName()) + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(deployComponent, deploy.radixDeployment.GetName()) if err != nil { return err } @@ -336,6 +337,17 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de return nil } +func (deploy *Deployment) getDeployComponentExistingVolumes(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, deployment *appsv1.Deployment) ([]corev1.Volume, error) { + if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { + volumes, err := volumemount.GetExistingJobAuxComponentVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent.GetName()) + if err != nil { + return nil, err + } + return volumes, nil + } + return deployment.Spec.Template.Spec.Volumes, nil +} + func (deploy *Deployment) getRadixBranchAndCommitId() (string, string) { const branchKey, commitIDKey = "radix-branch", "radix-commit" rdLabels := deploy.radixDeployment.Labels @@ -359,7 +371,7 @@ func getDeployComponentReplicas(deployComponent v1.RadixCommonDeployComponent) i return int32(*override) } - componentReplicas := DefaultReplicas + componentReplicas := defaults.DefaultReplicas if replicas := deployComponent.GetReplicas(); replicas != nil { componentReplicas = int32(*replicas) } @@ -501,7 +513,7 @@ func getContainerPorts(deployComponent v1.RadixCommonDeployComponent) []corev1.C for _, v := range componentPorts { containerPort := corev1.ContainerPort{ Name: v.Name, - ContainerPort: int32(v.Port), + ContainerPort: v.Port, Protocol: corev1.ProtocolTCP, } ports = append(ports, containerPort) diff --git a/pkg/apis/deployment/kubedeployment_test.go b/pkg/apis/deployment/kubedeployment_test.go index 2d3d31cca..aa8833d38 100644 --- a/pkg/apis/deployment/kubedeployment_test.go +++ b/pkg/apis/deployment/kubedeployment_test.go @@ -255,10 +255,7 @@ func applyDeploymentWithSyncWithComponentResources(t *testing.T, origRequests, o func TestDeployment_createJobAuxDeployment(t *testing.T) { deploy := &Deployment{radixDeployment: &v1.RadixDeployment{ObjectMeta: metav1.ObjectMeta{Name: "deployment1", UID: "uid1"}}} - jobDeployComponent := &v1.RadixDeployComponent{ - Name: "job1", - } - jobAuxDeployment := deploy.createJobAuxDeployment(jobDeployComponent) + jobAuxDeployment := deploy.createJobAuxDeployment("job1", "job1-aux") assert.Equal(t, "job1-aux", jobAuxDeployment.GetName()) resources := jobAuxDeployment.Spec.Template.Spec.Containers[0].Resources s := resources.Requests.Cpu().String() diff --git a/pkg/apis/deployment/radixcomponent.go b/pkg/apis/deployment/radixcomponent.go index 2fee75d33..7fa8a40a7 100644 --- a/pkg/apis/deployment/radixcomponent.go +++ b/pkg/apis/deployment/radixcomponent.go @@ -300,6 +300,7 @@ func getEnvironmentSpecificConfigForComponent(component radixv1.RadixComponent, } func getRadixComponentPort(radixComponent *radixv1.RadixComponent) string { + //nolint:staticcheck if radixComponent.PublicPort == "" && radixComponent.Public { return radixComponent.Ports[0].Name } diff --git a/pkg/apis/deployment/radixcomponent_test.go b/pkg/apis/deployment/radixcomponent_test.go index f787646c7..49293a139 100644 --- a/pkg/apis/deployment/radixcomponent_test.go +++ b/pkg/apis/deployment/radixcomponent_test.go @@ -1,3 +1,4 @@ +//nolint:staticcheck package deployment import ( @@ -175,7 +176,6 @@ func TestGetOAuth2AuthenticationForComponent(t *testing.T) { } } -//nolint:staticcheck func TestGetRadixComponentsForEnv_PublicPort_OldPublic(t *testing.T) { // New publicPort does not exist, old public does not exist componentName := "comp" diff --git a/pkg/apis/deployment/resources.go b/pkg/apis/deployment/resources.go new file mode 100644 index 000000000..5426e092c --- /dev/null +++ b/pkg/apis/deployment/resources.go @@ -0,0 +1,10 @@ +package deployment + +import ( + "github.com/equinor/radix-operator/pkg/apis/utils/resources" + "k8s.io/api/core/v1" +) + +func getJobAuxResources() v1.ResourceRequirements { + return resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)) +} diff --git a/pkg/apis/deployment/secretrefs.go b/pkg/apis/deployment/secretrefs.go index 5fa0fbb36..8a0f16245 100644 --- a/pkg/apis/deployment/secretrefs.go +++ b/pkg/apis/deployment/secretrefs.go @@ -30,9 +30,9 @@ func (deploy *Deployment) createSecretRefs(ctx context.Context, namespace string return nil, err } if credsSecret != nil && !isOwnerReference(credsSecret.ObjectMeta, secretProviderClass.ObjectMeta) { - credsSecret.ObjectMeta.OwnerReferences = append(credsSecret.ObjectMeta.OwnerReferences, getOwnerReferenceOfSecretProviderClass(secretProviderClass)) - _, err = deploy.kubeutil.ApplySecret(ctx, namespace, credsSecret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret - if err != nil { + updatedCredsSecret := credsSecret.DeepCopy() + updatedCredsSecret.ObjectMeta.OwnerReferences = append(credsSecret.ObjectMeta.OwnerReferences, getOwnerReferenceOfSecretProviderClass(secretProviderClass)) + if _, err = deploy.kubeutil.UpdateSecret(ctx, credsSecret, updatedCredsSecret); err != nil { return nil, err } } @@ -45,7 +45,7 @@ func (deploy *Deployment) getOrCreateSecretProviderClass(ctx context.Context, na secretProviderClass, err := deploy.kubeutil.GetSecretProviderClass(ctx, namespace, className) if err != nil { if errors.IsNotFound(err) { - return deploy.createAzureKeyVaultSecretProviderClassForRadixDeployment(ctx, namespace, appName, radixDeployComponentName, radixAzureKeyVault) + return deploy.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(ctx, namespace, appName, radixDeployComponentName, radixAzureKeyVault) } return nil, err } @@ -59,10 +59,10 @@ func (deploy *Deployment) getAzureKeyVaultCredsSecret(ctx context.Context, names return deploy.getOrCreateAzureKeyVaultCredsSecret(ctx, namespace, appName, radixDeployComponentName, azureKeyVault.Name) } -func (deploy *Deployment) createAzureKeyVaultSecretProviderClassForRadixDeployment(ctx context.Context, namespace string, appName string, radixDeployComponentName string, azureKeyVault radixv1.RadixAzureKeyVault) (*secretsstorev1.SecretProviderClass, error) { +func (deploy *Deployment) CreateAzureKeyVaultSecretProviderClassForRadixDeployment(ctx context.Context, namespace string, appName string, radixDeployComponentName string, azureKeyVault radixv1.RadixAzureKeyVault) (*secretsstorev1.SecretProviderClass, error) { radixDeploymentName := deploy.radixDeployment.GetName() tenantId := deploy.config.DeploymentSyncer.TenantID - identity := getIdentityFromRadixCommonDeployComponent(deploy, radixDeployComponentName) + identity := deploy.getIdentityFromRadixCommonDeployComponent(radixDeployComponentName) secretProviderClass, err := kube.BuildAzureKeyVaultSecretProviderClass(tenantId, appName, radixDeploymentName, radixDeployComponentName, azureKeyVault, identity) if err != nil { return nil, err @@ -73,7 +73,7 @@ func (deploy *Deployment) createAzureKeyVaultSecretProviderClassForRadixDeployme return deploy.kubeutil.CreateSecretProviderClass(ctx, namespace, secretProviderClass) } -func getIdentityFromRadixCommonDeployComponent(deploy *Deployment, radixDeployComponentName string) *radixv1.Identity { +func (deploy *Deployment) getIdentityFromRadixCommonDeployComponent(radixDeployComponentName string) *radixv1.Identity { if radixDeployComponent := deploy.radixDeployment.GetComponentByName(radixDeployComponentName); radixDeployComponent != nil { return radixDeployComponent.GetIdentity() } @@ -89,7 +89,7 @@ func (deploy *Deployment) getOrCreateAzureKeyVaultCredsSecret(ctx context.Contex if err != nil { if errors.IsNotFound(err) { secret = buildAzureKeyVaultCredentialsSecret(appName, componentName, secretName, azKeyVaultName) - return deploy.kubeutil.ApplySecret(ctx, namespace, secret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret + return deploy.kubeutil.CreateSecret(ctx, namespace, secret) } return nil, err } diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index 926d96584..8fd08df17 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -11,17 +11,13 @@ import ( "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "github.com/rs/zerolog/log" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( - secretDefaultData = "xx" - secretUsedBySecretStoreDriverLabel = "secrets-store.csi.k8s.io/used" -) - func tlsSecretDefaultData() map[string][]byte { return map[string][]byte{ v1.TLSCertKey: nil, @@ -67,13 +63,13 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, secretsToManage = append(secretsToManage, secretName) } - volumeMountSecretsToManage, err := deploy.createOrUpdateVolumeMountSecrets(ctx, namespace, component.GetName(), component.GetVolumeMounts()) + volumeMountSecretsToManage, err := volumemount.CreateOrUpdateVolumeMountSecrets(ctx, deploy.kubeutil, deploy.registration.Name, namespace, component.GetName(), component.GetVolumeMounts()) if err != nil { return err } secretsToManage = append(secretsToManage, volumeMountSecretsToManage...) - err = deploy.garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, component, secretsToManage) + err = volumemount.GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, deploy.kubeutil, namespace, component, secretsToManage) if err != nil { return err } @@ -113,57 +109,6 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, return nil } -func (deploy *Deployment) createOrUpdateVolumeMountSecrets(ctx context.Context, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { - var volumeMountSecretsToManage []string - for _, volumeMount := range volumeMounts { - switch GetCsiAzureVolumeMountType(&volumeMount) { - case radixv1.MountTypeBlob: - { - secretName, accountKey, accountName := deploy.getBlobFuseCredsSecrets(ctx, namespace, componentName, volumeMount.Name) - volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) - err := deploy.createOrUpdateVolumeMountsSecrets(ctx, namespace, componentName, secretName, accountName, accountKey) - if err != nil { - return nil, err - } - } - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure, radixv1.MountTypeBlobFuse2NfsCsiAzure, radixv1.MountTypeAzureFileCsiAzure: - { - secretName, accountKey, accountName := deploy.getCsiAzureVolumeMountCredsSecrets(ctx, namespace, componentName, volumeMount.Name) - volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) - err := deploy.createOrUpdateCsiAzureVolumeMountsSecrets(ctx, namespace, componentName, &volumeMount, secretName, accountName, accountKey) - if err != nil { - return nil, err - } - } - } - } - return volumeMountSecretsToManage, nil -} - -func (deploy *Deployment) getBlobFuseCredsSecrets(ctx context.Context, ns, componentName, volumeMountName string) (string, []byte, []byte) { - secretName := defaults.GetBlobFuseCredsSecretName(componentName, volumeMountName) - accountKey := []byte(secretDefaultData) - accountName := []byte(secretDefaultData) - if deploy.kubeutil.SecretExists(ctx, ns, secretName) { - oldSecret, _ := deploy.kubeutil.GetSecret(ctx, ns, secretName) - accountKey = oldSecret.Data[defaults.BlobFuseCredsAccountKeyPart] - accountName = oldSecret.Data[defaults.BlobFuseCredsAccountNamePart] - } - return secretName, accountKey, accountName -} - -func (deploy *Deployment) getCsiAzureVolumeMountCredsSecrets(ctx context.Context, namespace, componentName, volumeMountName string) (string, []byte, []byte) { - secretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, volumeMountName) - accountKey := []byte(secretDefaultData) - accountName := []byte(secretDefaultData) - if deploy.kubeutil.SecretExists(ctx, namespace, secretName) { - oldSecret, _ := deploy.kubeutil.GetSecret(ctx, namespace, secretName) - accountKey = oldSecret.Data[defaults.CsiAzureCredsAccountKeyPart] - accountName = oldSecret.Data[defaults.CsiAzureCredsAccountNamePart] - } - return secretName, accountKey, accountName -} - func (deploy *Deployment) garbageCollectSecretsNoLongerInSpec(ctx context.Context) error { secrets, err := deploy.kubeutil.ListSecrets(ctx, deploy.radixDeployment.GetNamespace()) if err != nil { @@ -177,8 +122,7 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpec(ctx context.Contex } if deploy.isEligibleForGarbageCollectSecretsForComponent(existingSecret, componentName) { - err := deploy.deleteSecret(ctx, existingSecret) - if err != nil { + if err := deploy.kubeutil.DeleteSecret(ctx, deploy.radixDeployment.GetNamespace(), existingSecret.Name); err != nil && !errors.IsNotFound(err) { return err } } @@ -214,8 +158,7 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpecForComponent(ctx co } log.Ctx(ctx).Debug().Msgf("Delete secret %s no longer in spec for component %s", secret.Name, component.GetName()) - err = deploy.deleteSecret(ctx, secret) - if err != nil { + if err = deploy.kubeutil.DeleteSecret(ctx, deploy.radixDeployment.GetNamespace(), secret.Name); err != nil && !errors.IsNotFound(err) { return err } } @@ -224,26 +167,11 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpecForComponent(ctx co } func (deploy *Deployment) listSecretsForComponent(ctx context.Context, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { - return deploy.listSecrets(ctx, getLabelSelectorForComponent(component)) + return listSecrets(ctx, deploy.kubeutil, deploy.radixDeployment.GetNamespace(), getLabelSelectorForComponent(component)) } -func (deploy *Deployment) listSecretsForVolumeMounts(ctx context.Context, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { - blobVolumeMountSecret := getLabelSelectorForBlobVolumeMountSecret(component) - secrets, err := deploy.listSecrets(ctx, blobVolumeMountSecret) - if err != nil { - return nil, err - } - csiAzureVolumeMountSecret := getLabelSelectorForCsiAzureVolumeMountSecret(component) - csiSecrets, err := deploy.listSecrets(ctx, csiAzureVolumeMountSecret) - if err != nil { - return nil, err - } - secrets = append(secrets, csiSecrets...) - return secrets, err -} - -func (deploy *Deployment) listSecrets(ctx context.Context, labelSelector string) ([]*v1.Secret, error) { - secrets, err := deploy.kubeutil.ListSecretsWithSelector(ctx, deploy.radixDeployment.GetNamespace(), labelSelector) +func listSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, labelSelector string) ([]*v1.Secret, error) { + secrets, err := kubeUtil.ListSecretsWithSelector(ctx, namespace, labelSelector) if err != nil { return nil, err @@ -288,17 +216,17 @@ func buildAzureKeyVaultCredentialsSecret(appName, componentName, secretName, azK ObjectMeta: metav1.ObjectMeta{ Name: secretName, Labels: map[string]string{ - kube.RadixAppLabel: appName, - kube.RadixComponentLabel: componentName, - kube.RadixSecretRefTypeLabel: string(radixv1.RadixSecretRefTypeAzureKeyVault), - kube.RadixSecretRefNameLabel: strings.ToLower(azKeyVaultName), - secretUsedBySecretStoreDriverLabel: "true", // used by CSI Azure Key vault secret store driver for secret rotation + kube.RadixAppLabel: appName, + kube.RadixComponentLabel: componentName, + kube.RadixSecretRefTypeLabel: string(radixv1.RadixSecretRefTypeAzureKeyVault), + kube.RadixSecretRefNameLabel: strings.ToLower(azKeyVaultName), + defaults.SecretUsedBySecretStoreDriverLabel: "true", // used by CSI Azure Key vault secret store driver for secret rotation }, }, } data := make(map[string][]byte) - defaultValue := []byte(secretDefaultData) + defaultValue := []byte(defaults.SecretDefaultData) data["clientid"] = defaultValue data["clientsecret"] = defaultValue @@ -318,7 +246,7 @@ func (deploy *Deployment) createClientCertificateSecret(ctx context.Context, ns, }, } - defaultValue := []byte(secretDefaultData) + defaultValue := []byte(defaults.SecretDefaultData) // Will need to set fake data in order to apply the secret. The user then need to set data to real values data := make(map[string][]byte) @@ -353,27 +281,3 @@ func (deploy *Deployment) removeOrphanedSecrets(ctx context.Context, ns, secretN return nil } - -// GarbageCollectSecrets delete secrets, excluding with names in the excludeSecretNames -func (deploy *Deployment) GarbageCollectSecrets(ctx context.Context, secrets []*v1.Secret, excludeSecretNames []string) error { - for _, secret := range secrets { - if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { - continue - } - err := deploy.deleteSecret(ctx, secret) - if err != nil { - return err - } - } - return nil -} - -func (deploy *Deployment) deleteSecret(ctx context.Context, secret *v1.Secret) error { - log.Ctx(ctx).Debug().Msgf("Delete secret %s", secret.Name) - err := deploy.kubeclient.CoreV1().Secrets(deploy.radixDeployment.GetNamespace()).Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - log.Ctx(ctx).Info().Msgf("Deleted secret: %s in namespace %s", secret.GetName(), deploy.radixDeployment.GetNamespace()) - return nil -} diff --git a/pkg/apis/deployment/service.go b/pkg/apis/deployment/service.go index ed8c737b9..f6f5b2ba3 100644 --- a/pkg/apis/deployment/service.go +++ b/pkg/apis/deployment/service.go @@ -3,8 +3,9 @@ package deployment import ( "context" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -59,7 +60,7 @@ func getServiceConfig(component v1.RadixCommonDeployComponent, radixDeployment * } selector := map[string]string{kube.RadixComponentLabel: component.GetName()} - if isDeployComponentJobSchedulerDeployment(component) { + if internal.IsDeployComponentJobSchedulerDeployment(component) { selector[kube.RadixPodIsJobSchedulerLabel] = "true" } diff --git a/pkg/apis/deployment/serviceaccountspec.go b/pkg/apis/deployment/serviceaccountspec.go index 508c254ac..4e1cf643f 100644 --- a/pkg/apis/deployment/serviceaccountspec.go +++ b/pkg/apis/deployment/serviceaccountspec.go @@ -3,7 +3,8 @@ package deployment import ( "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" ) @@ -67,7 +68,7 @@ func (spec *radixComponentServiceAccountSpec) AutomountServiceAccountToken() *bo // NewServiceAccountSpec Create ServiceAccountSpec based on RadixDeployment and RadixCommonDeployComponent func NewServiceAccountSpec(radixDeploy *v1.RadixDeployment, deployComponent v1.RadixCommonDeployComponent) ServiceAccountSpec { isComponent := deployComponent.GetType() == v1.RadixComponentTypeComponent - isJobScheduler := isDeployComponentJobSchedulerDeployment(deployComponent) + isJobScheduler := internal.IsDeployComponentJobSchedulerDeployment(deployComponent) if isComponent && isRadixAPI(radixDeploy) { return &radixAPIServiceAccountSpec{} diff --git a/pkg/apis/deployment/serviceaccountspec_test.go b/pkg/apis/deployment/serviceaccountspec_test.go index 66520f7a4..f41a48c79 100644 --- a/pkg/apis/deployment/serviceaccountspec_test.go +++ b/pkg/apis/deployment/serviceaccountspec_test.go @@ -5,7 +5,8 @@ import ( "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/stretchr/testify/assert" ) @@ -40,7 +41,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(false), spec.AutomountServiceAccountToken()) assert.Equal(t, utils.GetComponentServiceAccountName(rd.Spec.Components[1].Name), spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -48,7 +49,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(false), spec.AutomountServiceAccountToken()) assert.Equal(t, defaultServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[1], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[1], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -71,7 +72,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixAPIServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -95,7 +96,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixGithubWebhookServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) diff --git a/pkg/apis/deployment/volume_test.go b/pkg/apis/deployment/volume_test.go new file mode 100644 index 000000000..a67544868 --- /dev/null +++ b/pkg/apis/deployment/volume_test.go @@ -0,0 +1,38 @@ +package deployment + +import ( + "context" + "testing" + + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_EmptyDir(t *testing.T) { + appName, envName, compName := "anyapp", "anyenv", "anycomp" + + tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t) + builder := utils.NewDeploymentBuilder(). + WithRadixApplication(utils.NewRadixApplicationBuilder().WithAppName(appName).WithRadixRegistration(utils.NewRegistrationBuilder().WithName(appName))). + WithAppName(appName). + WithEnvironment(envName). + WithComponents( + utils.NewDeployComponentBuilder().WithName(compName).WithVolumeMounts( + v1.RadixVolumeMount{Name: "cache", Path: "/cache", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("50M")}}, + v1.RadixVolumeMount{Name: "log", Path: "/log", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("100M")}}, + ), + ) + + rd, err := ApplyDeploymentWithSync(tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, certClient, builder) + require.NoError(t, err) + assert.NotNil(t, rd) + + deployment, err := kubeclient.AppsV1().Deployments(utils.GetEnvironmentNamespace(appName, envName)).Get(context.Background(), compName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2) + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) +} diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go deleted file mode 100644 index 1d5109919..000000000 --- a/pkg/apis/deployment/volumemount.go +++ /dev/null @@ -1,1079 +0,0 @@ -package deployment - -import ( - "context" - "fmt" - "sort" - "strings" - - commonUtils "github.com/equinor/radix-common/utils" - "github.com/equinor/radix-operator/pkg/apis/defaults" - "github.com/equinor/radix-operator/pkg/apis/kube" - radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - "github.com/equinor/radix-operator/pkg/apis/utils" - "github.com/rs/zerolog/log" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -const ( - persistentVolumeClaimKind = "PersistentVolumeClaim" - - blobfuseDriver = "azure/blobfuse" - defaultMountOptions = "--file-cache-timeout-in-seconds=120" - - blobFuseVolumeNameTemplate = "blobfuse-%s-%s" // blobfuse-- - blobFuseVolumeNodeMountPathTemplate = "/tmp/%s/%s/%s/%s/%s/%s" // /tmp////// - - csiVolumeNameTemplate = "%s-%s-%s-%s" // --- - csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- - csiStorageClassNameTemplate = "sc-%s-%s" // sc-- - csiVolumeNodeMountPathTemplate = "%s/%s/%s/%s/%s/%s" // ///// - - csiStorageClassProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key - csiStorageClassProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret - csiStorageClassNodeStageSecretNameParameter = "csi.storage.k8s.io/node-stage-secret-name" // Usually equal to csiStorageClassProvisionerSecretNameParameter - csiStorageClassNodeStageSecretNamespaceParameter = "csi.storage.k8s.io/node-stage-secret-namespace" // Usually equal to csiStorageClassProvisionerSecretNamespaceParameter - csiAzureStorageClassSkuNameParameter = "skuName" // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - csiStorageClassContainerNameParameter = "containerName" // Container name - foc container storages - csiStorageClassShareNameParameter = "shareName" // File Share name - for file storages - csiStorageClassTmpPathMountOption = "tmp-path" // Path within the node, where the volume mount has been mounted to - csiStorageClassGidMountOption = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting - csiStorageClassUidMountOption = "uid" // Volume mount owner UserID. Used instead of GroupID - csiStorageClassUseAdlsMountOption = "use-adls" // Use ADLS or Block Blob - csiStorageClassStreamingEnabledMountOption = "streaming" // Enable Streaming - csiStorageClassStreamingCacheMountOption = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory - csiStorageClassStreamingMaxBlocksPerFileMountOption = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming - csiStorageClassStreamingMaxBuffersMountOption = "max-buffers" // The total number of buffers to be cached in memory (in MB). - csiStorageClassStreamingBlockSizeMountOption = "block-size-mb" // The size of each block to be cached in memory (in MB). - csiStorageClassStreamingBufferSizeMountOption = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). - csiStorageClassProtocolParameter = "protocol" // Protocol - csiStorageClassProtocolParameterFuse = "fuse" // Protocol "blobfuse" - csiStorageClassProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" - csiStorageClassProtocolParameterNfs = "nfs" // Protocol "nfs" - - csiSecretStoreDriver = "secrets-store.csi.k8s.io" - csiVolumeSourceVolumeAttrSecretProviderClassName = "secretProviderClass" - csiAzureKeyVaultSecretMountPathTemplate = "/mnt/azure-key-vault/%s" - - volumeNameMaxLength = 63 -) - -// These are valid storage class provisioners -const ( - // provisionerBlobCsiAzure Use of azure/csi driver for blob in Azure storage account - provisionerBlobCsiAzure string = "blob.csi.azure.com" - // provisionerFileCsiAzure Use of azure/csi driver for files in Azure storage account - provisionerFileCsiAzure string = "file.csi.azure.com" -) - -var ( - csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}, provisionerFileCsiAzure: struct{}{}} -) - -// getStorageClassProvisionerByVolumeMountType convert volume mount type to Storage Class provisioner -func getStorageClassProvisionerByVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) (string, bool) { - if radixVolumeMount.BlobFuse2 != nil { - return provisionerBlobCsiAzure, true - } - if radixVolumeMount.AzureFile != nil { - return provisionerFileCsiAzure, true - } - switch radixVolumeMount.Type { - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure, radixv1.MountTypeBlobFuse2NfsCsiAzure: - return provisionerBlobCsiAzure, true - case radixv1.MountTypeAzureFileCsiAzure: - return provisionerFileCsiAzure, true - } - return "", false -} - -// isKnownCsiAzureVolumeMount Supported volume mount type CSI Azure Blob volume -func isKnownCsiAzureVolumeMount(volumeMount string) bool { - switch volumeMount { - case string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure): - return true - } - return false -} - -// GetRadixDeployComponentVolumeMounts Gets list of v1.VolumeMount for radixv1.RadixCommonDeployComponent -func GetRadixDeployComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.VolumeMount, error) { - componentName := deployComponent.GetName() - volumeMounts := make([]corev1.VolumeMount, 0) - componentVolumeMounts, err := getRadixComponentVolumeMounts(deployComponent) - if err != nil { - return nil, err - } - volumeMounts = append(volumeMounts, componentVolumeMounts...) - secretRefsVolumeMounts := getRadixComponentSecretRefsVolumeMounts(deployComponent, componentName, radixDeploymentName) - volumeMounts = append(volumeMounts, secretRefsVolumeMounts...) - return volumeMounts, nil -} - -func getRadixComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.VolumeMount, error) { - if isDeployComponentJobSchedulerDeployment(deployComponent) { - return nil, nil - } - - var volumeMounts []corev1.VolumeMount - for _, volumeMount := range deployComponent.GetVolumeMounts() { - name, err := getVolumeMountVolumeName(&volumeMount, deployComponent.GetName()) - if err != nil { - return nil, err - } - volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: name, MountPath: volumeMount.Path}) - } - return volumeMounts, nil -} - -func getRadixComponentSecretRefsVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent, componentName, radixDeploymentName string) []corev1.VolumeMount { - secretRefs := deployComponent.GetSecretRefs() - var volumeMounts []corev1.VolumeMount - for _, azureKeyVault := range secretRefs.AzureKeyVaults { - k8sSecretTypeMap := make(map[corev1.SecretType]bool) - for _, keyVaultItem := range azureKeyVault.Items { - kubeSecretType := kube.GetSecretTypeForRadixAzureKeyVault(keyVaultItem.K8sSecretType) - if _, ok := k8sSecretTypeMap[kubeSecretType]; !ok { - k8sSecretTypeMap[kubeSecretType] = true - } - } - for kubeSecretType := range k8sSecretTypeMap { - volumeMountName := trimVolumeNameToValidLength(kube.GetAzureKeyVaultSecretRefSecretName(componentName, radixDeploymentName, azureKeyVault.Name, kubeSecretType)) - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: volumeMountName, - ReadOnly: true, - MountPath: getCsiAzureKeyVaultSecretMountPath(azureKeyVault), - }) - } - } - return volumeMounts -} - -func getCsiAzureKeyVaultSecretMountPath(azureKeyVault radixv1.RadixAzureKeyVault) string { - if azureKeyVault.Path == nil || *(azureKeyVault.Path) == "" { - return fmt.Sprintf(csiAzureKeyVaultSecretMountPathTemplate, azureKeyVault.Name) - } - return *azureKeyVault.Path -} - -func getBlobFuseVolumeMountName(volumeMount *radixv1.RadixVolumeMount, componentName string) string { - return trimVolumeNameToValidLength(fmt.Sprintf(blobFuseVolumeNameTemplate, componentName, volumeMount.Name)) -} - -func getCsiAzureVolumeMountName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - csiVolumeType, err := getCsiRadixVolumeTypeIdForName(volumeMount) - if err != nil { - return "", err - } - if len(volumeMount.Name) == 0 { - return "", fmt.Errorf("name is empty for volume mount in the component %s", componentName) - } - csiAzureVolumeStorageName := GetRadixVolumeMountStorage(volumeMount) - if len(csiAzureVolumeStorageName) == 0 { - return "", fmt.Errorf("storage is empty for volume mount %s in the component %s", volumeMount.Name, componentName) - } - if len(volumeMount.Path) == 0 { - return "", fmt.Errorf("path is empty for volume mount %s in the component %s", volumeMount.Name, componentName) - } - return trimVolumeNameToValidLength(fmt.Sprintf(csiVolumeNameTemplate, csiVolumeType, componentName, volumeMount.Name, csiAzureVolumeStorageName)), nil -} - -// GetCsiAzureVolumeMountType Gets the CSI Azure volume mount type -func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radixv1.MountType { - if radixVolumeMount.BlobFuse2 != nil { - switch radixVolumeMount.BlobFuse2.Protocol { - case radixv1.BlobFuse2ProtocolFuse2, "": // default protocol if not set - return radixv1.MountTypeBlobFuse2Fuse2CsiAzure - case radixv1.BlobFuse2ProtocolNfs: - return radixv1.MountTypeBlobFuse2NfsCsiAzure - default: - return "unsupported" - } - } - if radixVolumeMount.AzureFile != nil { - return radixv1.MountTypeAzureFileCsiAzure - } - return radixVolumeMount.Type -} - -func getCsiRadixVolumeTypeIdForName(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { - if radixVolumeMount.BlobFuse2 != nil { - switch radixVolumeMount.BlobFuse2.Protocol { - case radixv1.BlobFuse2ProtocolFuse2, "": - return "csi-blobfuse2-fuse2", nil - case radixv1.BlobFuse2ProtocolNfs: - return "csi-blobfuse2-nfs", nil - default: - return "", fmt.Errorf("unknown blobfuse2 protocol %s", radixVolumeMount.BlobFuse2.Protocol) - } - } - if radixVolumeMount.AzureFile != nil { - return "csi-az-file", nil - } - switch radixVolumeMount.Type { - case radixv1.MountTypeBlobFuse2FuseCsiAzure: - return "csi-az-blob", nil - case radixv1.MountTypeAzureFileCsiAzure: - return "csi-az-file", nil - } - return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) -} - -// GetVolumesForComponent Gets volumes for Radix deploy component or job -func (deploy *Deployment) GetVolumesForComponent(ctx context.Context, deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.Volume, error) { - return GetVolumes(ctx, deploy.kubeclient, deploy.kubeutil, deploy.getNamespace(), deploy.radixDeployment.Spec.Environment, deployComponent, deploy.radixDeployment.GetName()) -} - -// GetVolumes Get volumes of a component by RadixVolumeMounts -func GetVolumes(ctx context.Context, kubeclient kubernetes.Interface, kubeutil *kube.Kube, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { - var volumes []corev1.Volume - - volumeMountVolumes, err := getComponentVolumeMountVolumes(ctx, kubeclient, namespace, environment, deployComponent) - if err != nil { - return nil, err - } - volumes = append(volumes, volumeMountVolumes...) - - storageRefsVolumes, err := getComponentSecretRefsVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) - if err != nil { - return nil, err - } - volumes = append(volumes, storageRefsVolumes...) - - return volumes, nil -} - -func getComponentSecretRefsVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { - var volumes []corev1.Volume - azureKeyVaultVolumes, err := getComponentSecretRefsAzureKeyVaultVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) - if err != nil { - return nil, err - } - volumes = append(volumes, azureKeyVaultVolumes...) - return volumes, nil -} - -func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { - secretRef := deployComponent.GetSecretRefs() - var volumes []corev1.Volume - for _, azureKeyVault := range secretRef.AzureKeyVaults { - secretProviderClassName := kube.GetComponentSecretProviderClassName(radixDeploymentName, deployComponent.GetName(), radixv1.RadixSecretRefTypeAzureKeyVault, azureKeyVault.Name) - secretProviderClass, err := kubeutil.GetSecretProviderClass(ctx, namespace, secretProviderClassName) - if err != nil { - return nil, err - } - for _, secretObject := range secretProviderClass.Spec.SecretObjects { - volumeName := trimVolumeNameToValidLength(secretObject.SecretName) - volume := corev1.Volume{ - Name: volumeName, - } - provider := string(secretProviderClass.Spec.Provider) - switch provider { - case "azure": - volume.VolumeSource.CSI = &corev1.CSIVolumeSource{ - Driver: csiSecretStoreDriver, - ReadOnly: commonUtils.BoolPtr(true), - VolumeAttributes: map[string]string{csiVolumeSourceVolumeAttrSecretProviderClassName: secretProviderClass.Name}, - } - - useAzureIdentity := azureKeyVault.UseAzureIdentity != nil && *azureKeyVault.UseAzureIdentity - if !useAzureIdentity { - azKeyVaultName, azKeyVaultNameExists := secretProviderClass.Spec.Parameters[defaults.CsiSecretProviderClassParameterKeyVaultName] - if !azKeyVaultNameExists { - return nil, fmt.Errorf("missing Azure Key vault name in the secret provider class %s", secretProviderClass.Name) - } - credsSecretName := defaults.GetCsiAzureKeyVaultCredsSecretName(deployComponent.GetName(), azKeyVaultName) - volume.VolumeSource.CSI.NodePublishSecretRef = &corev1.LocalObjectReference{Name: credsSecretName} - } - default: - log.Ctx(ctx).Error().Msgf("Not supported provider %s in the secret provider class %s", provider, secretProviderClass.Name) - continue - } - volumes = append(volumes, volume) - } - } - return volumes, nil -} - -func getComponentVolumeMountVolumes(ctx context.Context, kubeclient kubernetes.Interface, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.Volume, error) { - var volumes []corev1.Volume - - volumeSourceFunc := func(volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { - switch { - case volumeMount.HasDeprecatedVolume(): - return getComponentVolumeMountDeprecatedVolumeSource(ctx, volumeMount, namespace, environment, deployComponent.GetName(), kubeclient) - case volumeMount.HasBlobFuse2(): - return getComponentVolumeMountBlobFuse2VolumeSource(ctx, volumeMount, namespace, deployComponent.GetName(), kubeclient) - case volumeMount.HasAzureFile(): - return getComponentVolumeMountAzureFileVolumeSource(ctx, volumeMount, namespace, deployComponent.GetName(), kubeclient) - case volumeMount.HasEmptyDir(): - return getComponentVolumeMountEmptyDirVolumeSource(volumeMount.EmptyDir), nil - } - return nil, fmt.Errorf("missing configuration for volumeMount %s", volumeMount.Name) - } - - for _, volumeMount := range deployComponent.GetVolumeMounts() { - volumeSource, err := volumeSourceFunc(&volumeMount) - if err != nil { - return nil, err - } - volumeName, err := getVolumeMountVolumeName(&volumeMount, deployComponent.GetName()) - if err != nil { - return nil, err - } - volumes = append(volumes, corev1.Volume{ - Name: volumeName, - VolumeSource: *volumeSource, - }) - } - return volumes, nil -} - -func getComponentVolumeMountDeprecatedVolumeSource(ctx context.Context, volumeMount *radixv1.RadixVolumeMount, namespace, environment, componentName string, kubeclient kubernetes.Interface) (*corev1.VolumeSource, error) { - switch volumeMount.Type { - case radixv1.MountTypeBlob: - return getBlobFuseVolume(namespace, environment, componentName, volumeMount), nil - case radixv1.MountTypeAzureFileCsiAzure, radixv1.MountTypeBlobFuse2FuseCsiAzure: - return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) - } - - return nil, fmt.Errorf("unsupported volume type %s", volumeMount.Type) -} - -func getComponentVolumeMountBlobFuse2VolumeSource(ctx context.Context, volumeMount *radixv1.RadixVolumeMount, namespace, componentName string, kubeclient kubernetes.Interface) (*corev1.VolumeSource, error) { - return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) -} - -func getComponentVolumeMountAzureFileVolumeSource(ctx context.Context, volumeMount *radixv1.RadixVolumeMount, namespace, componentName string, kubeclient kubernetes.Interface) (*corev1.VolumeSource, error) { - return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) -} - -func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolumeMount) *corev1.VolumeSource { - return &corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - SizeLimit: &spec.SizeLimit, - }, - } -} - -func getCsiAzureVolume(ctx context.Context, kubeclient kubernetes.Interface, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { - existingNotTerminatingPvcForComponentStorage, err := getPvcNotTerminating(ctx, kubeclient, namespace, componentName, radixVolumeMount) - if err != nil { - return nil, err - } - - var pvcName string - if existingNotTerminatingPvcForComponentStorage != nil { - pvcName = existingNotTerminatingPvcForComponentStorage.Name - } else { - pvcName, err = createCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) - if err != nil { - return nil, err - } - } - return &corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - }, - }, nil -} - -func getVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - switch { - case volumeMount.HasDeprecatedVolume(): - return getVolumeMountDeprecatedVolumeName(volumeMount, componentName) - case volumeMount.HasBlobFuse2(): - return getVolumeMountBlobFuse2VolumeName(volumeMount, componentName) - case volumeMount.HasAzureFile(): - return getVolumeMountAzureFileVolumeName(volumeMount, componentName) - } - - return fmt.Sprintf("radix-vm-%s", volumeMount.Name), nil -} - -func getVolumeMountAzureFileVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - return getCsiAzureVolumeMountName(volumeMount, componentName) -} - -func getVolumeMountBlobFuse2VolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - return getCsiAzureVolumeMountName(volumeMount, componentName) -} - -func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - switch volumeMount.Type { - case radixv1.MountTypeBlob: - return getBlobFuseVolumeMountName(volumeMount, componentName), nil - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeAzureFileCsiAzure: - return getCsiAzureVolumeMountName(volumeMount, componentName) - } - - return "", fmt.Errorf("unsupported type %s", volumeMount.Type) -} - -func getPvcNotTerminating(ctx context.Context, kubeclient kubernetes.Interface, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { - existingPvcForComponentStorage, err := kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentStorage(componentName, radixVolumeMount.Name), - }) - if err != nil { - return nil, err - } - existingPvcs := sortPvcsByCreatedTimestampDesc(existingPvcForComponentStorage.Items) - if len(existingPvcs) == 0 { - return nil, nil - } - for _, pvc := range existingPvcs { - switch pvc.Status.Phase { - case corev1.ClaimPending, corev1.ClaimBound: - return &pvc, nil - } - } - return nil, nil -} - -func createCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { - volumeName, err := getCsiAzureVolumeMountName(radixVolumeMount, componentName) - if err != nil { - return "", err - } - return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil // volumeName: --- -} - -// GetCsiAzureStorageClassName hold a name of CSI volume storage class -func GetCsiAzureStorageClassName(namespace, volumeName string) string { - return fmt.Sprintf(csiStorageClassNameTemplate, namespace, volumeName) // volumeName: --- -} - -func getBlobFuseVolume(namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount) *corev1.VolumeSource { - secretName := defaults.GetBlobFuseCredsSecretName(componentName, volumeMount.Name) - - flexVolumeOptions := make(map[string]string) - flexVolumeOptions["name"] = volumeMount.Name - flexVolumeOptions["container"] = volumeMount.Container - flexVolumeOptions["mountoptions"] = defaultMountOptions - flexVolumeOptions["tmppath"] = fmt.Sprintf(blobFuseVolumeNodeMountPathTemplate, namespace, componentName, environment, radixv1.MountTypeBlob, volumeMount.Name, volumeMount.Container) - - return &corev1.VolumeSource{ - FlexVolume: &corev1.FlexVolumeSource{ - Driver: blobfuseDriver, - Options: flexVolumeOptions, - SecretRef: &corev1.LocalObjectReference{ - Name: secretName, - }, - }, - } -} - -func (deploy *Deployment) createOrUpdateVolumeMountsSecrets(ctx context.Context, namespace, componentName, secretName string, accountName, accountKey []byte) error { - blobfusecredsSecret := corev1.Secret{ - Type: blobfuseDriver, - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Labels: map[string]string{ - kube.RadixAppLabel: deploy.registration.Name, - kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(radixv1.MountTypeBlob), - }, - }, - } - - // Will need to set fake data in order to apply the secret. The user then need to set data to real values - data := make(map[string][]byte) - data[defaults.BlobFuseCredsAccountKeyPart] = accountKey - data[defaults.BlobFuseCredsAccountNamePart] = accountName - - blobfusecredsSecret.Data = data - - _, err := deploy.kubeutil.ApplySecret(ctx, namespace, &blobfusecredsSecret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret - if err != nil { - return err - } - - return nil -} -func (deploy *Deployment) createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, accountName, accountKey []byte) error { - secret := corev1.Secret{ - Type: corev1.SecretTypeOpaque, - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Labels: map[string]string{ - kube.RadixAppLabel: deploy.registration.Name, - kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), - kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, - }, - }, - } - - // Will need to set fake data in order to apply the secret. The user then need to set data to real values - data := make(map[string][]byte) - data[defaults.CsiAzureCredsAccountKeyPart] = accountKey - data[defaults.CsiAzureCredsAccountNamePart] = accountName - - secret.Data = data - - _, err := deploy.kubeutil.ApplySecret(ctx, namespace, &secret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret - if err != nil { - return err - } - - return nil -} - -func (deploy *Deployment) garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Context, component radixv1.RadixCommonDeployComponent, excludeSecretNames []string) error { - secrets, err := deploy.listSecretsForVolumeMounts(ctx, component) - if err != nil { - return err - } - return deploy.GarbageCollectSecrets(ctx, secrets, excludeSecretNames) -} - -func (deploy *Deployment) getCsiAzureStorageClasses(ctx context.Context, namespace, componentName string) (*storagev1.StorageClassList, error) { - return deploy.kubeclient.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzureStorageClass(namespace, componentName), - }) -} - -func (deploy *Deployment) getCsiAzurePersistentVolumeClaims(ctx context.Context, namespace, componentName string) (*corev1.PersistentVolumeClaimList, error) { - return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName), - }) -} - -func (deploy *Deployment) getPersistentVolumesForPvc(ctx context.Context) (*corev1.PersistentVolumeList, error) { - return deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) -} - -func getLabelSelectorForCsiAzureStorageClass(namespace, componentName string) string { - return fmt.Sprintf("%s=%s, %s=%s, %s in (%s, %s, %s, %s)", kube.RadixNamespace, namespace, kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure)) -} - -func getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName string) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure)) -} - -func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentStorage(componentName, radixVolumeMountName string) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s), %s = %s", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure), kube.RadixVolumeMountNameLabel, radixVolumeMountName) -} - -func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appName, namespace, componentName, pvcName, storageClassName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { - requestsVolumeMountSize, err := resource.ParseQuantity(getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount)) - if err != nil { - requestsVolumeMountSize = resource.MustParse("1Mi") - } - volumeAccessMode := getVolumeAccessMode(getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount)) - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvcName, - Namespace: namespace, - Labels: map[string]string{ - kube.RadixAppLabel: appName, - kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), - kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, - }, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{volumeAccessMode}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceStorage: requestsVolumeMountSize}, // it seems correct number is not needed for CSI driver - }, - StorageClassName: &storageClassName, - }, - } - return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) -} - -func populateCsiAzureStorageClass(storageClass *storagev1.StorageClass, appName string, volumeRootMount string, namespace string, componentName string, storageClassName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string) error { - reclaimPolicy := corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimPolicy. PersistentVolumeReclaimPolicy deletes volume on unmount. - bindingMode := getBindingMode(getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount)) - storageClass.ObjectMeta.Name = storageClassName - storageClass.ObjectMeta.Labels = getCsiAzureStorageClassLabels(appName, namespace, componentName, radixVolumeMount) - storageClass.Provisioner = provisioner - storageClass.Parameters = getCsiAzureStorageClassParameters(secretName, namespace, radixVolumeMount) - mountOptions, err := getCsiAzureStorageClassMountOptions(volumeRootMount, namespace, componentName, radixVolumeMount) - if err != nil { - return err - } - storageClass.MountOptions = mountOptions - storageClass.ReclaimPolicy = &reclaimPolicy - storageClass.VolumeBindingMode = &bindingMode - return nil -} - -func getBindingMode(bindingModeValue string) storagev1.VolumeBindingMode { - if strings.EqualFold(strings.ToLower(bindingModeValue), strings.ToLower(string(storagev1.VolumeBindingWaitForFirstConsumer))) { - return storagev1.VolumeBindingWaitForFirstConsumer - } - return storagev1.VolumeBindingImmediate -} - -func getCsiAzureStorageClassLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { - return map[string]string{ - kube.RadixAppLabel: appName, - kube.RadixNamespace: namespace, - kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), - kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, - } -} - -func getCsiAzureStorageClassParameters(secretName string, namespace string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { - parameters := map[string]string{ - csiStorageClassProvisionerSecretNameParameter: secretName, - csiStorageClassProvisionerSecretNamespaceParameter: namespace, - csiStorageClassNodeStageSecretNameParameter: secretName, - csiStorageClassNodeStageSecretNamespaceParameter: namespace, - } - skuName := getRadixBlobFuse2VolumeMountSkuName(radixVolumeMount) - if len(skuName) > 0 { - parameters[csiAzureStorageClassSkuNameParameter] = skuName - } - switch GetCsiAzureVolumeMountType(radixVolumeMount) { - case radixv1.MountTypeBlobFuse2FuseCsiAzure: - parameters[csiStorageClassContainerNameParameter] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse - case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: - parameters[csiStorageClassContainerNameParameter] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse2 - case radixv1.MountTypeBlobFuse2NfsCsiAzure: - parameters[csiStorageClassContainerNameParameter] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterNfs - case radixv1.MountTypeAzureFileCsiAzure: - parameters[csiStorageClassShareNameParameter] = getRadixAzureFileVolumeMountShareName(radixVolumeMount) - } - return parameters -} - -func getCsiAzureStorageClassMountOptions(volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { - csiVolumeTypeId, err := getCsiRadixVolumeTypeIdForName(radixVolumeMount) - if err != nil { - return nil, err - } - tmpPath := fmt.Sprintf(csiVolumeNodeMountPathTemplate, volumeRootMount, namespace, csiVolumeTypeId, componentName, radixVolumeMount.Name, GetRadixVolumeMountStorage(radixVolumeMount)) - return getCsiAzureStorageClassMountOptionsForAzureBlob(tmpPath, radixVolumeMount) -} - -func getCsiAzureStorageClassMountOptionsForAzureBlob(tmpPath string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { - mountOptions := []string{ - // fmt.Sprintf("--%s=%s", csiStorageClassTmpPathMountOption, tmpPath),//TODO fix this path to be able to mount on external mount - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - } - gid := getRadixBlobFuse2VolumeMountGid(radixVolumeMount) - if len(gid) > 0 { - mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiStorageClassGidMountOption, gid)) - } else { - uid := getRadixBlobFuse2VolumeMountUid(radixVolumeMount) - if len(uid) > 0 { - mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiStorageClassUidMountOption, uid)) - } - } - if getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount) == string(corev1.ReadOnlyMany) { - mountOptions = append(mountOptions, "-o ro") - } - if radixVolumeMount.BlobFuse2 != nil { - mountOptions = append(mountOptions, getStreamingMountOptions(radixVolumeMount.BlobFuse2.Streaming)...) - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassUseAdlsMountOption, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) - } - return mountOptions, nil -} - -func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []string { - var mountOptions []string - if streaming != nil && streaming.Enabled != nil && !*streaming.Enabled { - return nil - } - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%t", csiStorageClassStreamingEnabledMountOption, true)) - if streaming == nil { - return mountOptions - } - if streaming.StreamCache != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingCacheMountOption, *streaming.StreamCache)) - } - if streaming.BlockSize != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingBlockSizeMountOption, *streaming.BlockSize)) - } - if streaming.BufferSize != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingBufferSizeMountOption, *streaming.BufferSize)) - } - if streaming.MaxBuffers != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingMaxBuffersMountOption, *streaming.MaxBuffers)) - } - if streaming.MaxBlocksPerFile != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingMaxBlocksPerFileMountOption, *streaming.MaxBlocksPerFile)) - } - return mountOptions -} - -func getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.AccessMode - } - return radixVolumeMount.AccessMode -} - -func getRadixBlobFuse2VolumeMountUid(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.UID - } - return radixVolumeMount.UID -} - -func getRadixBlobFuse2VolumeMountGid(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.GID - } - return radixVolumeMount.GID -} - -func getRadixBlobFuse2VolumeMountSkuName(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.SkuName - } - return radixVolumeMount.SkuName -} - -func getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.Container - } - return radixVolumeMount.Storage -} - -func getRadixAzureFileVolumeMountShareName(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.AzureFile != nil { - return radixVolumeMount.AzureFile.Share - } - return radixVolumeMount.Storage -} - -func getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.RequestsStorage - } - return radixVolumeMount.RequestsStorage -} - -func getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.BindingMode - } - return radixVolumeMount.BindingMode -} - -func (deploy *Deployment) deletePersistentVolumeClaim(ctx context.Context, namespace, pvcName string) error { - if len(namespace) > 0 && len(pvcName) > 0 { - return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) - } - log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) - return nil -} - -func (deploy *Deployment) deleteCsiAzureStorageClasses(ctx context.Context, storageClassName string) error { - if len(storageClassName) > 0 { - return deploy.kubeclient.StorageV1().StorageClasses().Delete(ctx, storageClassName, metav1.DeleteOptions{}) - } - log.Ctx(ctx).Debug().Msg("Skip deleting StorageClass - name is empty") - return nil -} - -func (deploy *Deployment) deletePersistentVolume(ctx context.Context, pvName string) error { - if len(pvName) > 0 { - return deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) - } - log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") - return nil -} - -// GetRadixVolumeMountStorage get RadixVolumeMount storage property, depend on volume type -func GetRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.Type == radixv1.MountTypeBlob { - return radixVolumeMount.Container // Outdated - } - blobFuse2VolumeMountContainer := getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - if len(blobFuse2VolumeMountContainer) != 0 { - return blobFuse2VolumeMountContainer - } - azureFileVolumeMountShare := getRadixAzureFileVolumeMountShareName(radixVolumeMount) - if len(azureFileVolumeMountShare) != 0 { - return azureFileVolumeMountShare - } - return radixVolumeMount.Storage -} - -func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx context.Context, excludePvcNames map[string]any) error { - pvList, err := deploy.getPersistentVolumesForPvc(ctx) - if err != nil { - return err - } - for _, pv := range pvList.Items { - if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != persistentVolumeClaimKind || - !knownCSIDriver(pv.Spec.CSI) || - pv.Status.Phase != corev1.VolumeReleased { - continue - } - if _, ok := excludePvcNames[pv.Spec.ClaimRef.Name]; ok { - continue - } - log.Ctx(ctx).Info().Msgf("Delete orphaned Csi Azure PersistantVolume %s of PersistantVolumeClaim %s", pv.Name, pv.Spec.ClaimRef.Name) - err := deploy.deletePersistentVolume(ctx, pv.Name) - if err != nil { - return err - } - } - return nil -} - -func knownCSIDriver(csiPersistentVolumeSource *corev1.CSIPersistentVolumeSource) bool { - if csiPersistentVolumeSource == nil { - return false - } - _, ok := csiVolumeProvisioners[csiPersistentVolumeSource.Driver] - return ok -} - -// createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - StorageClasses, PersistentVolumeClaims, PersistentVolume -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment) error { - namespace := deploy.radixDeployment.GetNamespace() - appName := deploy.radixDeployment.Spec.AppName - componentName := desiredDeployment.ObjectMeta.Name - volumeRootMount := "/tmp" // TODO: add to environment variable, so this volume can be mounted to external disk - scList, err := deploy.getCsiAzureStorageClasses(ctx, namespace, componentName) - if err != nil { - return err - } - pvcList, err := deploy.getCsiAzurePersistentVolumeClaims(ctx, namespace, componentName) - if err != nil { - return err - } - - scMap := utils.GetStorageClassMap(&scList.Items) - pvcMap := utils.GetPersistentVolumeClaimMap(&pvcList.Items) - radixVolumeMountMap := deploy.getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName) - var actualStorageClassNames []string - actualPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx, namespace) - if err != nil { - return err - } - for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { - if volume.PersistentVolumeClaim == nil { - continue - } - radixVolumeMount, existsRadixVolumeMount := radixVolumeMountMap[volume.Name] - if !existsRadixVolumeMount { - return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) - } - storageClass, storageClassIsCreated, err := deploy.getOrCreateCsiAzureVolumeMountStorageClass(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap) - if err != nil { - return err - } - actualStorageClassNames = append(actualStorageClassNames, storageClass.Name) - pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, storageClass, storageClassIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, pvcMap) - if err != nil { - return err - } - volume.PersistentVolumeClaim.ClaimName = pvc.Name - actualPvcNames[pvc.Name] = struct{}{} - } - err = deploy.garbageCollectCsiAzureStorageClasses(ctx, scList, actualStorageClassNames) - if err != nil { - return err - } - err = deploy.garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, namespace, pvcList, actualPvcNames) - if err != nil { - return err - } - err = deploy.garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, actualPvcNames) - if err != nil && !k8serrors.IsNotFound(err) { - return err - } - return nil -} - -func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, namespace string) (map[string]any, error) { - pvcNames := make(map[string]any) - deploymentList, err := deploy.kubeclient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, deployment := range deploymentList.Items { - addUsedPersistenceVolumeClaimsFrom(deployment.Spec.Template, pvcNames) - } - jobsList, err := deploy.kubeclient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, job := range jobsList.Items { - addUsedPersistenceVolumeClaimsFrom(job.Spec.Template, pvcNames) - } - return pvcNames, nil -} - -func addUsedPersistenceVolumeClaimsFrom(podTemplate corev1.PodTemplateSpec, pvcMap map[string]any) { - for _, volume := range podTemplate.Spec.Volumes { - if volume.PersistentVolumeClaim != nil && len(volume.PersistentVolumeClaim.ClaimName) > 0 { - pvcMap[volume.PersistentVolumeClaim.ClaimName] = struct{}{} - } - } -} - -func (deploy *Deployment) garbageCollectCsiAzureStorageClasses(ctx context.Context, scList *storagev1.StorageClassList, excludeStorageClassName []string) error { - for _, storageClass := range scList.Items { - if commonUtils.ContainsString(excludeStorageClassName, storageClass.Name) { - continue - } - log.Ctx(ctx).Debug().Msgf("Delete Csi Azure StorageClass %s", storageClass.Name) - err := deploy.deleteCsiAzureStorageClasses(ctx, storageClass.Name) - if err != nil { - return err - } - } - return nil -} - -func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, namespace string, pvcList *corev1.PersistentVolumeClaimList, excludePvcNames map[string]any) error { - for _, pvc := range pvcList.Items { - if _, ok := excludePvcNames[pvc.Name]; ok { - continue - } - pvName := pvc.Spec.VolumeName - log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolumeClaim %s in namespace %s", pvc.Name, namespace) - err := deploy.deletePersistentVolumeClaim(ctx, namespace, pvc.Name) - if err != nil { - return err - } - log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pvName, namespace) - err = deploy.deletePersistentVolume(ctx, pvName) - if err != nil { - return err - } - } - return nil -} - -func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Context, storageClass *storagev1.StorageClass, requiredNewPvc bool, appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumeClaimName string, pvcMap map[string]*corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) { - if pvc, ok := pvcMap[persistentVolumeClaimName]; ok { - if pvc.Spec.StorageClassName == nil || len(*pvc.Spec.StorageClassName) == 0 { - return pvc, nil - } - if !requiredNewPvc && strings.EqualFold(*pvc.Spec.StorageClassName, storageClass.Name) { - return pvc, nil - } - - log.Ctx(ctx).Debug().Msgf("Delete in garbage-collect an old PersistentVolumeClaim %s in namespace %s: changed StorageClass name to %s", pvc.Name, namespace, storageClass.Name) - } - persistentVolumeClaimName, err := createCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) - if err != nil { - return nil, err - } - log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for StorageClass %s", persistentVolumeClaimName, namespace, storageClass.Name) - return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, storageClass.Name, radixVolumeMount) -} - -// getOrCreateCsiAzureVolumeMountStorageClass returns creates or existing StorageClass, storageClassIsCreated=true, if created; error, if any -func (deploy *Deployment) getOrCreateCsiAzureVolumeMountStorageClass(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, scMap map[string]*storagev1.StorageClass) (*storagev1.StorageClass, bool, error) { - var volumeMountProvisioner, foundProvisioner = getStorageClassProvisionerByVolumeMountType(radixVolumeMount) - if !foundProvisioner { - return nil, false, fmt.Errorf("not found Storage Class provisioner for volume mount type %s", string(GetCsiAzureVolumeMountType(radixVolumeMount))) - } - storageClassName := GetCsiAzureStorageClassName(namespace, volumeName) - csiVolumeSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) - if existingStorageClass, exists := scMap[storageClassName]; exists { - desiredStorageClass := existingStorageClass.DeepCopy() - err := populateCsiAzureStorageClass(desiredStorageClass, appName, volumeRootMount, namespace, componentName, storageClassName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) - if err != nil { - return nil, false, err - } - if equal, err := utils.EqualStorageClasses(existingStorageClass, desiredStorageClass); equal || err != nil { - return existingStorageClass, false, err - } - - log.Ctx(ctx).Info().Msgf("Delete StorageClass %s in namespace %s", existingStorageClass.Name, namespace) - err = deploy.deleteCsiAzureStorageClasses(ctx, existingStorageClass.Name) - if err != nil { - return nil, false, err - } - } - - log.Ctx(ctx).Debug().Msgf("Create StorageClass %s in namespace %s", storageClassName, namespace) - storageClass := &storagev1.StorageClass{} - err := populateCsiAzureStorageClass(storageClass, appName, volumeRootMount, namespace, componentName, storageClassName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) - if err != nil { - return nil, false, err - } - desiredStorageClass, err := deploy.kubeclient.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{}) - return desiredStorageClass, true, err -} - -func (deploy *Deployment) getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName string) map[string]*radixv1.RadixVolumeMount { - volumeMountMap := make(map[string]*radixv1.RadixVolumeMount) - for _, component := range deploy.radixDeployment.Spec.Components { - if findCsiAzureVolumeForComponent(volumeMountMap, component.VolumeMounts, componentName, &component) { - break - } - } - for _, component := range deploy.radixDeployment.Spec.Jobs { - if findCsiAzureVolumeForComponent(volumeMountMap, component.VolumeMounts, componentName, &component) { - break - } - } - return volumeMountMap -} - -func findCsiAzureVolumeForComponent(volumeMountMap map[string]*radixv1.RadixVolumeMount, volumeMounts []radixv1.RadixVolumeMount, componentName string, component radixv1.RadixCommonDeployComponent) bool { - if !strings.EqualFold(componentName, component.GetName()) { - return false - } - for _, radixVolumeMount := range volumeMounts { - if radixVolumeMount.BlobFuse2 == nil && radixVolumeMount.AzureFile == nil && !isKnownCsiAzureVolumeMount(string(GetCsiAzureVolumeMountType(&radixVolumeMount))) { - continue - } - radixVolumeMount := radixVolumeMount - volumeMountName, err := getCsiAzureVolumeMountName(&radixVolumeMount, componentName) - if err != nil { - return false - } - volumeMountMap[volumeMountName] = &radixVolumeMount - } - return true -} - -func getVolumeAccessMode(modeValue string) corev1.PersistentVolumeAccessMode { - switch strings.ToLower(modeValue) { - case strings.ToLower(string(corev1.ReadWriteOnce)): - return corev1.ReadWriteOnce - case strings.ToLower(string(corev1.ReadWriteMany)): - return corev1.ReadWriteMany - case strings.ToLower(string(corev1.ReadWriteOncePod)): - return corev1.ReadWriteOncePod - } - return corev1.ReadOnlyMany // default access mode -} - -func sortPvcsByCreatedTimestampDesc(persistentVolumeClaims []corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { - sort.SliceStable(persistentVolumeClaims, func(i, j int) bool { - return (persistentVolumeClaims)[j].ObjectMeta.CreationTimestamp.Before(&(persistentVolumeClaims)[i].ObjectMeta.CreationTimestamp) - }) - return persistentVolumeClaims -} - -func trimVolumeNameToValidLength(volumeName string) string { - const randSize = 5 - if len(volumeName) <= volumeNameMaxLength { - return volumeName - } - - randString := strings.ToLower(commonUtils.RandStringStrSeed(randSize, volumeName)) - return fmt.Sprintf("%s-%s", volumeName[:63-randSize-1], randString) -} diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go deleted file mode 100644 index a28d5c23a..000000000 --- a/pkg/apis/deployment/volumemount_test.go +++ /dev/null @@ -1,1751 +0,0 @@ -package deployment - -import ( - "context" - "fmt" - "strings" - "testing" - - "github.com/equinor/radix-common/utils/pointers" - "github.com/equinor/radix-operator/pkg/apis/config" - "github.com/equinor/radix-operator/pkg/apis/kube" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - "github.com/equinor/radix-operator/pkg/apis/utils" - radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" - radix "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" - kedav2 "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned" - kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" - prometheusclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" - prometheusfake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - kubefake "k8s.io/client-go/kubernetes/fake" - secretProviderClient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" - secretproviderfake "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/fake" -) - -type VolumeMountTestSuite struct { - suite.Suite - radixCommonDeployComponentFactories []v1.RadixCommonDeployComponentFactory -} - -type TestEnv struct { - kubeclient kubernetes.Interface - radixclient radixclient.Interface - secretproviderclient secretProviderClient.Interface - prometheusclient prometheusclient.Interface - kubeUtil *kube.Kube - kedaClient kedav2.Interface -} - -type volumeMountTestScenario struct { - name string - radixVolumeMount v1.RadixVolumeMount - expectedVolumeName string - expectedVolumeNameIsPrefix bool - expectedError string - expectedPvcNamePrefix string -} - -type deploymentVolumesTestScenario struct { - name string - props expectedPvcScProperties - radixVolumeMounts []v1.RadixVolumeMount - volumes []corev1.Volume - existingStorageClassesBeforeTestRun []storagev1.StorageClass - existingPvcsBeforeTestRun []corev1.PersistentVolumeClaim - existingStorageClassesAfterTestRun []storagev1.StorageClass - existingPvcsAfterTestRun []corev1.PersistentVolumeClaim -} - -type pvcTestScenario struct { - volumeMountTestScenario - pvc corev1.PersistentVolumeClaim -} - -func TestVolumeMountTestSuite(t *testing.T) { - suite.Run(t, new(VolumeMountTestSuite)) -} - -func (suite *VolumeMountTestSuite) SetupSuite() { - suite.radixCommonDeployComponentFactories = []v1.RadixCommonDeployComponentFactory{ - v1.RadixDeployComponentFactory{}, - v1.RadixDeployJobComponentFactory{}, - } -} - -func getTestEnv() TestEnv { - testEnv := TestEnv{ - kubeclient: kubefake.NewSimpleClientset(), - radixclient: radix.NewSimpleClientset(), - kedaClient: kedafake.NewSimpleClientset(), - secretproviderclient: secretproviderfake.NewSimpleClientset(), - prometheusclient: prometheusfake.NewSimpleClientset(), - } - kubeUtil, _ := kube.New(testEnv.kubeclient, testEnv.radixclient, testEnv.kedaClient, testEnv.secretproviderclient) - testEnv.kubeUtil = kubeUtil - return testEnv -} - -func getDeployment(testEnv TestEnv) *Deployment { - return &Deployment{ - kubeclient: testEnv.kubeclient, - radixclient: testEnv.radixclient, - kubeutil: testEnv.kubeUtil, - prometheusperatorclient: testEnv.prometheusclient, - config: &config.Config{}, - } -} - -func (suite *VolumeMountTestSuite) Test_NoVolumeMounts() { - suite.T().Run("app", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - BuildComponent() - - volumeMounts, _ := GetRadixDeployComponentVolumeMounts(component, "") - assert.Equal(t, 0, len(volumeMounts)) - } - }) -} - -func (suite *VolumeMountTestSuite) Test_ValidFileCsiAzureVolumeMounts() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "csi-az-file-app-volume1-storageName1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "csi-az-file-app-volume2-storageName2", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath3"}, - expectedVolumeName: "csi-az-file-app-volume-with-long-name-storageName-with-lo-", - expectedVolumeNameIsPrefix: true, - }, - } - suite.T().Run("One File CSI Azure volume mount ", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - if len(volumeMounts) > 0 { - mount := volumeMounts[0] - assert.Less(t, len(mount.Name), 64) - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - } - } - }) - suite.T().Run("Multiple File CSI Azure volume mount", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - builder := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount, scenarios[2].radixVolumeMount) - - component := builder.BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 3, len(volumeMounts)) - for idx, testCase := range scenarios { - if len(volumeMounts) > 0 { - assert.Less(t, len(volumeMounts[idx].Name), 64) - if testCase.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volumeMounts[idx].Name, testCase.expectedVolumeName)) - } else { - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - } - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - } - } - }) -} - -func (suite *VolumeMountTestSuite) Test_ValidBlobCsiAzureVolumeMounts() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "csi-az-blob-app-volume1-storageName1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "csi-az-blob-app-volume2-storageName2", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath2"}, - expectedVolumeName: "csi-az-blob-app-volume-with-long-name-storageName-with-lo-", - expectedVolumeNameIsPrefix: true, - }, - } - suite.T().Run("One Blob CSI Azure volume mount ", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory).WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - if len(volumeMounts) > 0 { - mount := volumeMounts[0] - assert.Less(t, len(volumeMounts[0].Name), 64) - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - } - } - }) - suite.T().Run("Multiple Blob CSI Azure volume mount ", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount, scenarios[2].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Equal(t, 3, len(volumeMounts)) - assert.Nil(t, err) - for idx, testCase := range scenarios { - if len(volumeMounts) > 0 { - assert.Less(t, len(volumeMounts[idx].Name), 64) - if testCase.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volumeMounts[idx].Name, testCase.expectedVolumeName)) - } else { - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - } - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - } - } - }) -} - -func (suite *VolumeMountTestSuite) Test_FailBlobCsiAzureVolumeMounts() { - scenarios := []volumeMountTestScenario{ - { - name: "Missed volume mount name", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: "storageName1", Path: "TestPath1"}, - expectedError: "name is empty for volume mount in the component app", - }, - { - name: "Missed volume mount storage", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Path: "TestPath1"}, - expectedError: "storage is empty for volume mount volume1 in the component app", - }, - { - name: "Missed volume mount path", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storageName1"}, - expectedError: "path is empty for volume mount volume1 in the component app", - }, - } - suite.T().Run("Failing Blob CSI Azure volume mount", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - for _, testCase := range scenarios { - t.Logf("Test case %s for component %s", testCase.name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(testCase.radixVolumeMount). - BuildComponent() - - _, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.NotNil(t, err) - assert.Equal(t, testCase.expectedError, err.Error()) - } - } - }) -} - -// Blobfuse support has been deprecated, this test to be deleted, when Blobfuse logic is deleted -func (suite *VolumeMountTestSuite) Test_BlobfuseAzureVolumeMounts() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlob, Name: "volume1", Container: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "blobfuse-app-volume1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlob, Name: "volume2", Container: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "blobfuse-app-volume2", - }, - } - suite.T().Run("One Blobfuse Azure volume mount", func(t *testing.T) { - t.Parallel() - component := utils.NewDeployComponentBuilder().WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(&component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - mount := volumeMounts[0] - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - }) - suite.T().Run("Multiple Blobfuse Azure volume mount", func(t *testing.T) { - t.Parallel() - component := utils.NewDeployComponentBuilder().WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(&component, "") - assert.Nil(t, err) - for idx, testCase := range scenarios { - assert.Equal(t, 2, len(volumeMounts)) - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - }) -} - -func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { - namespace := "some-namespace" - environment := "some-env" - componentName := "some-component" - suite.T().Run("No volumes in component", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts().BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 0) - }) - scenarios := []volumeMountTestScenario{ - { - name: "Blob CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-volume1-storage1", - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume1-storage1", - }, - { - name: "File CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume1-storage1", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-volume1-storage1", - }, - { - name: "Blob CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-volume-with-long-name-storageN-", - expectedVolumeNameIsPrefix: true, - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume-with-long-name-storageN-", - }, - { - name: "File CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume-with-long-name-storageN-", - expectedVolumeNameIsPrefix: true, - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-volume-with-long-name-storageN-", - }, - } - blobFuseScenario := volumeMountTestScenario{ - name: "Blob Azure FlexVolume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlob, Name: "volume1", Container: "storage1", Path: "path1"}, - expectedVolumeName: "blobfuse-some-component-volume1", - } - suite.T().Run("CSI Azure volumes", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - for _, scenario := range scenarios { - t.Logf("Scenario %s", scenario.name) - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 1) - volume := volumes[0] - if scenario.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volume.Name, scenario.expectedVolumeName)) - } else { - assert.Equal(t, scenario.expectedVolumeName, volume.Name) - } - assert.Less(t, len(volume.Name), 64) - assert.NotNil(t, volume.PersistentVolumeClaim) - assert.Contains(t, volume.PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) - } - }) - suite.T().Run("Blobfuse-flex volume", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(blobFuseScenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 1) - volume := volumes[0] - assert.Equal(t, blobFuseScenario.expectedVolumeName, volume.Name) - assert.Nil(t, volume.PersistentVolumeClaim) - assert.NotNil(t, volume.FlexVolume) - assert.Equal(t, "azure/blobfuse", volume.FlexVolume.Driver) - assert.Equal(t, "volume1", volume.FlexVolume.Options["name"]) - assert.Equal(t, "storage1", volume.FlexVolume.Options["container"]) - assert.Equal(t, "--file-cache-timeout-in-seconds=120", volume.FlexVolume.Options["mountoptions"]) - assert.Equal(t, "/tmp/some-namespace/some-component/some-env/blob/volume1/storage1", volume.FlexVolume.Options["tmppath"]) - }) - suite.T().Run("CSI Azure and Blobfuse-flex volumes", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - for _, scenario := range append(scenarios, blobFuseScenario) { - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 1) - volume := volumes[0] - if scenario.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volume.Name, scenario.expectedVolumeName)) - } else { - assert.Equal(t, scenario.expectedVolumeName, volume.Name) - } - assert.Less(t, len(volume.Name), 64) - assert.Equal(t, len(scenario.expectedPvcNamePrefix) > 0, volume.PersistentVolumeClaim != nil) - } - }) - suite.T().Run("Unsupported volume type", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - mounts := []v1.RadixVolumeMount{ - {Type: "unsupported-type", Name: "volume1", Container: "storage1", Path: "path1"}, - } - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(mounts...).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Len(t, volumes, 0) - assert.NotNil(t, err) - assert.Equal(t, "unsupported volume type unsupported-type", err.Error()) - }) -} - -func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { - namespace := "some-namespace" - environment := "some-env" - componentName := "some-component" - scenarios := []pvcTestScenario{ - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "Blob CSI Azure volume, PVS phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-volume1-storage1", - expectedPvcNamePrefix: "existing-blob-pvc-name1", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-blob-pvc-name1" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume1" - pvc.Status.Phase = corev1.ClaimBound - }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "Blob CSI Azure volume, PVS phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storage2", Path: "path2", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-volume2-storage2", - expectedPvcNamePrefix: "existing-blob-pvc-name2", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-blob-pvc-name2" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume2" - pvc.Status.Phase = corev1.ClaimPending - }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, PVS phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume3", Storage: "storage3", Path: "path3", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume3-storage3", - expectedPvcNamePrefix: "existing-file-pvc-name1", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-file-pvc-name1" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume3" - pvc.Status.Phase = corev1.ClaimBound - }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, PVS phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume4", Storage: "storage4", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume4-storage4", - expectedPvcNamePrefix: "existing-file-pvc-name2", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-file-pvc-name2" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume4" - pvc.Status.Phase = corev1.ClaimBound - }), - }, - } - - suite.T().Run("CSI Azure volumes with existing PVC", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - for _, scenario := range scenarios { - t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) - _, _ = testEnv.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) - - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 1) - assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) - assert.NotNil(t, volumes[0].PersistentVolumeClaim) - assert.Equal(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.pvc.Name) - } - }) - - suite.T().Run("CSI Azure volumes with no existing PVC", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - for _, scenario := range scenarios { - t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) - - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 1) - assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) - assert.NotNil(t, volumes[0].PersistentVolumeClaim) - assert.NotEqual(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.pvc.Name) - assert.NotContains(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) - } - }) -} - -func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { - appName := "any-app" - environment := "some-env" - namespace := fmt.Sprintf("%s-%s", appName, environment) - componentName := "some-component" - scenarios := []pvcTestScenario{ - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "Blob CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-blob-volume1-storage1", - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume1-storage1", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "Blob CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume2-storage2", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume1", Storage: "storage3", Path: "path3", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume1-storage3", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume1-storage3", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume2", Storage: "storage4", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume2-storage4", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume2-storage4", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), - }, - } - - suite.T().Run("No volumes", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case for component %s", factory.GetTargetType()) - - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) - deployComponent := deployment.radixDeployment.Spec.Components[0] - - volumes, err := deployment.GetVolumesForComponent(context.Background(), &deployComponent) - - assert.Nil(t, err) - assert.Len(t, volumes, 0) - } - }) - suite.T().Run("Exists volume", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - for _, factory := range suite.radixCommonDeployComponentFactories { - for _, scenario := range scenarios { - t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) - - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) - deployComponent := deployment.radixDeployment.Spec.Components[0] - - volumes, err := deployment.GetVolumesForComponent(context.Background(), &deployComponent) - - assert.Nil(t, err) - assert.Len(t, volumes, 1) - assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) - assert.NotNil(t, volumes[0].PersistentVolumeClaim) - assert.Contains(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) - } - } - }) -} - -type expectedPvcScProperties struct { - appName string - environment string - componentName string - radixVolumeMountName string - radixStorageName string - pvcName string - storageClassName string - radixVolumeMountType v1.MountType - requestsVolumeMountSize string - volumeAccessMode corev1.PersistentVolumeAccessMode - volumeName string - scProvisioner string - scSecretName string - scTmpPath string - scGid string - scUid string - namespace string -} - -func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { - appName := "any-app" - environment := "some-env" - componentName := "some-component" - scenarios := []volumeMountTestScenario{ - { - name: "Blob CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-blob-volume1-storage1", - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume1-storage1", - }, - { - name: "Blob CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume2-storage2", - }, - { - name: "File CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume1", Storage: "storage3", Path: "path3", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume1-storage3", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume1-storage3", - }, - { - name: "File CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume2", Storage: "storage4", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume2-storage4", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume2-storage4", - }, - { - name: "Blob CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume-with-long-name", Storage: "storage-with-long-name", Path: "path2", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-blob-volume-with-long-name-sto-", - expectedVolumeNameIsPrefix: true, - expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume-with-long-name-", - }, - { - name: "File CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume-with-long-name", Storage: "storage-with-long-name", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume-with-long-name-sto-", - expectedVolumeNameIsPrefix: true, - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume-with-long-name-", - }, - } - - suite.T().Run("No volumes", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case for component %s", factory.GetTargetType()) - - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) - deployComponent := deployment.radixDeployment.Spec.Components[0] - - volumes, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") - - assert.Nil(t, err) - assert.Len(t, volumes, 0) - } - }) - suite.T().Run("Exists volume", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - for _, factory := range suite.radixCommonDeployComponentFactories { - for _, scenario := range scenarios { - t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) - - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) - deployComponent := deployment.radixDeployment.Spec.Components[0] - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") - - assert.Nil(t, err) - assert.Len(t, volumeMounts, 1) - if scenario.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volumeMounts[0].Name, scenario.expectedVolumeName)) - } else { - assert.Equal(t, scenario.expectedVolumeName, volumeMounts[0].Name) - } - assert.Less(t, len(volumeMounts[0].Name), 64) - assert.Equal(t, scenario.radixVolumeMount.Path, volumeMounts[0].MountPath) - } - } - }) -} - -func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureBlobVolumeResources() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "csi-az-file-app-volume1-storageName1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "csi-az-file-app-volume2-storageName2", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath3"}, - expectedVolumeName: "csi-az-file-app-volume-with-long-name-storageName-with-lo-", - expectedVolumeNameIsPrefix: true, - }, - } - suite.T().Run("One File CSI Azure volume mount ", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - if len(volumeMounts) > 0 { - mount := volumeMounts[0] - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - } - } - }) - suite.T().Run("Multiple File CSI Azure volume mount", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount, scenarios[2].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 3, len(volumeMounts)) - for idx, testCase := range scenarios { - if len(volumeMounts) > 0 { - assert.Less(t, len(volumeMounts[idx].Name), 64) - if testCase.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volumeMounts[idx].Name, testCase.expectedVolumeName)) - } else { - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - } - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - } - } - }) -} - -func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { - appName := "any-app" - environment := "some-env" - componentName := "some-component" - - var scenarios []deploymentVolumesTestScenario - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewStorageClassName string - expectedNewScTmpPath string - } - getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.StorageClassName = utils.StringPtr(scenarioProps.expectedNewStorageClassName) - }), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), - }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.ObjectMeta.Name = scenarioProps.expectedNewStorageClassName - sc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // setStorageClassMountOption(sc, "--tmp-path", scenarioProps.expectedNewScTmpPath) //TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes - setStorageClassStorageParameter(props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, sc) - sc.Parameters[csiStorageClassProvisionerSecretNameParameter] = scenarioProps.expectedNewSecretName - sc.Parameters[csiStorageClassNodeStageSecretNameParameter] = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewStorageClassName: "sc-any-app-some-env-csi-az-blob-some-component-volume101-storage101", - expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume101/storage101", - }), - getScenario(getPropsCsiFileVolume2Storage2(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-file-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-file-some-component-volume101-storage101-12345", - expectedNewStorageClassName: "sc-any-app-some-env-csi-az-file-some-component-volume101-storage101", - expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-file/some-component/volume101/storage101", - }), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - storageClassForAnotherNamespace := createRandomStorageClass(props, utils.RandString(10), utils.RandString(10)) - storageClassForAnotherComponent := createRandomStorageClass(props, props.namespace, utils.RandString(10)) - pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and StorageClasses", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), - storageClassForAnotherNamespace, - storageClassForAnotherComponent, - }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), - storageClassForAnotherNamespace, - storageClassForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set readonly volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), - }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = append(sc.MountOptions, "-o ro") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - }), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), - }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), - }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o gid=1000", - "--streaming=true", - "--use-adls=false", - } - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o gid=1000", - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", - "--use-adls=false", - } - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has disabled streaming", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - Enabled: pointers.Ptr(false), - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o gid=1000", - "--use-adls=false", - } - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - suite.T().Run("CSI Azure volume PVCs and StorageClasses", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - for _, scenario := range scenarios { - t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - deployment.radixDeployment = buildRd(appName, environment, componentName, scenario.radixVolumeMounts) - putExistingDeploymentVolumesScenarioDataToFakeCluster(&scenario, deployment) - desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) - - // action - err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment) - assert.Nil(t, err) - - existingPvcs, existingScs, err := getExistingPvcsAndStorageClassesFromFakeCluster(deployment) - assert.Nil(t, err) - equalPvcLists, err := utils.EqualPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs, true) - assert.Nil(t, err) - assert.True(t, equalPvcLists) - equalStorageClassLists, err := utils.EqualStorageClassLists(&scenario.existingStorageClassesAfterTestRun, &existingScs) - assert.Nil(t, err) - assert.True(t, equalStorageClassLists) - } - } - }) -} - -func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources() { - appName := "app" - namespace := "some-namespace" - environment := "some-env" - componentName1, componentNameLong := "component1", "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" - type expectedVolumeProps struct { - expectedVolumeNamePrefix string - expectedVolumeMountPath string - expectedNodePublishSecretRefName string - expectedVolumeAttributePrefixes map[string]string - } - scenarios := []struct { - name string - deployComponentBuilders []utils.DeployCommonComponentBuilder - componentName string - azureKeyVaults []v1.RadixAzureKeyVault - expectedVolumeProps []expectedVolumeProps - radixVolumeMounts []v1.RadixVolumeMount - }{ - { - name: "No Azure Key volumes as no RadixAzureKeyVault-s", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{}, - expectedVolumeProps: []expectedVolumeProps{}, - }, - { - name: "No Azure Key volumes as no secret names in secret object", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, - }, - { - name: "One Azure Key volume for one secret objects secret name", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{{ - Name: "kv1", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }}, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv1-", - }, - }, - }, - }, - { - name: "Multiple Azure Key volumes for each RadixAzureKeyVault", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{ - { - Name: "kv1", - Path: utils.StringPtr("/mnt/customPath"), - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }, - { - Name: "kv2", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, - }, - }, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - expectedVolumeMountPath: "/mnt/customPath", - expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv1-", - }, - }, - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", - expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv2-", - }, - }, - }, - }, - { - name: "Volume name should be trimmed when exceeding 63 chars", - componentName: componentNameLong, - azureKeyVaults: []v1.RadixAzureKeyVault{{ - Name: "kv1", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }}, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", - }, - }, - }, - }, - } - suite.T().Run("CSI Azure Key vault volumes", func(t *testing.T) { - t.Parallel() - for _, scenario := range scenarios { - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - deployment.radixDeployment = buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { - var builders []utils.DeployComponentBuilder - builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(scenario.componentName). - WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) - return builders - }) - radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) - for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.createAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) - if err != nil { - t.Log(err.Error()) - } else { - t.Logf("created secret provider class %s", spc.Name) - } - } - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, radixDeployComponent, deployment.radixDeployment.GetName()) - assert.Nil(t, err) - assert.Len(t, volumes, len(scenario.expectedVolumeProps)) - if len(scenario.expectedVolumeProps) == 0 { - continue - } - - for i := 0; i < len(volumes); i++ { - volume := volumes[i] - assert.Less(t, len(volume.Name), 64, "volume name is too long") - assert.NotNil(t, volume.CSI) - assert.NotNil(t, volume.CSI.VolumeAttributes) - assert.NotNil(t, volume.CSI.NodePublishSecretRef) - assert.Equal(t, "secrets-store.csi.k8s.io", volume.CSI.Driver) - - volumeProp := scenario.expectedVolumeProps[i] - for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { - spcValue, exists := volume.CSI.VolumeAttributes[attrKey] - assert.True(t, exists) - assert.True(t, strings.HasPrefix(spcValue, attrValue)) - } - assert.True(t, strings.Contains(volume.Name, volumeProp.expectedVolumeNamePrefix)) - assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) - } - } - }) - - suite.T().Run("CSI Azure Key vault volume mounts", func(t *testing.T) { - t.Parallel() - for _, scenario := range scenarios { - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - deployment.radixDeployment = buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { - var builders []utils.DeployComponentBuilder - builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(scenario.componentName). - WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) - return builders - }) - radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) - for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.createAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) - if err != nil { - t.Log(err.Error()) - } else { - t.Logf("created secret provider class %s", spc.Name) - } - } - volumeMounts, err := GetRadixDeployComponentVolumeMounts(radixDeployComponent, deployment.radixDeployment.GetName()) - assert.Nil(t, err) - assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) - if len(scenario.expectedVolumeProps) == 0 { - continue - } - - for i := 0; i < len(volumeMounts); i++ { - volumeMount := volumeMounts[i] - volumeProp := scenario.expectedVolumeProps[i] - assert.Less(t, len(volumeMount.Name), 64, "volumemount name is too long") - assert.True(t, strings.Contains(volumeMount.Name, volumeProp.expectedVolumeNamePrefix)) - assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath) - assert.True(t, volumeMount.ReadOnly) - } - } - }) -} - -func Test_EmptyDir(t *testing.T) { - appName, envName, compName := "anyapp", "anyenv", "anycomp" - - tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t) - builder := utils.NewDeploymentBuilder(). - WithRadixApplication(utils.NewRadixApplicationBuilder().WithAppName(appName).WithRadixRegistration(utils.NewRegistrationBuilder().WithName(appName))). - WithAppName(appName). - WithEnvironment(envName). - WithComponents( - utils.NewDeployComponentBuilder().WithName(compName).WithVolumeMounts( - v1.RadixVolumeMount{Name: "cache", Path: "/cache", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("50M")}}, - v1.RadixVolumeMount{Name: "log", Path: "/log", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("100M")}}, - ), - ) - - rd, err := ApplyDeploymentWithSync(tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, certClient, builder) - require.NoError(t, err) - assert.NotNil(t, rd) - - deployment, err := kubeclient.AppsV1().Deployments(utils.GetEnvironmentNamespace(appName, envName)).Get(context.Background(), compName, metav1.GetOptions{}) - require.NoError(t, err) - assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2) - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) - -} - -func createRandomStorageClass(props expectedPvcScProperties, namespace, componentName string) storagev1.StorageClass { - return createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.ObjectMeta.Name = utils.RandString(10) - sc.ObjectMeta.Labels[kube.RadixNamespace] = namespace - sc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName - }) -} - -func createRandomPvc(props expectedPvcScProperties, namespace, componentName string) corev1.PersistentVolumeClaim { - return createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = utils.RandString(10) - pvc.ObjectMeta.Namespace = namespace - pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName - pvc.Spec.StorageClassName = utils.StringPtr(utils.RandString(10)) - }) -} - -// TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes -// func setStorageClassMountOption(sc *storagev1.StorageClass, key, value string) { -// mountOptions := sc.MountOptions -// for i, option := range mountOptions { -// if strings.Contains(option, key) { -// mountOptions[i] = fmt.Sprintf("%s=%s", key, value) -// return -// } -// } -// fmt.Printf("MountOption %s not found for the storage class", key) -// } - -func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcScProperties)) expectedPvcScProperties { - appName := "any-app" - environment := "some-env" - componentName := "some-component" - props := expectedPvcScProperties{ - appName: appName, - environment: environment, - namespace: fmt.Sprintf("%s-%s", appName, environment), - componentName: componentName, - radixVolumeMountName: "volume1", - radixStorageName: "storage1", - pvcName: "pvc-csi-az-blob-some-component-volume1-storage1-12345", - storageClassName: "sc-any-app-some-env-csi-az-blob-some-component-volume1-storage1", - radixVolumeMountType: v1.MountTypeBlobFuse2FuseCsiAzure, - requestsVolumeMountSize: "1Mi", - volumeAccessMode: corev1.ReadOnlyMany, // default access mode - volumeName: "csi-az-blob-some-component-volume1-storage1", - scProvisioner: provisionerBlobCsiAzure, - scSecretName: "some-component-volume1-csiazurecreds", - scTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume1/storage1", - scGid: "1000", - scUid: "", - } - if modify != nil { - modify(&props) - } - return props -} - -func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcScProperties)) expectedPvcScProperties { - appName := "any-app" - environment := "some-env" - componentName := "some-component" - props := expectedPvcScProperties{ - appName: appName, - environment: environment, - namespace: fmt.Sprintf("%s-%s", appName, environment), - componentName: componentName, - radixVolumeMountName: "volume1", - radixStorageName: "storage1", - pvcName: "pvc-csi-blobfuse2-fuse2-some-component-volume1-storage1-12345", - storageClassName: "sc-any-app-some-env-csi-blobfuse2-fuse2-some-component-volume1-storage1", - radixVolumeMountType: v1.MountTypeBlobFuse2Fuse2CsiAzure, - requestsVolumeMountSize: "1Mi", - volumeAccessMode: corev1.ReadOnlyMany, // default access mode - volumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", - scProvisioner: provisionerBlobCsiAzure, - scSecretName: "some-component-volume1-csiazurecreds", - scTmpPath: "/tmp/any-app-some-env/csi-blobfuse2-fuse2/some-component/volume1/storage1", - scGid: "1000", - scUid: "", - } - if modify != nil { - modify(&props) - } - return props -} - -func getPropsCsiFileVolume2Storage2(modify func(*expectedPvcScProperties)) expectedPvcScProperties { - appName := "any-app" - environment := "some-env" - componentName := "some-component" - props := expectedPvcScProperties{ - appName: appName, - environment: environment, - namespace: fmt.Sprintf("%s-%s", appName, environment), - componentName: componentName, - radixVolumeMountName: "volume2", - radixStorageName: "storage2", - pvcName: "pvc-csi-az-file-some-component-volume2-storage2-12345", - storageClassName: "sc-any-app-some-env-csi-az-file-some-component-volume2-storage2", - radixVolumeMountType: v1.MountTypeAzureFileCsiAzure, - requestsVolumeMountSize: "1Mi", - volumeAccessMode: corev1.ReadOnlyMany, // default access mode - volumeName: "csi-az-file-some-component-volume2-storage2", - scProvisioner: provisionerFileCsiAzure, - scSecretName: "some-component-volume2-csiazurecreds", - scTmpPath: "/tmp/any-app-some-env/csi-az-file/some-component/volume2/storage2", - scGid: "1000", - scUid: "", - } - if modify != nil { - modify(&props) - } - return props -} - -func putExistingDeploymentVolumesScenarioDataToFakeCluster(scenario *deploymentVolumesTestScenario, deployment *Deployment) { - for _, pvc := range scenario.existingPvcsBeforeTestRun { - _, _ = deployment.kubeclient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) - } - for _, sc := range scenario.existingStorageClassesBeforeTestRun { - _, _ = deployment.kubeclient.StorageV1().StorageClasses().Create(context.Background(), &sc, metav1.CreateOptions{}) - } -} - -func getExistingPvcsAndStorageClassesFromFakeCluster(deployment *Deployment) ([]corev1.PersistentVolumeClaim, []storagev1.StorageClass, error) { - var pvcItems []corev1.PersistentVolumeClaim - var scItems []storagev1.StorageClass - pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) - if err != nil { - return pvcItems, scItems, err - } - if pvcList != nil && pvcList.Items != nil { - pvcItems = pvcList.Items - } - storageClassList, err := deployment.kubeclient.StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{}) - if err != nil { - return pvcItems, scItems, err - } - if storageClassList != nil && storageClassList.Items != nil { - scItems = storageClassList.Items - } - return pvcItems, scItems, nil -} - -func getDesiredDeployment(componentName string, volumes []corev1.Volume) *appsv1.Deployment { - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: componentName, - Labels: map[string]string{ - kube.RadixComponentLabel: componentName, - }, - Annotations: make(map[string]string), - }, - Spec: appsv1.DeploymentSpec{ - Replicas: pointers.Ptr(DefaultReplicas), - Selector: &metav1.LabelSelector{MatchLabels: make(map[string]string)}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: componentName}}, - Volumes: volumes, - }, - }, - }, - } -} - -func buildRd(appName string, environment string, componentName string, radixVolumeMounts []v1.RadixVolumeMount) *v1.RadixDeployment { - return utils.ARadixDeployment(). - WithAppName(appName). - WithEnvironment(environment). - WithComponents(utils.NewDeployComponentBuilder(). - WithName(componentName). - WithVolumeMounts(radixVolumeMounts...)). - BuildRD() -} - -func createPvc(namespace, componentName string, mountType v1.MountType, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { - appName := "app" - pvc := corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: utils.RandString(10), // Set in test scenario - Namespace: namespace, - Labels: map[string]string{ - kube.RadixAppLabel: appName, - kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(mountType), - kube.RadixVolumeMountNameLabel: utils.RandString(10), // Set in test scenario - }, - }, - } - if modify != nil { - modify(&pvc) - } - return pvc -} - -func buildRdWithComponentBuilders(appName string, environment string, componentBuilders func() []utils.DeployComponentBuilder) *v1.RadixDeployment { - return utils.ARadixDeployment(). - WithAppName(appName). - WithEnvironment(environment). - WithComponents(componentBuilders()...). - BuildRD() -} - -func createExpectedStorageClass(props expectedPvcScProperties, modify func(class *storagev1.StorageClass)) storagev1.StorageClass { - mountOptions := []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - // fmt.Sprintf("--tmp-path=%s", props.scTmpPath), //TODO: this option does not work with blobfuse2 in some reason - investigate - } - idOption := getStorageClassIdMountOption(props) - if len(idOption) > 0 { - mountOptions = append(mountOptions, idOption) - } - reclaimPolicy := corev1.PersistentVolumeReclaimRetain - bindingMode := storagev1.VolumeBindingImmediate - sc := storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: props.storageClassName, - Labels: map[string]string{ - kube.RadixAppLabel: props.appName, - kube.RadixNamespace: props.namespace, - kube.RadixComponentLabel: props.componentName, - kube.RadixMountTypeLabel: string(props.radixVolumeMountType), - kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, - }, - }, - Provisioner: props.scProvisioner, - Parameters: map[string]string{ - csiStorageClassProvisionerSecretNameParameter: props.scSecretName, - csiStorageClassProvisionerSecretNamespaceParameter: props.namespace, - csiStorageClassNodeStageSecretNameParameter: props.scSecretName, - csiStorageClassNodeStageSecretNamespaceParameter: props.namespace, - }, - MountOptions: mountOptions, - ReclaimPolicy: &reclaimPolicy, - VolumeBindingMode: &bindingMode, - } - setStorageClassStorageParameter(props.radixVolumeMountType, props.radixStorageName, &sc) - if modify != nil { - modify(&sc) - } - return sc -} - -func setStorageClassStorageParameter(radixVolumeMountType v1.MountType, storageName string, sc *storagev1.StorageClass) { - switch radixVolumeMountType { - case v1.MountTypeBlobFuse2FuseCsiAzure: - sc.Parameters[csiStorageClassContainerNameParameter] = storageName - sc.Parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse - case v1.MountTypeBlobFuse2Fuse2CsiAzure: - sc.Parameters[csiStorageClassContainerNameParameter] = storageName - sc.Parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse2 - case v1.MountTypeBlobFuse2NfsCsiAzure: - sc.Parameters[csiStorageClassContainerNameParameter] = storageName - sc.Parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterNfs - case v1.MountTypeAzureFileCsiAzure: - sc.Parameters[csiStorageClassShareNameParameter] = storageName - } -} - -func getStorageClassIdMountOption(props expectedPvcScProperties) string { - if len(props.scGid) > 0 { - return fmt.Sprintf("-o gid=%s", props.scGid) - } - if len(props.scUid) > 0 { - return fmt.Sprintf("-o uid=%s", props.scGid) - } - return "" -} - -func createExpectedPvc(props expectedPvcScProperties, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { - labels := map[string]string{ - kube.RadixAppLabel: props.appName, - kube.RadixComponentLabel: props.componentName, - kube.RadixMountTypeLabel: string(props.radixVolumeMountType), - kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, - } - pvc := corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: props.pvcName, - Namespace: props.namespace, - Labels: labels, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver - }, - StorageClassName: utils.StringPtr(props.storageClassName), - }, - } - if modify != nil { - modify(&pvc) - } - return pvc -} - -func createVolume(pvcProps expectedPvcScProperties, modify func(*corev1.Volume)) corev1.Volume { - volume := corev1.Volume{ - Name: pvcProps.volumeName, - VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcProps.pvcName, - }}, - } - if modify != nil { - modify(&volume) - } - return volume -} - -func createRadixVolumeMount(props expectedPvcScProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { - volumeMount := v1.RadixVolumeMount{ - Type: props.radixVolumeMountType, - Name: props.radixVolumeMountName, - Storage: props.radixStorageName, - Path: "path1", - GID: "1000", - } - if modify != nil { - modify(&volumeMount) - } - return volumeMount -} -func createBlobFuse2RadixVolumeMount(props expectedPvcScProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { - volumeMount := v1.RadixVolumeMount{ - Name: props.radixVolumeMountName, - Path: "path1", - BlobFuse2: &v1.RadixBlobFuse2VolumeMount{ - Container: props.radixStorageName, - GID: "1000", - }, - } - if modify != nil { - modify(&volumeMount) - } - return volumeMount -} diff --git a/pkg/apis/deployment/jobschedulercomponent.go b/pkg/apis/internal/deployment/jobschedulercomponent.go similarity index 61% rename from pkg/apis/deployment/jobschedulercomponent.go rename to pkg/apis/internal/deployment/jobschedulercomponent.go index 514185110..557856c5a 100644 --- a/pkg/apis/deployment/jobschedulercomponent.go +++ b/pkg/apis/internal/deployment/jobschedulercomponent.go @@ -9,30 +9,31 @@ import ( radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" ) -type jobSchedulerComponent struct { +type JobSchedulerComponent struct { *radixv1.RadixDeployJobComponent - radixDeployment *radixv1.RadixDeployment + RadixDeployment *radixv1.RadixDeployment } -func newJobSchedulerComponent(jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment) radixv1.RadixCommonDeployComponent { - return &jobSchedulerComponent{ +// NewJobSchedulerComponent Constructor +func NewJobSchedulerComponent(jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment) radixv1.RadixCommonDeployComponent { + return &JobSchedulerComponent{ jobComponent, rd, } } -func (js *jobSchedulerComponent) GetHealthChecks() *radixv1.RadixHealthChecks { +func (js *JobSchedulerComponent) GetHealthChecks() *radixv1.RadixHealthChecks { return nil } -func (js *jobSchedulerComponent) GetImage() string { +func (js *JobSchedulerComponent) GetImage() string { containerRegistry := os.Getenv(defaults.ContainerRegistryEnvironmentVariable) radixJobScheduler := os.Getenv(defaults.OperatorRadixJobSchedulerEnvironmentVariable) radixJobSchedulerImageUrl := fmt.Sprintf("%s/%s", containerRegistry, radixJobScheduler) return radixJobSchedulerImageUrl } -func (js *jobSchedulerComponent) GetPorts() []radixv1.ComponentPort { +func (js *JobSchedulerComponent) GetPorts() []radixv1.ComponentPort { if js.RadixDeployJobComponent.SchedulerPort == nil { return nil } @@ -45,25 +46,25 @@ func (js *jobSchedulerComponent) GetPorts() []radixv1.ComponentPort { } } -func (js *jobSchedulerComponent) GetEnvironmentVariables() radixv1.EnvVarsMap { +func (js *JobSchedulerComponent) GetEnvironmentVariables() radixv1.EnvVarsMap { envVarsMap := js.EnvironmentVariables.DeepCopy() if envVarsMap == nil { envVarsMap = radixv1.EnvVarsMap{} } - envVarsMap[defaults.RadixDeploymentEnvironmentVariable] = js.radixDeployment.Name + envVarsMap[defaults.RadixDeploymentEnvironmentVariable] = js.RadixDeployment.Name envVarsMap[defaults.OperatorEnvLimitDefaultMemoryEnvironmentVariable] = os.Getenv(defaults.OperatorEnvLimitDefaultMemoryEnvironmentVariable) return envVarsMap } -func (js *jobSchedulerComponent) GetSecrets() []string { +func (js *JobSchedulerComponent) GetSecrets() []string { return nil } -func (js *jobSchedulerComponent) GetMonitoring() bool { +func (js *JobSchedulerComponent) GetMonitoring() bool { return false } -func (js *jobSchedulerComponent) GetResources() *radixv1.ResourceRequirements { +func (js *JobSchedulerComponent) GetResources() *radixv1.ResourceRequirements { return &radixv1.ResourceRequirements{ Limits: map[string]string{ "memory": "500M", @@ -75,25 +76,26 @@ func (js *jobSchedulerComponent) GetResources() *radixv1.ResourceRequirements { } } -func (js *jobSchedulerComponent) GetReadOnlyFileSystem() *bool { +func (js *JobSchedulerComponent) GetReadOnlyFileSystem() *bool { return pointers.Ptr(true) } -func (js *jobSchedulerComponent) IsAlwaysPullImageOnDeploy() bool { +func (js *JobSchedulerComponent) IsAlwaysPullImageOnDeploy() bool { return true } -func (js *jobSchedulerComponent) GetNode() *radixv1.RadixNode { +func (js *JobSchedulerComponent) GetNode() *radixv1.RadixNode { // Job configuration in radixconfig.yaml contains section "node", which supposed to configure scheduled jobs by RadixDeployment // "node" section settings should not be applied to the JobScheduler component itself return nil } -func (js *jobSchedulerComponent) GetRuntime() *radixv1.Runtime { +func (js *JobSchedulerComponent) GetRuntime() *radixv1.Runtime { return &radixv1.Runtime{Architecture: radixv1.RuntimeArchitectureAmd64} } -func isDeployComponentJobSchedulerDeployment(deployComponent radixv1.RadixCommonDeployComponent) bool { - _, isJobScheduler := interface{}(deployComponent).(*jobSchedulerComponent) +// IsDeployComponentJobSchedulerDeployment Checks if deployComponent is a JobScheduler deployment +func IsDeployComponentJobSchedulerDeployment(deployComponent radixv1.RadixCommonDeployComponent) bool { + _, isJobScheduler := interface{}(deployComponent).(*JobSchedulerComponent) return isJobScheduler } diff --git a/pkg/apis/internal/strings.go b/pkg/apis/internal/strings.go new file mode 100644 index 000000000..58a1fa1ed --- /dev/null +++ b/pkg/apis/internal/strings.go @@ -0,0 +1,9 @@ +package internal + +// EqualTillPostfix Compares two strings till the postfix +func EqualTillPostfix(value1, value2 string, postfixLength int) bool { + if len(value1) < postfixLength || len(value2) < postfixLength { + return false + } + return value1[:len(value1)-postfixLength] == value2[:len(value2)-postfixLength] +} diff --git a/pkg/apis/kube/secret_provider.go b/pkg/apis/kube/secret_provider.go index aa2e8a222..20307f293 100644 --- a/pkg/apis/kube/secret_provider.go +++ b/pkg/apis/kube/secret_provider.go @@ -61,8 +61,7 @@ func GetComponentSecretProviderClassName(radixDeploymentName, radixDeployCompone // by naming component the same as secret-ref object hash := strings.ToLower(commonUtils.RandStringStrSeed(5, strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", radixDeployComponentName, radixDeploymentName, radixSecretRefType, secretRefName)))) - return strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", radixDeployComponentName, radixSecretRefType, secretRefName, - hash)) + return strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", radixDeployComponentName, radixSecretRefType, secretRefName, hash)) } // BuildAzureKeyVaultSecretProviderClass Build a SecretProviderClass for Azure Key vault secret-ref diff --git a/pkg/apis/metrics/custom_metrics.go b/pkg/apis/metrics/custom_metrics.go index 355dd0e0b..c5aca2613 100644 --- a/pkg/apis/metrics/custom_metrics.go +++ b/pkg/apis/metrics/custom_metrics.go @@ -5,14 +5,13 @@ import ( "fmt" "time" - "github.com/equinor/radix-operator/pkg/apis/utils" - "k8s.io/apimachinery/pkg/api/resource" - "github.com/equinor/radix-operator/pkg/apis/defaults" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) var ( diff --git a/pkg/apis/radix/v1/radixapptypes.go b/pkg/apis/radix/v1/radixapptypes.go index 189c73ee8..d598e1e07 100644 --- a/pkg/apis/radix/v1/radixapptypes.go +++ b/pkg/apis/radix/v1/radixapptypes.go @@ -380,9 +380,9 @@ type RadixComponent struct { // +optional Monitoring *bool `json:"monitoring"` - // Deprecated, use publicPort instead. + // Deprecated: For backwards compatibility Public is still supported, new code should use PublicPort instead // +optional - Public bool `json:"public,omitempty"` // Deprecated: For backwards compatibility Public is still supported, new code should use PublicPort instead + Public bool `json:"public,omitempty"` // Defines which port (name) from the ports list that shall be accessible from the internet. // More info: https://www.radix.equinor.com/references/reference-radix-config/#publicport @@ -919,9 +919,9 @@ type RadixPrivateImageHubCredential struct { // RadixVolumeMount defines an external storage resource. type RadixVolumeMount struct { + // Deprecated: use BlobFuse2 instead. // Type defines the storage type. - // Deprecated, use BlobFuse2 or AzureFile instead. - // +kubebuilder:validation:Enum=blob;azure-blob;azure-file;"" + // +kubebuilder:validation:Enum=azure-blob;"" // +optional Type MountType `json:"type"` @@ -931,12 +931,12 @@ type RadixVolumeMount struct { // +kubebuilder:validation:MaxLength=40 Name string `json:"name"` - // Deprecated. Only required by the deprecated type: blob. + // Deprecated: Only required by the deprecated type: blob. // +optional Container string `json:"container,omitempty"` // Outdated. Use Storage instead + // Deprecated: use BlobFuse2 instead. // Storage defines the name of the container in the external storage resource. - // Deprecated, use BlobFuse2 or AzureFile instead. // +optional Storage string `json:"storage"` // Container name, file Share name, etc. @@ -944,48 +944,43 @@ type RadixVolumeMount struct { // +kubebuilder:validation:MinLength=1 Path string `json:"path"` // Path within the pod (replica), where the volume mount has been mounted to + // Deprecated: use BlobFuse2 instead. // GID defines the group ID (number) which will be set as owner of the mounted volume. - // Deprecated, use BlobFuse2 or AzureFile instead. // +optional GID string `json:"gid,omitempty"` // Optional. Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting. https://github.com/kubernetes-sigs/blob-csi-driver/blob/master/docs/driver-parameters.md + // Deprecated: use BlobFuse2 instead. // UID defines the user ID (number) which will be set as owner of the mounted volume. - // Deprecated, use BlobFuse2 or AzureFile instead. // +optional UID string `json:"uid,omitempty"` // Optional. Volume mount owner UserID. Used instead of GID. - // TODO: describe + // Deprecated: use BlobFuse2 instead. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. // +optional SkuName string `json:"skuName,omitempty"` // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - // TODO: describe + // Deprecated: use BlobFuse2 instead. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. // +optional RequestsStorage string `json:"requestsStorage,omitempty"` // Requests resource storage size. Default "1Mi". https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + // Deprecated: use BlobFuse2 instead. // Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. // +kubebuilder:validation:Enum=ReadOnlyMany;ReadWriteOnce;ReadWriteMany;"" // +optional AccessMode string `json:"accessMode,omitempty"` // Available values: ReadOnlyMany (default) - read-only by many nodes, ReadWriteOnce - read-write by a single node, ReadWriteMany - read-write by many nodes. https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes + // Deprecated: use BlobFuse2 instead. // Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. // +kubebuilder:validation:Enum=Immediate;WaitForFirstConsumer;"" // +optional BindingMode string `json:"bindingMode,omitempty"` // Volume binding mode. Available values: Immediate (default), WaitForFirstConsumer. https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode - // BlobFuse2 settings for Azure Storage FUSE CSI driver + // BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2 BlobFuse2 *RadixBlobFuse2VolumeMount `json:"blobFuse2,omitempty"` - // AzureFile settings for Azure File CSI driver - AzureFile *RadixAzureFileVolumeMount `json:"azureFile,omitempty"` - // EmptyDir settings for EmptyDir volume EmptyDir *RadixEmptyDirVolumeMount `json:"emptyDir,omitempty"` } @@ -998,10 +993,6 @@ func (v *RadixVolumeMount) HasBlobFuse2() bool { return v.BlobFuse2 != nil } -func (v *RadixVolumeMount) HasAzureFile() bool { - return v.AzureFile != nil -} - func (v *RadixVolumeMount) HasEmptyDir() bool { return v.EmptyDir != nil } @@ -1019,15 +1010,13 @@ type BlobFuse2Protocol string const ( // BlobFuse2ProtocolFuse2 Use of fuse2 protocol for storage account for blobfuse2 BlobFuse2ProtocolFuse2 BlobFuse2Protocol = "fuse2" - // BlobFuse2ProtocolNfs Use of NFS storage account for blobfuse2 - BlobFuse2ProtocolNfs BlobFuse2Protocol = "nfs" ) // RadixBlobFuse2VolumeMount defines an external storage resource, configured to use Blobfuse2 - A Microsoft supported Azure Storage FUSE driver. // More info: https://github.com/Azure/azure-storage-fuse type RadixBlobFuse2VolumeMount struct { // Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2. - // +kubebuilder:validation:Enum=fuse2;nfs;"" + // +kubebuilder:validation:Enum=fuse2;"" // +optional Protocol BlobFuse2Protocol `json:"protocol,omitempty"` @@ -1074,45 +1063,26 @@ type RadixBlobFuse2VolumeMount struct { // More info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md // +optional Streaming *RadixVolumeMountStreaming `json:"streaming,omitempty"` // Optional. Streaming configuration. Used for blobfuse2. -} -// RadixAzureFileVolumeMount defines an external storage resource, configured to use Azure File with CSI driver. -// More info: https://github.com/kubernetes-sigs/azurefile-csi-driver -// https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/docs/driver-parameters.md -type RadixAzureFileVolumeMount struct { - // Share. Name of the file share in the external storage resource. + // UseAzureIdentity defines that credentials for accessing Azure Storage will be acquired using Azure Workload Identity instead of using a ClientID and Secret. // +optional - Share string `json:"share,omitempty"` + UseAzureIdentity *bool `json:"useAzureIdentity,omitempty"` - // GID defines the group ID (number) which will be set as owner of the mounted volume. + // Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret. // +optional - GID string `json:"gid,omitempty"` // Optional. Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting. https://github.com/kubernetes-sigs/blob-csi-driver/blob/master/docs/driver-parameters.md + StorageAccount string `json:"storageAccount,omitempty"` - // UID defines the user ID (number) which will be set as owner of the mounted volume. + // ResourceGroup of a storage account. Applicable when using a workload identity. // +optional - UID string `json:"uid,omitempty"` // Optional. Volume mount owner UserID. Used instead of GID. + ResourceGroup string `json:"resourceGroup,omitempty"` - // SKU Type of Azure storage. - // More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types + // SubscriptionId of a storage account. Applicable when using a workload identity. // +optional - SkuName string `json:"skuName,omitempty"` // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types + SubscriptionId string `json:"subscriptionId,omitempty"` - // Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + // TenantId of a storage account. Applicable when using a workload identity. // +optional - RequestsStorage string `json:"requestsStorage,omitempty"` // Requests resource storage size. Default "1Mi". https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - - // Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // +kubebuilder:validation:Enum=ReadOnlyMany;ReadWriteOnce;ReadWriteMany;"" - // +optional - AccessMode string `json:"accessMode,omitempty"` // Available values: ReadOnlyMany (default) - read-only by many nodes, ReadWriteOnce - read-write by a single node, ReadWriteMany - read-write by many nodes. https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes - - // Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // +kubebuilder:validation:Enum=Immediate;WaitForFirstConsumer;"" - // +optional - BindingMode string `json:"bindingMode,omitempty"` // Volume binding mode. Available values: Immediate (default), WaitForFirstConsumer. https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode + TenantId string `json:"tenantId,omitempty"` } // RadixVolumeMountStreaming configure streaming to read and write large files that will not fit in the file cache on the local disk. Used for blobfuse2. @@ -1148,16 +1118,10 @@ type MountType string // These are valid types of mount const ( - // MountTypeBlob Use of azure/blobfuse flexvolume - MountTypeBlob MountType = "blob" // MountTypeBlobFuse2FuseCsiAzure Use of azure/csi driver for blobfuse2, protocol Fuse in Azure storage account MountTypeBlobFuse2FuseCsiAzure MountType = "azure-blob" // MountTypeBlobFuse2Fuse2CsiAzure Use of azure/csi driver for blobfuse2, protocol Fuse2 in Azure storage account MountTypeBlobFuse2Fuse2CsiAzure MountType = "blobfuse2-fuse2" - // MountTypeBlobFuse2NfsCsiAzure Use of azure/csi driver for blobfuse2, protocol NFS in Azure storage account - MountTypeBlobFuse2NfsCsiAzure MountType = "blobfuse2-nfs" - // MountTypeAzureFileCsiAzure Use of azure/csi driver for Azure File in Azure storage account - MountTypeAzureFileCsiAzure MountType = "azure-file" ) // RadixNode defines node attributes, where container should be scheduled diff --git a/pkg/apis/radix/v1/zz_generated.deepcopy.go b/pkg/apis/radix/v1/zz_generated.deepcopy.go index c5d7443a2..d671b7661 100644 --- a/pkg/apis/radix/v1/zz_generated.deepcopy.go +++ b/pkg/apis/radix/v1/zz_generated.deepcopy.go @@ -980,22 +980,6 @@ func (in *RadixApplyConfigSpec) DeepCopy() *RadixApplyConfigSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RadixAzureFileVolumeMount) DeepCopyInto(out *RadixAzureFileVolumeMount) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RadixAzureFileVolumeMount. -func (in *RadixAzureFileVolumeMount) DeepCopy() *RadixAzureFileVolumeMount { - if in == nil { - return nil - } - out := new(RadixAzureFileVolumeMount) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RadixAzureKeyVault) DeepCopyInto(out *RadixAzureKeyVault) { *out = *in @@ -1335,6 +1319,11 @@ func (in *RadixBlobFuse2VolumeMount) DeepCopyInto(out *RadixBlobFuse2VolumeMount *out = new(RadixVolumeMountStreaming) (*in).DeepCopyInto(*out) } + if in.UseAzureIdentity != nil { + in, out := &in.UseAzureIdentity, &out.UseAzureIdentity + *out = new(bool) + **out = **in + } return } @@ -3216,11 +3205,6 @@ func (in *RadixVolumeMount) DeepCopyInto(out *RadixVolumeMount) { *out = new(RadixBlobFuse2VolumeMount) (*in).DeepCopyInto(*out) } - if in.AzureFile != nil { - in, out := &in.AzureFile, &out.AzureFile - *out = new(RadixAzureFileVolumeMount) - **out = **in - } if in.EmptyDir != nil { in, out := &in.EmptyDir, &out.EmptyDir *out = new(RadixEmptyDirVolumeMount) diff --git a/pkg/apis/radixvalidators/errors.go b/pkg/apis/radixvalidators/errors.go index c0a7f4a84..4f366df4f 100644 --- a/pkg/apis/radixvalidators/errors.go +++ b/pkg/apis/radixvalidators/errors.go @@ -9,34 +9,35 @@ import ( ) var ( - ErrMissingPrivateImageHubUsername = errors.New("missing private image hub username") - ErrEnvForDNSAppAliasNotDefined = errors.New("env for dns app alias not defined") - ErrComponentForDNSAppAliasNotDefined = errors.New("component for dns app alias not defined") - ErrExternalAliasCannotBeEmpty = errors.New("external alias cannot be empty") - ErrEnvForDNSExternalAliasNotDefined = errors.New("env for dns external alias not defined") - ErrComponentForDNSExternalAliasNotDefined = errors.New("component for dns external alias not defined") - ErrComponentForDNSExternalAliasIsNotMarkedAsPublic = errors.New("component for dns external alias is not marked as public") - ErrComponentHasInvalidHealthCheck = errors.New("component has invalid health check") - ErrEnvironmentReferencedByComponentDoesNotExist = errors.New("environment referenced by component does not exist") - ErrInvalidPortNameLength = errors.New("invalid port name length") - ErrPortNameIsRequiredForPublicComponent = errors.New("port name is required for public component") - ErrMonitoringPortNameIsNotFoundComponent = errors.New("monitoring port name is not found component") - ErrMultipleMatchingPortNames = errors.New("multiple matching port names") - ErrSchedulerPortCannotBeEmptyForJob = errors.New("scheduler port cannot be empty for job") - ErrPayloadPathCannotBeEmptyForJob = errors.New("payload path cannot be empty for job") - ErrMemoryResourceRequirementFormat = errors.New("memory resource requirement format") - ErrCPUResourceRequirementFormat = errors.New("cpu resource requirement format") - ErrInvalidVerificationType = errors.New("invalid verification") - ErrInvalidHealthCheckProbe = errors.New("probe configuration error, only one action allowed") - ErrSuccessThresholdMustBeOne = errors.New("success threshold must be equal to one") - ErrResourceRequestOverLimit = errors.New("resource request over limit") - ErrInvalidResource = errors.New("invalid resource") - ErrDuplicateExternalAlias = errors.New("duplicate external alias") - ErrInvalidBranchName = errors.New("invalid branch name") - ErrCombiningTriggersWithResourcesIsIllegal = errors.New("combining triggers with resources is invalid") - ErrMaxReplicasForHPANotSetOrZero = errors.New("max replicas for hpa not set or zero") - ErrMinReplicasGreaterThanMaxReplicas = errors.New("min replicas greater than max replicas") - ErrNoScalingResourceSet = errors.New("no scaling resource set") // Deprecated: Replaaced by ErrInvalidTriggerDefinition + ErrMissingPrivateImageHubUsername = errors.New("missing private image hub username") + ErrEnvForDNSAppAliasNotDefined = errors.New("env for dns app alias not defined") + ErrComponentForDNSAppAliasNotDefined = errors.New("component for dns app alias not defined") + ErrExternalAliasCannotBeEmpty = errors.New("external alias cannot be empty") + ErrEnvForDNSExternalAliasNotDefined = errors.New("env for dns external alias not defined") + ErrComponentForDNSExternalAliasNotDefined = errors.New("component for dns external alias not defined") + ErrComponentForDNSExternalAliasIsNotMarkedAsPublic = errors.New("component for dns external alias is not marked as public") + ErrComponentHasInvalidHealthCheck = errors.New("component has invalid health check") + ErrEnvironmentReferencedByComponentDoesNotExist = errors.New("environment referenced by component does not exist") + ErrInvalidPortNameLength = errors.New("invalid port name length") + ErrPortNameIsRequiredForPublicComponent = errors.New("port name is required for public component") + ErrMonitoringPortNameIsNotFoundComponent = errors.New("monitoring port name is not found component") + ErrMultipleMatchingPortNames = errors.New("multiple matching port names") + ErrSchedulerPortCannotBeEmptyForJob = errors.New("scheduler port cannot be empty for job") + ErrPayloadPathCannotBeEmptyForJob = errors.New("payload path cannot be empty for job") + ErrMemoryResourceRequirementFormat = errors.New("memory resource requirement format") + ErrCPUResourceRequirementFormat = errors.New("cpu resource requirement format") + ErrInvalidVerificationType = errors.New("invalid verification") + ErrInvalidHealthCheckProbe = errors.New("probe configuration error, only one action allowed") + ErrSuccessThresholdMustBeOne = errors.New("success threshold must be equal to one") + ErrResourceRequestOverLimit = errors.New("resource request over limit") + ErrInvalidResource = errors.New("invalid resource") + ErrDuplicateExternalAlias = errors.New("duplicate external alias") + ErrInvalidBranchName = errors.New("invalid branch name") + ErrCombiningTriggersWithResourcesIsIllegal = errors.New("combining triggers with resources is invalid") + ErrMaxReplicasForHPANotSetOrZero = errors.New("max replicas for hpa not set or zero") + ErrMinReplicasGreaterThanMaxReplicas = errors.New("min replicas greater than max replicas") + // Deprecated: Replaced by ErrInvalidTriggerDefinition + ErrNoScalingResourceSet = errors.New("no scaling resource set") ErrNoDefinitionInTrigger = errors.New("no definition in trigger") ErrMoreThanOneDefinitionInTrigger = errors.New("each trigger must contain only one definition") ErrInvalidTriggerDefinition = errors.New("invalid trigger definition") @@ -291,10 +292,6 @@ func volumeMountBlobFuse2ValidationError(cause error) error { return fmt.Errorf("blobFuse2 failed validation. %w", cause) } -func volumeMountAzureFileValidationError(cause error) error { - return fmt.Errorf("azureFile failed validation. %w", cause) -} - func volumeMountEmptyDirValidationError(cause error) error { return fmt.Errorf("emptyDir failed validation. %w", cause) } diff --git a/pkg/apis/radixvalidators/testdata/radixconfig.yaml b/pkg/apis/radixvalidators/testdata/radixconfig.yaml index 5a65b9b71..c649b6385 100644 --- a/pkg/apis/radixvalidators/testdata/radixconfig.yaml +++ b/pkg/apis/radixvalidators/testdata/radixconfig.yaml @@ -93,7 +93,7 @@ spec: azure: clientId: 11111111-2222-3333-4444-555555555555 volumeMounts: - - type: blob + - type: azure-blob name: blobvol container: blobcontainer path: /path/to/mount @@ -202,7 +202,7 @@ spec: azure: clientId: 11111111-2222-3333-4444-555555555555 volumeMounts: - - type: blob + - type: azure-blob name: blobvol container: blobcontainer path: /path/to/mount diff --git a/pkg/apis/radixvalidators/validate_ra.go b/pkg/apis/radixvalidators/validate_ra.go index 8cff539fc..5062f443d 100644 --- a/pkg/apis/radixvalidators/validate_ra.go +++ b/pkg/apis/radixvalidators/validate_ra.go @@ -137,6 +137,7 @@ func validatePrivateImageHubs(app *radixv1.RadixApplication) error { // RAContainsOldPublic Checks to see if the radix config is using the deprecated config for public port func RAContainsOldPublic(app *radixv1.RadixApplication) bool { for _, component := range app.Spec.Components { + //nolint:staticcheck if component.Public { return true } @@ -580,6 +581,7 @@ func validateVerificationType(verificationType *radixv1.VerificationType) error func componentHasPublicPort(component *radixv1.RadixComponent) bool { return slice.Any(component.GetPorts(), func(p radixv1.ComponentPort) bool { + //nolint:staticcheck return len(p.Name) > 0 && (p.Name == component.PublicPort || component.Public) }) } @@ -1584,7 +1586,7 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { } volumeSourceCount := len(slice.FindAll( - []bool{v.HasDeprecatedVolume(), v.HasBlobFuse2(), v.HasAzureFile(), v.HasEmptyDir()}, + []bool{v.HasDeprecatedVolume(), v.HasBlobFuse2(), v.HasEmptyDir()}, func(b bool) bool { return b }), ) if volumeSourceCount > 1 { @@ -1603,10 +1605,6 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { if err := validateVolumeMountBlobFuse2(v.BlobFuse2); err != nil { return volumeMountValidationError(v.Name, err) } - case v.HasAzureFile(): - if err := validateVolumeMountAzureFile(v.AzureFile); err != nil { - return volumeMountValidationError(v.Name, err) - } case v.HasEmptyDir(): if err := validateVolumeMountEmptyDir(v.EmptyDir); err != nil { return volumeMountValidationError(v.Name, err) @@ -1618,32 +1616,26 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { } func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { - if !slices.Contains([]radixv1.MountType{radixv1.MountTypeBlob, radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeAzureFileCsiAzure}, v.Type) { + //nolint:staticcheck + if v.Type != radixv1.MountTypeBlobFuse2FuseCsiAzure { return volumeMountDeprecatedSourceValidationError(ErrVolumeMountInvalidType) } - + //nolint:staticcheck if len(v.RequestsStorage) > 0 { + //nolint:staticcheck if _, err := resource.ParseQuantity(v.RequestsStorage); err != nil { return volumeMountDeprecatedSourceValidationError(fmt.Errorf("%w. %w", ErrVolumeMountInvalidRequestsStorage, err)) } } - - switch v.Type { - case radixv1.MountTypeBlob: - if len(v.Container) == 0 { - return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingContainer) - } - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeAzureFileCsiAzure: - if len(v.Storage) == 0 { - return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingStorage) - } + //nolint:staticcheck + if v.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure && len(v.Container) == 0 { + return volumeMountBlobFuse2ValidationError(ErrVolumeMountMissingContainer) } - return nil } func validateVolumeMountBlobFuse2(fuse2 *radixv1.RadixBlobFuse2VolumeMount) error { - if !slices.Contains([]radixv1.BlobFuse2Protocol{radixv1.BlobFuse2ProtocolFuse2, radixv1.BlobFuse2ProtocolNfs, ""}, fuse2.Protocol) { + if !slices.Contains([]radixv1.BlobFuse2Protocol{radixv1.BlobFuse2ProtocolFuse2, ""}, fuse2.Protocol) { return volumeMountBlobFuse2ValidationError(ErrVolumeMountInvalidProtocol) } @@ -1659,10 +1651,6 @@ func validateVolumeMountBlobFuse2(fuse2 *radixv1.RadixBlobFuse2VolumeMount) erro return nil } -func validateVolumeMountAzureFile(_ *radixv1.RadixAzureFileVolumeMount) error { - return volumeMountAzureFileValidationError(ErrVolumeMountTypeNotImplemented) -} - func validateVolumeMountEmptyDir(emptyDir *radixv1.RadixEmptyDirVolumeMount) error { if emptyDir.SizeLimit.IsZero() { return volumeMountEmptyDirValidationError(ErrVolumeMountMissingSizeLimit) @@ -1735,6 +1723,7 @@ func getEnv(app *radixv1.RadixApplication, name string) *radixv1.Environment { func doesComponentHaveAPublicPort(app *radixv1.RadixApplication, name string) bool { for _, component := range app.Spec.Components { if component.Name == name { + //nolint:staticcheck return component.Public || component.PublicPort != "" } } diff --git a/pkg/apis/radixvalidators/validate_ra_test.go b/pkg/apis/radixvalidators/validate_ra_test.go index d65a6e903..9264f87be 100644 --- a/pkg/apis/radixvalidators/validate_ra_test.go +++ b/pkg/apis/radixvalidators/validate_ra_test.go @@ -1,3 +1,4 @@ +//nolint:staticcheck package radixvalidators_test import ( @@ -1334,79 +1335,14 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountMissingType, }, - "multiple types: deprecated source and blobfuse2": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "anyname", - Path: "/path", - Type: radixv1.MountTypeBlob, - BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{}, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMultipleTypes, - }, - "multiple types: blobfuse2 and emptyDir": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "anyname", - Path: "/path", - Type: radixv1.MountTypeBlob, - BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{}, - EmptyDir: &radixv1.RadixEmptyDirVolumeMount{}, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMultipleTypes, - }, - "deprecated blob: valid": { + "deprecated azure-blob: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ { - Type: "blob", + Type: "azure-blob", Name: "some_name", Path: "some_path", - Container: "any-container", - // RequestsStorage: "50M", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: nil, - }, - "deprecated blob: missing container": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "blob", - Name: "some_name", - Path: "some_path", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMissingContainer, - }, - "deprecated azure-blob: valid": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "azure-blob", - Name: "some_name", - Path: "some_path", - Storage: "any-storage", + Container: "any-storage", }, } @@ -1428,23 +1364,7 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { return volumeMounts }, updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMissingStorage, - }, - "deprecated azure-file: valid": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "azure-file", - Name: "some_name", - Path: "some_path", - Storage: "any-storage", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: nil, + expectedError: radixvalidators.ErrVolumeMountMissingContainer, }, "deprecated common: invalid type": { volumeMounts: func() []radixv1.RadixVolumeMount { @@ -1461,22 +1381,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountInvalidType, }, - "deprecated common: invalid requestsStorage": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "blob", - Name: "some_name", - Path: "some_path", - RequestsStorage: "50x", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountInvalidRequestsStorage, - }, "blobfuse2: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ @@ -1512,24 +1416,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: nil, }, - "blobfuse2: valid protocol nfs": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "some_name", - Path: "some_path", - BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{ - Protocol: radixv1.BlobFuse2ProtocolNfs, - Container: "any-container", - }, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: nil, - }, "blobfuse2: valid requestsStorage": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ @@ -1599,21 +1485,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountInvalidRequestsStorage, }, - "azureFile: not implemented": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "some_name", - Path: "some_path", - AzureFile: &radixv1.RadixAzureFileVolumeMount{}, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountTypeNotImplemented, - }, "emptyDir: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ @@ -2290,10 +2161,8 @@ func Test_ValidateApplicationCanBeAppliedWithDNSAliases(t *testing.T) { otherAppName = "anyapp2" raEnv = "test" raComponentName = "app" - raPublicPort = 8080 someEnv = "dev" someComponentName = "component-abc" - somePort = 9090 alias1 = "alias1" alias2 = "alias2" ) diff --git a/pkg/apis/test/utils.go b/pkg/apis/test/utils.go index be897bf10..4c9a55e21 100644 --- a/pkg/apis/test/utils.go +++ b/pkg/apis/test/utils.go @@ -6,6 +6,7 @@ import ( "os" "time" + commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -170,8 +171,9 @@ func (tu *Utils) ApplyApplicationUpdate(applicationBuilder utils.ApplicationBuil // ApplyDeployment Will help persist a deployment func (tu *Utils) ApplyDeployment(ctx context.Context, deploymentBuilder utils.DeploymentBuilder) (*radixv1.RadixDeployment, error) { envs := make(map[string]struct{}) - if deploymentBuilder.GetApplicationBuilder() != nil { - ra, _ := tu.ApplyApplication(deploymentBuilder.GetApplicationBuilder()) + applicationBuilder := deploymentBuilder.GetApplicationBuilder() + if !commonUtils.IsNil(applicationBuilder) { + ra, _ := tu.ApplyApplication(applicationBuilder) for _, env := range ra.Spec.Environments { envs[env.Name] = struct{}{} } diff --git a/pkg/apis/utils/applicationcomponent_builder.go b/pkg/apis/utils/applicationcomponent_builder.go index dae81bd3b..bda82dd4e 100644 --- a/pkg/apis/utils/applicationcomponent_builder.go +++ b/pkg/apis/utils/applicationcomponent_builder.go @@ -13,7 +13,8 @@ type RadixApplicationComponentBuilder interface { WithHealthChecks(startupProbe, readynessProbe, livenessProbe *radixv1.RadixProbe) RadixApplicationComponentBuilder WithImage(string) RadixApplicationComponentBuilder WithImageTagName(imageTagName string) RadixApplicationComponentBuilder - WithPublic(bool) RadixApplicationComponentBuilder // Deprecated: For backwards compatibility WithPublic is still supported, new code should use WithPublicPort instead + // Deprecated: For backwards compatibility WithPublic is still supported, new code should use WithPublicPort instead + WithPublic(bool) RadixApplicationComponentBuilder WithPublicPort(string) RadixApplicationComponentBuilder WithPort(string, int32) RadixApplicationComponentBuilder WithPorts([]radixv1.ComponentPort) RadixApplicationComponentBuilder @@ -44,28 +45,29 @@ type radixApplicationComponentBuilder struct { dockerfileName string image string alwaysPullImageOnDeploy *bool - public bool // Deprecated: For backwards compatibility public is still supported, new code should use publicPort instead - publicPort string - monitoringConfig radixv1.MonitoringConfig - ports []radixv1.ComponentPort - secrets []string - secretRefs radixv1.RadixSecretRefs - ingressConfiguration []string - environmentConfig []RadixEnvironmentConfigBuilder - variables radixv1.EnvVarsMap - resources radixv1.ResourceRequirements - node radixv1.RadixNode - authentication *radixv1.Authentication - volumeMounts []radixv1.RadixVolumeMount - enabled *bool - identity *radixv1.Identity - readOnlyFileSystem *bool - monitoring *bool - imageTagName string - horizontalScaling *radixv1.RadixHorizontalScaling - runtime *radixv1.Runtime - network *radixv1.Network - healtChecks *radixv1.RadixHealthChecks + // Deprecated: For backwards compatibility public is still supported, new code should use publicPort instead + public bool + publicPort string + monitoringConfig radixv1.MonitoringConfig + ports []radixv1.ComponentPort + secrets []string + secretRefs radixv1.RadixSecretRefs + ingressConfiguration []string + environmentConfig []RadixEnvironmentConfigBuilder + variables radixv1.EnvVarsMap + resources radixv1.ResourceRequirements + node radixv1.RadixNode + authentication *radixv1.Authentication + volumeMounts []radixv1.RadixVolumeMount + enabled *bool + identity *radixv1.Identity + readOnlyFileSystem *bool + monitoring *bool + imageTagName string + horizontalScaling *radixv1.RadixHorizontalScaling + runtime *radixv1.Runtime + network *radixv1.Network + healtChecks *radixv1.RadixHealthChecks } func (rcb *radixApplicationComponentBuilder) WithName(name string) RadixApplicationComponentBuilder { diff --git a/pkg/apis/utils/branch/path_matcher_test.go b/pkg/apis/utils/branch/path_matcher_test.go index 203192e65..9c280cff5 100644 --- a/pkg/apis/utils/branch/path_matcher_test.go +++ b/pkg/apis/utils/branch/path_matcher_test.go @@ -1,7 +1,6 @@ package branch import ( - "strings" "testing" "github.com/stretchr/testify/assert" @@ -40,12 +39,13 @@ func TestMatchesPattern(t *testing.T) { assert.False(t, MatchesPattern("release", "release/q3/0.1.3")) assert.False(t, MatchesPattern("release/*", "release")) assert.False(t, MatchesPattern("release/**/*", "release")) - assert.False(t, MatchesPattern("(test)|(main)/*", "release/t")) + assert.False(t, MatchesPattern("(test)|(main)/.*", "release/t")) assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1q.0.2")) assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1..2")) assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1.2")) assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+\\.*", "v1.2.20-asdf")) + assert.True(t, MatchesPattern("test/*/tull", "test/test1/test2/tull")) assert.True(t, MatchesPattern("release/*", "release/q3/0.1.3")) assert.True(t, MatchesPattern("test/*/tull", "test/test1/test2/tull")) assert.True(t, MatchesPattern("**", "test")) @@ -75,12 +75,10 @@ func TestMatchesPattern(t *testing.T) { assert.True(t, MatchesPattern("release/**/*", "release/q3/0.1.3")) assert.True(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1.0.2")) assert.True(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v123.033.2112")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1q.0.2")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1..2")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1.2")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+\\.*", "v1.2.20-asdf")) assert.True(t, MatchesPattern("(test)|(main)/*", "test/t")) assert.True(t, MatchesPattern("(test)|(main)/*", "main/t")) } - -func TestMatchesPattern2(t *testing.T) { - replace := strings.NewReplacer("**", "*", ".**", ".*", "*", ".*", "..*", ".*").Replace("*/w*f**f*.***.**.*") - assert.NotEmpty(t, replace) - //assert.True(t, MatchesPattern(".**", "test/test-test")) -} diff --git a/pipeline-runner/internal/hash/encoding.go b/pkg/apis/utils/hash/encoding.go similarity index 100% rename from pipeline-runner/internal/hash/encoding.go rename to pkg/apis/utils/hash/encoding.go diff --git a/pipeline-runner/internal/hash/hash.go b/pkg/apis/utils/hash/hash.go similarity index 50% rename from pipeline-runner/internal/hash/hash.go rename to pkg/apis/utils/hash/hash.go index c532aa3a0..5ffd9ba2a 100644 --- a/pipeline-runner/internal/hash/hash.go +++ b/pkg/apis/utils/hash/hash.go @@ -4,6 +4,9 @@ import ( "encoding/hex" "fmt" "strings" + + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + corev1 "k8s.io/api/core/v1" ) type Algorithm string @@ -60,3 +63,39 @@ func extractAlgorithmFromHashString(hashString string) Algorithm { } return Algorithm(parts[0]) } + +// Constants used to generate hash for RadixApplication and BuildSecret if they are nil. Do not change. +const ( + magicValueForNilRadixApplication = "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7" + magicValueForNilBuildSecretData = "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj" +) + +func CreateRadixApplicationHash(ra *v1.RadixApplication) (string, error) { + return ToHashString(SHA256, getRadixApplicationOrMagicValue(ra)) +} + +func CompareRadixApplicationHash(targetHash string, ra *v1.RadixApplication) (bool, error) { + return CompareWithHashString(getRadixApplicationOrMagicValue(ra), targetHash) +} + +func CreateBuildSecretHash(secret *corev1.Secret) (string, error) { + return ToHashString(SHA256, getBuildSecretOrMagicValue(secret)) +} + +func CompareBuildSecretHash(targetHash string, secret *corev1.Secret) (bool, error) { + return CompareWithHashString(getBuildSecretOrMagicValue(secret), targetHash) +} + +func getRadixApplicationOrMagicValue(ra *v1.RadixApplication) any { + if ra == nil { + return magicValueForNilRadixApplication + } + return ra.Spec +} + +func getBuildSecretOrMagicValue(secret *corev1.Secret) any { + if secret == nil || len(secret.Data) == 0 { + return magicValueForNilBuildSecretData + } + return secret.Data +} diff --git a/pipeline-runner/internal/hash/hash_test.go b/pkg/apis/utils/hash/hash_test.go similarity index 98% rename from pipeline-runner/internal/hash/hash_test.go rename to pkg/apis/utils/hash/hash_test.go index 572e75c16..2d0fadf1c 100644 --- a/pipeline-runner/internal/hash/hash_test.go +++ b/pkg/apis/utils/hash/hash_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/equinor/radix-operator/pipeline-runner/internal/hash" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/pipeline-runner/internal/hash/sha256.go b/pkg/apis/utils/hash/sha256.go similarity index 100% rename from pipeline-runner/internal/hash/sha256.go rename to pkg/apis/utils/hash/sha256.go diff --git a/pkg/apis/utils/persistentvolumeclaim.go b/pkg/apis/utils/persistentvolumeclaim.go deleted file mode 100644 index 1880037ce..000000000 --- a/pkg/apis/utils/persistentvolumeclaim.go +++ /dev/null @@ -1,92 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - - "github.com/equinor/radix-operator/pkg/apis/kube" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/strategicpatch" -) - -// GetPersistentVolumeClaimMap Get map from PersistentVolumeClaim with name as key -func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim) map[string]*corev1.PersistentVolumeClaim { - return getPersistentVolumeClaimMap(pvcList, false) -} - -func getPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) map[string]*corev1.PersistentVolumeClaim { - pvcMap := make(map[string]*corev1.PersistentVolumeClaim) - for _, pvc := range *pvcList { - pvc := pvc - name := pvc.Name - if ignoreRandomPostfixInName { - name = ShortenString(name, 6) - } - pvcMap[name] = &pvc - } - return pvcMap -} - -// EqualPvcLists Compare two PersistentVolumeClaim lists. When ignoreRandomPostfixInName=true - last 6 chars of the name (e.g.'-abc12') are ignored during comparison -func EqualPvcLists(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { - if len(*pvcList1) != len(*pvcList2) { - return false, nil - } - pvcMap1 := getPersistentVolumeClaimMap(pvcList1, ignoreRandomPostfixInName) - pvcMap2 := getPersistentVolumeClaimMap(pvcList2, ignoreRandomPostfixInName) - for pvcName, pvc1 := range pvcMap1 { - pvc2, ok := pvcMap2[pvcName] - if !ok { - return false, fmt.Errorf("PVS not found by name %s in second list", pvcName) - } - if equal, err := EqualPvcs(pvc1, pvc2, ignoreRandomPostfixInName); err != nil || !equal { - return false, err - } - } - return true, nil -} - -// EqualPvcs Compare two PersistentVolumeClaim pointers -func EqualPvcs(pvc1 *corev1.PersistentVolumeClaim, pvc2 *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { - pvc1Copy, labels1 := getPvcCopyWithLabels(pvc1, ignoreRandomPostfixInName) - pvc2Copy, labels2 := getPvcCopyWithLabels(pvc2, ignoreRandomPostfixInName) - patchBytes, err := getPvcPatch(pvc1Copy, pvc2Copy) - if err != nil { - return false, err - } - if !EqualStringMaps(labels1, labels2) { - return false, fmt.Errorf("PVC-s labels are not equal") - } - if !kube.IsEmptyPatch(patchBytes) { - return false, fmt.Errorf("PVC-s are not equal: %s", patchBytes) - } - return true, nil -} - -func getPvcCopyWithLabels(pvc *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (*corev1.PersistentVolumeClaim, map[string]string) { - pvcCopy := pvc.DeepCopy() - pvcCopy.ObjectMeta.ManagedFields = nil // HACK: to avoid ManagedFields comparison - if ignoreRandomPostfixInName { - pvcCopy.ObjectMeta.Name = ShortenString(pvcCopy.ObjectMeta.Name, 6) - } - // to avoid label order variations - labels := pvcCopy.ObjectMeta.Labels - pvcCopy.ObjectMeta.Labels = map[string]string{} - return pvcCopy, labels -} - -func getPvcPatch(pvc1, pvc2 *corev1.PersistentVolumeClaim) ([]byte, error) { - json1, err := json.Marshal(pvc1) - if err != nil { - return nil, err - } - json2, err := json.Marshal(pvc2) - if err != nil { - return nil, err - } - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(json1, json2, corev1.PersistentVolumeClaim{}) - if err != nil { - return nil, err - } - return patchBytes, nil -} diff --git a/pkg/apis/utils/storageclass.go b/pkg/apis/utils/storageclass.go deleted file mode 100644 index 3886e77ca..000000000 --- a/pkg/apis/utils/storageclass.go +++ /dev/null @@ -1,88 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - "github.com/equinor/radix-operator/pkg/apis/kube" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/util/strategicpatch" -) - -// GetStorageClassMap Get map from StorageClassList with name as key -func GetStorageClassMap(scList *[]storagev1.StorageClass) map[string]*storagev1.StorageClass { - scMap := make(map[string]*storagev1.StorageClass) - for _, sc := range *scList { - sc := sc - scMap[sc.Name] = &sc - } - return scMap -} - -// EqualStorageClassLists Compare two StorageClass lists -func EqualStorageClassLists(scList1, scList2 *[]storagev1.StorageClass) (bool, error) { - if len(*scList1) != len(*scList2) { - return false, fmt.Errorf("different StorageClass list sizes: %v, %v", len(*scList1), len(*scList2)) - } - scMap1 := GetStorageClassMap(scList1) - scMap2 := GetStorageClassMap(scList2) - for scName, sc1 := range scMap1 { - sc2, ok := scMap2[scName] - if !ok { - return false, fmt.Errorf("StorageClass not found by name %s in second list", scName) - } - if equal, err := EqualStorageClasses(sc1, sc2); err != nil || !equal { - return false, err - } - } - return true, nil -} - -// EqualStorageClasses Compare two StorageClass pointers -func EqualStorageClasses(sc1, sc2 *storagev1.StorageClass) (bool, error) { - sc1Copy, labels1, params1, mountOptions1 := getStorageClassCopyWithCollections(sc1) - sc2Copy, labels2, params2, mountOptions2 := getStorageClassCopyWithCollections(sc2) - patchBytes, err := getStorageClassesPatch(sc1Copy, sc2Copy) - if err != nil { - return false, err - } - if !EqualStringMaps(labels1, labels2) { - return false, nil //StorageClasses labels are not equal - } - if !EqualStringMaps(params1, params2) { - return false, nil //StorageClasses parameters are not equal - } - if !EqualStringLists(mountOptions1, mountOptions2) { - return false, nil //StorageClass-es MountOptions are not equal - } - if !kube.IsEmptyPatch(patchBytes) { - return false, nil //StorageClasses properties are not equal - } - return true, nil -} - -func getStorageClassCopyWithCollections(sc *storagev1.StorageClass) (*storagev1.StorageClass, map[string]string, map[string]string, []string) { - scCopy := sc.DeepCopy() - scCopy.ObjectMeta.ManagedFields = nil //HACK: to avoid ManagedFields comparison - //to avoid label order variations - labels := scCopy.ObjectMeta.Labels - scCopy.ObjectMeta.Labels = map[string]string{} - //to avoid Parameters order variations - scParams := scCopy.Parameters - scCopy.Parameters = map[string]string{} - //to avoid MountOptions order variations - scMountOptions := scCopy.MountOptions - scCopy.MountOptions = []string{} - return scCopy, labels, scParams, scMountOptions -} - -func getStorageClassesPatch(sc1, sc2 *storagev1.StorageClass) ([]byte, error) { - json1, err := json.Marshal(sc1) - if err != nil { - return []byte{}, err - } - json2, err := json.Marshal(sc2) - if err != nil { - return []byte{}, err - } - return strategicpatch.CreateTwoWayMergePatch(json1, json2, storagev1.StorageClass{}) -} diff --git a/pkg/apis/volumemount/defaults.go b/pkg/apis/volumemount/defaults.go new file mode 100644 index 000000000..7b7dc0157 --- /dev/null +++ b/pkg/apis/volumemount/defaults.go @@ -0,0 +1,43 @@ +package volumemount + +const ( + // CsiVolumeSourceDriverSecretStore Driver name for the secret store + CsiVolumeSourceDriverSecretStore = "secrets-store.csi.k8s.io" + // CsiVolumeSourceVolumeAttributeSecretProviderClass Secret provider class volume attribute + CsiVolumeSourceVolumeAttributeSecretProviderClass = "secretProviderClass" + // ReadOnlyMountOption The readonly volume mount option for CSI fuse driver + ReadOnlyMountOption = "-o ro" + + csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- + csiPersistentVolumeNameTemplate = "pv-radixvolumemount-%s" // pv- + + csiMountOptionGid = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting + csiMountOptionUid = "uid" // Volume mount owner UserID. Used instead of GroupID + csiMountOptionUseAdls = "use-adls" // Use ADLS or Block Blob + csiMountOptionStreamingEnabled = "streaming" // Enable Streaming + csiMountOptionStreamingCache = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory + csiMountOptionStreamingMaxBlocksPerFile = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming + csiMountOptionStreamingMaxBuffers = "max-buffers" // The total number of buffers to be cached in memory (in MB). + csiMountOptionStreamingBlockSize = "block-size-mb" // The size of each block to be cached in memory (in MB). + csiMountOptionStreamingBufferSize = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). + + csiVolumeAttributeProtocolParameterFuse = "fuse" // Protocol "blobfuse" + csiVolumeAttributeProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" + csiVolumeAttributeStorageAccount = "storageAccount" + csiVolumeAttributeClientID = "clientID" + csiVolumeAttributeResourceGroup = "resourcegroup" + csiVolumeAttributeSubscriptionId = "subscriptionid" + csiVolumeAttributeTenantId = "tenantID" + + csiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" + csiVolumeMountAttributePvcName = "csi.storage.k8s.io/pvc/name" + csiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pvc/namespace" + csiVolumeMountAttributeSecretNamespace = "secretnamespace" + csiVolumeMountAttributeProtocol = "protocol" // Protocol + csiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages + csiVolumeMountAttributeProvisionerIdentity = "storage.kubernetes.io/csiProvisionerIdentity" + + nameRandPartLength = 5 // The length of a trailing random part in names + + provisionerBlobCsiAzure string = "blob.csi.azure.com" // Use of azure/csi driver for blob in Azure storage account +) diff --git a/pkg/apis/volumemount/persistentvolume.go b/pkg/apis/volumemount/persistentvolume.go new file mode 100644 index 000000000..8b9f45cf5 --- /dev/null +++ b/pkg/apis/volumemount/persistentvolume.go @@ -0,0 +1,98 @@ +package volumemount + +import ( + "strings" + + "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/slice" + "github.com/equinor/radix-operator/pkg/apis/internal" + corev1 "k8s.io/api/core/v1" +) + +// EqualPersistentVolumes Compare two PersistentVolumes +func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { + if pv1 == nil || pv2 == nil || pv1.Spec.CSI == nil || pv2.Spec.CSI == nil { + return false + } + // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner. When all volume mounts gets labels, uncomment these lines + //if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { + // return false + //} + expectedClonedVolumeAttrs := cloneMap(pv1.Spec.CSI.VolumeAttributes, csiVolumeMountAttributePvName, csiVolumeMountAttributePvcName, csiVolumeMountAttributeProvisionerIdentity) + actualClonedVolumeAttrs := cloneMap(pv2.Spec.CSI.VolumeAttributes, csiVolumeMountAttributePvName, csiVolumeMountAttributePvcName, csiVolumeMountAttributeProvisionerIdentity) + if !utils.EqualStringMaps(expectedClonedVolumeAttrs, actualClonedVolumeAttrs) { + return false + } + if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { + return false + } + + if pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || + len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || + (len(pv1.Spec.AccessModes) == 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || + pv1.Spec.CSI.Driver != pv2.Spec.CSI.Driver { + return false + } + + if pv1.Spec.CSI.NodeStageSecretRef != nil { + if pv2.Spec.CSI.NodeStageSecretRef == nil || pv1.Spec.CSI.NodeStageSecretRef.Name != pv2.Spec.CSI.NodeStageSecretRef.Name { + return false + } + } else if pv2.Spec.CSI.NodeStageSecretRef != nil { + return false + } + + if pv1.Spec.ClaimRef != nil { + if pv2.Spec.ClaimRef == nil || + !internal.EqualTillPostfix(pv1.Spec.ClaimRef.Name, pv2.Spec.ClaimRef.Name, nameRandPartLength) || + !internal.EqualTillPostfix(pv1.Spec.ClaimRef.Name, pv2.Spec.ClaimRef.Name, nameRandPartLength) || + pv1.Spec.ClaimRef.Namespace != pv2.Spec.ClaimRef.Namespace || + pv1.Spec.ClaimRef.Kind != pv2.Spec.ClaimRef.Kind { + return false + } + } else if pv2.Spec.ClaimRef != nil { + return false + } + return true +} + +func cloneMap(original map[string]string, ignoreKeys ...string) map[string]string { + clonedMap := make(map[string]string, len(original)) + ignoreKeysMap := convertToSet(ignoreKeys) + for key, value := range original { + if _, ok := ignoreKeysMap[key]; !ok { + clonedMap[key] = value + } + } + return clonedMap +} + +func convertToSet(ignoreKeys []string) map[string]struct{} { + return slice.Reduce(ignoreKeys, make(map[string]struct{}), func(acc map[string]struct{}, item string) map[string]struct{} { + acc[item] = struct{}{} + return acc + }) +} + +func getMountOptionsMap(mountOptions []string) map[string]string { + return slice.Reduce(mountOptions, make(map[string]string), func(acc map[string]string, item string) map[string]string { + if len(item) == 0 { + return acc + } + itemParts := strings.Split(item, "=") + key, value := "", "" + if len(itemParts) > 0 { + key = itemParts[0] + } + if key == "--tmp-path" { + return acc // ignore tmp-path, which eventually can be introduced + } + if len(itemParts) > 1 { + value = itemParts[1] + } + if len(key) > 0 { + acc[key] = value + } + return acc + }) +} diff --git a/pkg/apis/volumemount/persistentvolume_test.go b/pkg/apis/volumemount/persistentvolume_test.go new file mode 100644 index 000000000..1d1c4ddfc --- /dev/null +++ b/pkg/apis/volumemount/persistentvolume_test.go @@ -0,0 +1,256 @@ +package volumemount + +import ( + "testing" + + "github.com/equinor/radix-operator/pkg/apis/utils" + corev1 "k8s.io/api/core/v1" +) + +func Test_EqualPersistentVolumes(t *testing.T) { + createPv := func(modify func(pv *corev1.PersistentVolume)) *corev1.PersistentVolume { + pv := createExpectedPv(getPropsCsiBlobVolume1Storage1(nil), modify) + return &pv + } + createPvWithProps := func(modify func(*expectedPvcPvProperties)) *corev1.PersistentVolume { + pv := createExpectedPv(getPropsCsiBlobVolume1Storage1(modify), nil) + return &pv + } + tests := []struct { + name string + pv1 *corev1.PersistentVolume + pv2 *corev1.PersistentVolume + expected bool + }{ + { + name: "both nil", + pv1: nil, + pv2: nil, + expected: false, + }, + { + name: "one nil", + pv1: createPv(nil), + pv2: nil, + expected: false, + }, + { + name: "equal", + pv1: createPv(nil), + pv2: createPv(nil), + expected: true, + }, + { + name: "different access mode", + pv1: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + expected: false, + }, + { + name: "no access mode", + pv1: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = nil }), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + expected: false, + }, + { + name: "no ClaimRef", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef = nil }), + expected: false, + }, + { + name: "different ClaimRef name", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef.Name = utils.RandString(10) }), + expected: false, + }, + { + name: "different ClaimRef namespace", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef.Namespace = utils.RandString(10) }), + expected: false, + }, + { + name: "different ClaimRef kind", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef.Kind = "secret" }), + expected: false, + }, + { + name: "no CSI", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.CSI = nil }), + expected: false, + }, + { + name: "no CSI VolumeAttributes", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { pv.Spec.CSI.VolumeAttributes = nil }), + expected: false, + }, + { + name: "different CSI VolumeAttribute csiVolumeMountAttributeContainerName", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeContainerName] = utils.RandString(10) + }), + expected: false, + }, + { + name: "different CSI VolumeAttribute csiVolumeMountAttributeProtocol", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = utils.RandString(10) + }), + expected: false, + }, + { + name: "ignore different CSI VolumeAttribute csiVolumeMountAttributePvName", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvName] = utils.RandString(10) + }), + expected: true, + }, + { + name: "ignore different CSI VolumeAttribute csiVolumeMountAttributePvcName", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvcName] = utils.RandString(10) + }), + expected: true, + }, + { + name: "ignore different CSI VolumeAttribute csiVolumeMountAttributeProvisionerIdentity", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProvisionerIdentity] = utils.RandString(10) + }), + expected: true, + }, + { + name: "different CSI VolumeAttribute csiVolumeMountAttributePvcNamespace", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvcNamespace] = utils.RandString(10) + }), + expected: false, + }, + { + name: "different CSI VolumeAttribute csiVolumeMountAttributeSecretNamespace", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeSecretNamespace] = utils.RandString(10) + }), + expected: false, + }, + { + name: "extra CSI VolumeAttribute", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeAttributes["some-extra-attribute"] = utils.RandString(10) + }), + expected: false, + }, + { + name: "different CSI Driver", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.Driver = utils.RandString(10) + }), + expected: false, + }, + { + name: "no CSI NodeStageSecretRef", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.NodeStageSecretRef = nil + }), + expected: false, + }, + { + name: "different CSI NodeStageSecretRef", + pv1: createPv(nil), + pv2: createPv(func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.NodeStageSecretRef.Name = utils.RandString(10) + }), + expected: false, + }, + { + name: "different namespace", + pv1: createPv(nil), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.namespace = utils.RandString(10) + }), + expected: false, + }, + { + name: "different blobStorageName", + pv1: createPv(nil), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.blobStorageName = utils.RandString(10) + }), + expected: false, + }, + { + name: "different pvGid", + pv1: createPv(nil), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.pvGid = "7779" + }), + expected: false, + }, + { + name: "different pvUid", + pv1: createPvWithProps(func(props *expectedPvcPvProperties) { + props.pvGid = "" + props.pvUid = "7779" + }), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.pvGid = "" + props.pvUid = "8889" + }), + expected: false, + }, + { + name: "different pvProvisioner", + pv1: createPv(nil), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.pvProvisioner = utils.RandString(10) + }), + expected: false, + }, + { + name: "different pvSecretName", + pv1: createPv(nil), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.pvSecretName = utils.RandString(10) + }), + expected: false, + }, + { + name: "different readOnly", + pv1: createPvWithProps(func(props *expectedPvcPvProperties) { + props.readOnly = true + }), + pv2: createPvWithProps(func(props *expectedPvcPvProperties) { + props.readOnly = false + }), + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EqualPersistentVolumes(tt.pv1, tt.pv2); got != tt.expected { + t.Errorf("EqualPersistentVolumes() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/pkg/apis/volumemount/persistentvolumeclaim.go b/pkg/apis/volumemount/persistentvolumeclaim.go new file mode 100644 index 000000000..cb0649599 --- /dev/null +++ b/pkg/apis/volumemount/persistentvolumeclaim.go @@ -0,0 +1,45 @@ +package volumemount + +import ( + "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + corev1 "k8s.io/api/core/v1" +) + +// GetPersistentVolumeClaimMap Get map from PersistentVolumeClaim with name as key +func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim) map[string]*corev1.PersistentVolumeClaim { + pvcMap := make(map[string]*corev1.PersistentVolumeClaim) + for _, pvc := range *pvcList { + pvc := pvc + pvcMap[pvc.Name] = &pvc + } + return pvcMap +} + +// EqualPersistentVolumeClaims Compare two PersistentVolumeClaims +func EqualPersistentVolumeClaims(pvc1, pvc2 *corev1.PersistentVolumeClaim) bool { + if pvc1 == nil || pvc2 == nil { + return false + } + if pvc1.GetNamespace() != pvc2.GetNamespace() { + return false + } + if !utils.EqualStringMaps(pvc1.GetLabels(), pvc2.GetLabels()) { + return false + } + pvc1StorageCapacity, existsPvc1StorageCapacity := pvc1.Spec.Resources.Requests[corev1.ResourceStorage] + pvc2StorageCapacity, existsPvc2StorageCapacity := pvc2.Spec.Resources.Requests[corev1.ResourceStorage] + if (existsPvc1StorageCapacity != existsPvc2StorageCapacity) || + (existsPvc1StorageCapacity && pvc1StorageCapacity.Cmp(pvc2StorageCapacity) != 0) { + return false + } + if len(pvc1.Spec.AccessModes) != len(pvc2.Spec.AccessModes) { + return false + } + if len(pvc1.Spec.AccessModes) == 1 && pvc1.Spec.AccessModes[0] != pvc2.Spec.AccessModes[0] { + return false + } + volumeMode1 := pointers.Val(pvc1.Spec.VolumeMode) + volumeMode2 := pointers.Val(pvc2.Spec.VolumeMode) + return volumeMode1 == volumeMode2 +} diff --git a/pkg/apis/volumemount/persistentvolumeclaim_test.go b/pkg/apis/volumemount/persistentvolumeclaim_test.go new file mode 100644 index 000000000..5b5c64113 --- /dev/null +++ b/pkg/apis/volumemount/persistentvolumeclaim_test.go @@ -0,0 +1,156 @@ +package volumemount + +import ( + "github.com/equinor/radix-common/utils/pointers" + "k8s.io/apimachinery/pkg/api/resource" + "testing" + + "github.com/equinor/radix-operator/pkg/apis/kube" + corev1 "k8s.io/api/core/v1" +) + +func Test_EqualPersistentVolumeClaims(t *testing.T) { + createPvc := func(modify func(pv *corev1.PersistentVolumeClaim)) *corev1.PersistentVolumeClaim { + pv := createExpectedPvc(getPropsCsiBlobVolume1Storage1(nil), modify) + return &pv + } + tests := []struct { + name string + pvc1 *corev1.PersistentVolumeClaim + pvc2 *corev1.PersistentVolumeClaim + expected bool + }{ + { + name: "both nil", + pvc1: nil, + pvc2: nil, + expected: false, + }, + { + name: "one nil", + pvc1: createPvc(nil), + pvc2: nil, + expected: false, + }, + { + name: "equal", + pvc1: createPvc(nil), + pvc2: createPvc(nil), + expected: true, + }, + { + name: "different namespaces", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.ObjectMeta.Namespace = "namespace1" + }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.ObjectMeta.Namespace = "namespace2" + }), + expected: false, + }, + { + name: "no storage resource", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.Resources.Requests = map[corev1.ResourceName]resource.Quantity{} + }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("1M") + }), + expected: false, + }, + { + name: "different storage resource", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("1M") + }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("2M") + }), + expected: false, + }, + { + name: "no access mode", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.Spec.AccessModes = nil }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + expected: false, + }, + { + name: "different access mode", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + expected: false, + }, + { + name: "no volume mode", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.Spec.VolumeMode = nil }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.VolumeMode = pointers.Ptr(corev1.PersistentVolumeBlock) + }), + expected: false, + }, + { + name: "different volume mode", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.VolumeMode = pointers.Ptr(corev1.PersistentVolumeFilesystem) + }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.Spec.VolumeMode = pointers.Ptr(corev1.PersistentVolumeBlock) + }), + expected: false, + }, + { + name: "different app name label", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels[kube.RadixAppLabel] = "app1" }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels[kube.RadixAppLabel] = "app2" }), + expected: false, + }, + { + name: "different radix component label", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.ObjectMeta.Labels[kube.RadixComponentLabel] = "componentName1" + }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { + pv.ObjectMeta.Labels[kube.RadixComponentLabel] = "componentName2" + }), + expected: false, + }, + { + name: "different volume mount name label", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "name1" }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "name2" }), + expected: false, + }, + { + name: "different volume mount type label", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels[kube.RadixMountTypeLabel] = "type1" }), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels[kube.RadixMountTypeLabel] = "type2" }), + expected: false, + }, + { + name: "extra label", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) {}), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { pv.ObjectMeta.Labels["extra-label"] = "extra-value" }), + expected: false, + }, + { + name: "missing label", + pvc1: createPvc(func(pv *corev1.PersistentVolumeClaim) {}), + pvc2: createPvc(func(pv *corev1.PersistentVolumeClaim) { delete(pv.ObjectMeta.Labels, kube.RadixAppLabel) }), + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EqualPersistentVolumeClaims(tt.pvc1, tt.pvc2); got != tt.expected { + t.Errorf("EqualPersistentVolumeClaims() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/pkg/apis/volumemount/test.go b/pkg/apis/volumemount/test.go new file mode 100644 index 000000000..c69c94cae --- /dev/null +++ b/pkg/apis/volumemount/test.go @@ -0,0 +1,672 @@ +package volumemount + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" + "github.com/equinor/radix-operator/pkg/apis/internal" + "github.com/equinor/radix-operator/pkg/apis/kube" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/client/clientset/versioned" + radixfake "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + "github.com/google/uuid" + kedav2 "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned" + kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" + prometheusclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" + prometheusfake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake" + "github.com/stretchr/testify/suite" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + secretsstorevclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" + secretproviderfake "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/fake" +) + +func modifyPvc(pvc v1.PersistentVolumeClaim, modify func(pvc *v1.PersistentVolumeClaim)) v1.PersistentVolumeClaim { + modify(&pvc) + return pvc +} + +type TestSuite struct { + suite.Suite + radixCommonDeployComponentFactories []radixv1.RadixCommonDeployComponentFactory +} + +type TestEnv struct { + kubeClient kubernetes.Interface + radixClient versioned.Interface + secretProviderClient secretsstorevclient.Interface + prometheusClient prometheusclient.Interface + kubeUtil *kube.Kube + kedaClient kedav2.Interface +} + +type volumeMountTestScenario struct { + name string + radixVolumeMount radixv1.RadixVolumeMount + expectedVolumeName string + expectedError string + expectedPvcName string +} + +type deploymentVolumesTestScenario struct { + name string + props expectedPvcPvProperties + radixVolumeMounts []radixv1.RadixVolumeMount + volumes []v1.Volume + existingPvs []v1.PersistentVolume + existingPvcs []v1.PersistentVolumeClaim + expectedPvs []v1.PersistentVolume + expectedPvcs []v1.PersistentVolumeClaim +} + +type pvcTestScenario struct { + volumeMountTestScenario + pv v1.PersistentVolume + pvc v1.PersistentVolumeClaim +} + +const ( + appName1 = "any-app" + envName1 = "some-env" + componentName1 = "some-component" +) + +var ( + anotherComponentName = strings.ToLower(utils.RandString(10)) + anotherVolumeMountName = strings.ToLower(utils.RandString(10)) +) + +func TestVolumeMountTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (suite *TestSuite) SetupSuite() { + suite.radixCommonDeployComponentFactories = []radixv1.RadixCommonDeployComponentFactory{ + radixv1.RadixDeployComponentFactory{}, + radixv1.RadixDeployJobComponentFactory{}, + } +} + +func getTestEnv() TestEnv { + testEnv := TestEnv{ + kubeClient: fake.NewSimpleClientset(), + radixClient: radixfake.NewSimpleClientset(), + kedaClient: kedafake.NewSimpleClientset(), + secretProviderClient: secretproviderfake.NewSimpleClientset(), + prometheusClient: prometheusfake.NewSimpleClientset(), + } + kubeUtil, _ := kube.New(testEnv.kubeClient, testEnv.radixClient, testEnv.kedaClient, testEnv.secretProviderClient) + testEnv.kubeUtil = kubeUtil + return testEnv +} + +type expectedPvcPvProperties struct { + appName string + environment string + componentName string + radixVolumeMountName string + blobStorageName string + pvcName string + persistentVolumeName string + radixVolumeMountType radixv1.MountType + requestsVolumeMountSize string + volumeAccessMode v1.PersistentVolumeAccessMode + volumeName string + pvProvisioner string + pvSecretName string + pvGid string + pvUid string + namespace string + readOnly bool +} + +func modifyPv(pv v1.PersistentVolume, modify func(pv *v1.PersistentVolume)) v1.PersistentVolume { + modify(&pv) + return pv +} + +func createRandomVolumeMount(modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { + vm := radixv1.RadixVolumeMount{ + Name: strings.ToLower(operatorUtils.RandString(10)), + Path: "/tmp/" + strings.ToLower(operatorUtils.RandString(10)), + BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{ + Protocol: radixv1.BlobFuse2ProtocolFuse2, + Container: strings.ToLower(operatorUtils.RandString(10)), + }, + } + modify(&vm) + return vm +} + +func matchPvAndPvc(pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvcName] = pvc.GetName() + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvcNamespace] = pvc.GetNamespace() + pv.Spec.ClaimRef = &v1.ObjectReference{ + APIVersion: "radixv1", + Kind: k8s.KindPersistentVolumeClaim, + Name: pvc.GetName(), + Namespace: pvc.GetNamespace(), + } + pvc.Spec.VolumeName = pv.Name +} + +func modifyProps(props expectedPvcPvProperties, modify func(props *expectedPvcPvProperties)) expectedPvcPvProperties { + modify(&props) + return props +} + +func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) v1.PersistentVolume { + return createExpectedPv(props, func(pv *v1.PersistentVolume) { + pvName := getCsiAzurePvName() + pv.ObjectMeta.Name = pvName + pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace + pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvName] = pvName + }) +} + +func createRandomPvc(props expectedPvcPvProperties, namespace, componentName string) v1.PersistentVolumeClaim { + return createExpectedPvc(props, func(pvc *v1.PersistentVolumeClaim) { + pvcName, err := getCsiAzurePvcName(componentName, &radixv1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.blobStorageName, Path: "/tmp"}) + if err != nil { + panic(err) + } + pvName := getCsiAzurePvName() + pvc.ObjectMeta.Name = pvcName + pvc.ObjectMeta.Namespace = namespace + pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pvc.Spec.VolumeName = pvName + }) +} + +func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, namespace, componentName, anotherVolumeMountName string) v1.PersistentVolume { + return createAutoProvisionedPvWithStorageClass(props, func(pv *v1.PersistentVolume) { + pvName := "pvc-" + uuid.NewString() + pv.ObjectMeta.Name = pvName + if pv.ObjectMeta.Labels == nil { + pv.ObjectMeta.Labels = make(map[string]string) + } + pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace + pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = anotherVolumeMountName + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvName] = pvName + }) +} + +func createRandomAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, namespace, componentName, anotherVolumeMountName string) v1.PersistentVolumeClaim { + return createExpectedAutoProvisionedPvcWithStorageClass(props, func(pvc *v1.PersistentVolumeClaim) { + pvcName, err := getCsiAzurePvcName(componentName, &radixv1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.blobStorageName, Path: "/tmp"}) + if err != nil { + panic(err) + } + pvName := getCsiAzurePvName() + pvc.ObjectMeta.Name = pvcName + pvc.ObjectMeta.Namespace = namespace + pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = anotherVolumeMountName + pvc.Spec.VolumeName = pvName + }) +} + +func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { + props := expectedPvcPvProperties{ + appName: appName1, + environment: envName1, + namespace: operatorUtils.GetEnvironmentNamespace(appName1, envName1), + componentName: componentName1, + radixVolumeMountName: "volume1", + blobStorageName: "storage1", + pvcName: "pvc-csi-az-blob-some-component-volume1-storage1-12345", + persistentVolumeName: "pv-radixvolumemount-some-uuid", + radixVolumeMountType: radixv1.MountTypeBlobFuse2FuseCsiAzure, + requestsVolumeMountSize: "1Mi", + volumeAccessMode: v1.ReadOnlyMany, // default access mode + volumeName: "csi-az-blob-some-component-volume1-storage1", + pvProvisioner: provisionerBlobCsiAzure, + pvSecretName: "some-component-volume1-csiazurecreds", + pvGid: "1000", + pvUid: "", + readOnly: true, + } + if modify != nil { + modify(&props) + } + return props +} + +func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { + props := expectedPvcPvProperties{ + appName: appName1, + environment: envName1, + namespace: fmt.Sprintf("%s-%s", appName1, envName1), + componentName: componentName1, + radixVolumeMountName: "volume1", + blobStorageName: "storage1", + pvcName: "pvc-csi-blobfuse2-fuse2-some-component-volume1-storage1-12345", + persistentVolumeName: "pv-radixvolumemount-some-uuid", + radixVolumeMountType: radixv1.MountTypeBlobFuse2Fuse2CsiAzure, + requestsVolumeMountSize: "1Mi", + volumeAccessMode: v1.ReadOnlyMany, // default access mode + volumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", + pvProvisioner: provisionerBlobCsiAzure, + pvSecretName: "some-component-volume1-csiazurecreds", + pvGid: "1000", + pvUid: "", + readOnly: true, + } + if modify != nil { + modify(&props) + } + return props +} + +func putExistingDeploymentVolumesScenarioDataToFakeCluster(kubeClient kubernetes.Interface, scenario *deploymentVolumesTestScenario) { + for _, pvc := range scenario.existingPvcs { + _, _ = kubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) + } + for _, pv := range scenario.existingPvs { + _, _ = kubeClient.CoreV1().PersistentVolumes().Create(context.Background(), &pv, metav1.CreateOptions{}) + } +} + +func getExistingPvcsAndPersistentVolumeFromFakeCluster(kubeClient kubernetes.Interface) ([]v1.PersistentVolumeClaim, []v1.PersistentVolume, error) { + var pvcItems []v1.PersistentVolumeClaim + var pvItems []v1.PersistentVolume + pvcList, err := kubeClient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, nil, err + } + if pvcList != nil && pvcList.Items != nil { + pvcItems = pvcList.Items + } + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, nil, err + } + if pvList != nil && pvList.Items != nil { + pvItems = pvList.Items + } + return pvcItems, pvItems, nil +} + +func getDesiredDeployment(componentName string, volumes []v1.Volume) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: componentName, + Labels: map[string]string{ + kube.RadixComponentLabel: componentName, + }, + Annotations: make(map[string]string), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointers.Ptr(defaults.DefaultReplicas), + Selector: &metav1.LabelSelector{MatchLabels: make(map[string]string)}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: componentName}}, + Volumes: volumes, + }, + }, + }, + } +} + +func buildRd(appName string, environment string, componentName string, radixVolumeMounts []radixv1.RadixVolumeMount) *radixv1.RadixDeployment { + return operatorUtils.ARadixDeployment(). + WithAppName(appName). + WithEnvironment(environment). + WithComponents(operatorUtils.NewDeployComponentBuilder(). + WithName(componentName). + WithVolumeMounts(radixVolumeMounts...)). + BuildRD() +} + +func createPvc(namespace, componentName string, mountType radixv1.MountType, modify func(*v1.PersistentVolumeClaim)) v1.PersistentVolumeClaim { + appName := "app" + pvc := v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandString(10), // Set in test scenario + Namespace: namespace, + Labels: map[string]string{ + kube.RadixAppLabel: appName, + kube.RadixComponentLabel: componentName, + kube.RadixMountTypeLabel: string(mountType), + kube.RadixVolumeMountNameLabel: utils.RandString(10), // Set in test scenario + }, + }, + } + if modify != nil { + modify(&pvc) + } + return pvc +} + +func createExpectedPv(props expectedPvcPvProperties, modify func(pv *v1.PersistentVolume)) v1.PersistentVolume { + mountOptions := getMountOptions(props) + pv := v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: props.persistentVolumeName, + Labels: map[string]string{ + kube.RadixAppLabel: props.appName, + kube.RadixNamespace: props.namespace, + kube.RadixComponentLabel: props.componentName, + kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, + }, + }, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{v1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: props.pvProvisioner, + VolumeHandle: getVolumeHandle(props.namespace, props.componentName, props.persistentVolumeName, props.blobStorageName), + VolumeAttributes: map[string]string{ + csiVolumeMountAttributeContainerName: props.blobStorageName, + csiVolumeMountAttributeProtocol: csiVolumeAttributeProtocolParameterFuse2, + csiVolumeMountAttributePvName: props.persistentVolumeName, + csiVolumeMountAttributePvcName: props.pvcName, + csiVolumeMountAttributePvcNamespace: props.namespace, + csiVolumeMountAttributeSecretNamespace: props.namespace, + // skip auto-created by the provisioner "storage.kubernetes.io/csiProvisionerIdentity": "1732528668611-2190-blob.csi.azure.com" + }, + NodeStageSecretRef: &v1.SecretReference{ + Name: props.pvSecretName, + Namespace: props.namespace, + }, + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{props.volumeAccessMode}, + ClaimRef: &v1.ObjectReference{ + APIVersion: "radixv1", + Kind: k8s.KindPersistentVolumeClaim, + Namespace: props.namespace, + Name: props.pvcName, + }, + StorageClassName: "", + MountOptions: mountOptions, + PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain, + VolumeMode: pointers.Ptr(v1.PersistentVolumeFilesystem), + }, + Status: v1.PersistentVolumeStatus{Phase: v1.VolumeBound}, + } + setVolumeMountAttribute(&pv, props.radixVolumeMountType, props.blobStorageName, props.pvcName) + if modify != nil { + modify(&pv) + } + return pv +} + +func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modify func(pv *v1.PersistentVolume)) v1.PersistentVolume { + mountOptions := getMountOptionsInRandomOrder(props) + pv := v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: props.persistentVolumeName, + }, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{v1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: props.pvProvisioner, + VolumeHandle: "MC_clusters_ABC_northeurope##testdata#pvc-681b9ffc-66cc-4e09-90b2-872688b792542#some-app-namespace#", + VolumeAttributes: map[string]string{ + csiVolumeMountAttributeContainerName: props.blobStorageName, + csiVolumeMountAttributeProtocol: csiVolumeAttributeProtocolParameterFuse2, + csiVolumeMountAttributePvName: props.persistentVolumeName, + csiVolumeMountAttributePvcName: props.pvcName, + csiVolumeMountAttributePvcNamespace: props.namespace, + csiVolumeMountAttributeSecretNamespace: props.namespace, + csiVolumeMountAttributeProvisionerIdentity: "6540128941979-5154-blob.csi.azure.com", + }, + NodeStageSecretRef: &v1.SecretReference{ + Name: props.pvSecretName, + Namespace: props.namespace, + }, + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{props.volumeAccessMode}, + ClaimRef: &v1.ObjectReference{ + APIVersion: "radixv1", + Kind: k8s.KindPersistentVolumeClaim, + Namespace: props.namespace, + Name: props.pvcName, + }, + StorageClassName: "some-storage-class", + MountOptions: mountOptions, + PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain, + VolumeMode: pointers.Ptr(v1.PersistentVolumeFilesystem), + }, + Status: v1.PersistentVolumeStatus{Phase: v1.VolumeBound, LastPhaseTransitionTime: pointers.Ptr(metav1.Time{Time: time.Now()})}, + } + setVolumeMountAttribute(&pv, props.radixVolumeMountType, props.blobStorageName, props.pvcName) + if modify != nil { + modify(&pv) + } + return pv +} + +func getMountOptions(props expectedPvcPvProperties, extraOptions ...string) []string { + options := []string{ + "--file-cache-timeout-in-seconds=120", + "--use-attr-cache=true", + "--cancel-list-on-mount-seconds=0", + "-o allow_other", + "-o attr_timeout=120", + "-o entry_timeout=120", + "-o negative_timeout=120", + } + if props.readOnly { + options = append(options, ReadOnlyMountOption) + } + idOption := getPersistentVolumeIdMountOption(props) + if len(idOption) > 0 { + options = append(options, idOption) + } + return append(options, extraOptions...) +} + +func getMountOptionsInRandomOrder(props expectedPvcPvProperties, extraOptions ...string) []string { + options := []string{ + "--file-cache-timeout-in-seconds=120", + "--use-attr-cache=true", + "-o allow_other", + "--cancel-list-on-mount-seconds=0", + "-o negative_timeout=120", + "-o entry_timeout=120", + "-o attr_timeout=120", + } + idOption := getPersistentVolumeIdMountOption(props) + if len(idOption) > 0 { + options = append(options, idOption) + } + if props.readOnly { + options = append(options, ReadOnlyMountOption) + } + return append(options, extraOptions...) +} + +func setVolumeMountAttribute(pv *v1.PersistentVolume, radixVolumeMountType radixv1.MountType, containerName, pvcName string) { + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeContainerName] = containerName + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributePvcName] = pvcName + switch radixVolumeMountType { + case radixv1.MountTypeBlobFuse2FuseCsiAzure: + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = csiVolumeAttributeProtocolParameterFuse + case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = csiVolumeAttributeProtocolParameterFuse2 + } +} + +func getPersistentVolumeIdMountOption(props expectedPvcPvProperties) string { + if len(props.pvGid) > 0 { + return fmt.Sprintf("-o gid=%s", props.pvGid) + } + if len(props.pvUid) > 0 { + return fmt.Sprintf("-o uid=%s", props.pvUid) + } + return "" +} + +func createExpectedPvc(props expectedPvcPvProperties, modify func(*v1.PersistentVolumeClaim)) v1.PersistentVolumeClaim { + labels := map[string]string{ + kube.RadixAppLabel: props.appName, + kube.RadixComponentLabel: props.componentName, + kube.RadixMountTypeLabel: string(props.radixVolumeMountType), + kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, + } + pvc := v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: props.pvcName, + Namespace: props.namespace, + Labels: labels, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{props.volumeAccessMode}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver + }, + VolumeName: props.persistentVolumeName, + VolumeMode: pointers.Ptr(v1.PersistentVolumeFilesystem), + }, + Status: v1.PersistentVolumeClaimStatus{Phase: v1.ClaimBound}, + } + if modify != nil { + modify(&pvc) + } + return pvc +} + +func createExpectedAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, modify func(*v1.PersistentVolumeClaim)) v1.PersistentVolumeClaim { + labels := map[string]string{ + kube.RadixAppLabel: props.appName, + kube.RadixComponentLabel: props.componentName, + kube.RadixMountTypeLabel: string(props.radixVolumeMountType), + kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, + } + annotations := map[string]string{ + "pv.kubernetes.io/bind-completed": "yes", + "pv.kubernetes.io/bound-by-controller": "yes", + "volume.beta.kubernetes.io/storage-provisioner": props.pvProvisioner, + "volume.kubernetes.io/storage-provisioner": props.pvProvisioner, + } + pvc := v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: props.pvcName, + Namespace: props.namespace, + Labels: labels, + Annotations: annotations, + Finalizers: []string{"kubernetes.io/pvc-protection"}, + ResourceVersion: "630363277", + UID: types.UID("681b9ffc-66cc-4e09-90b2-872688b792542"), + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{props.volumeAccessMode}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver + }, + VolumeName: props.persistentVolumeName, + StorageClassName: pointers.Ptr("some-storage-class"), + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + AccessModes: []v1.PersistentVolumeAccessMode{props.volumeAccessMode}, + Capacity: map[v1.ResourceName]resource.Quantity{v1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, + }, + } + if modify != nil { + modify(&pvc) + } + return pvc +} + +func createTestVolume(pvcProps expectedPvcPvProperties, modify func(v *v1.Volume)) v1.Volume { + volume := v1.Volume{ + Name: pvcProps.volumeName, + VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcProps.pvcName, + }}, + } + if modify != nil { + modify(&volume) + } + return volume +} + +func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { + volumeMount := radixv1.RadixVolumeMount{ + Type: props.radixVolumeMountType, + Name: props.radixVolumeMountName, + Storage: props.blobStorageName, + Path: "path1", + GID: "1000", + } + if modify != nil { + modify(&volumeMount) + } + return volumeMount +} + +func createBlobFuse2RadixVolumeMount(props expectedPvcPvProperties, modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { + volumeMount := radixv1.RadixVolumeMount{ + Name: props.radixVolumeMountName, + Path: "path1", + BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{ + Container: props.blobStorageName, + GID: "1000", + }, + } + if modify != nil { + modify(&volumeMount) + } + return volumeMount +} + +func equalPersistentVolumes(expectedPvs, actualPvs *[]v1.PersistentVolume) bool { + if len(*expectedPvs) != len(*actualPvs) { + return false + } + for _, expectedPv := range *expectedPvs { + var hasEqualPv bool + for _, actualPv := range *actualPvs { + if EqualPersistentVolumes(&expectedPv, &actualPv) { + hasEqualPv = true + break + } + } + if !hasEqualPv { + return false + } + } + return true +} + +func equalPersistentVolumeClaims(list1, list2 *[]v1.PersistentVolumeClaim) bool { + if len(*list1) != len(*list2) { + return false + } + for _, pvc1 := range *list1 { + var hasEqualPvc bool + for _, pvc2 := range *list2 { + if internal.EqualTillPostfix(pvc1.GetName(), pvc2.GetName(), nameRandPartLength) && + EqualPersistentVolumeClaims(&pvc1, &pvc2) { + hasEqualPvc = true + break + } + } + if !hasEqualPvc { + return false + } + } + return true +} diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go new file mode 100644 index 000000000..ac96e4696 --- /dev/null +++ b/pkg/apis/volumemount/volumemount.go @@ -0,0 +1,1094 @@ +package volumemount + +import ( + "context" + "errors" + "fmt" + "strings" + + commonUtils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-common/utils/slice" + "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + "github.com/equinor/radix-operator/pkg/apis/kube" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/google/uuid" + "github.com/rs/zerolog/log" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + csiVolumeTypeBlobFuse2ProtocolFuse = "csi-az-blob" + csiVolumeTypeBlobFuse2ProtocolFuse2 = "csi-blobfuse2-fuse2" + csiVolumeNameTemplate = "%s-%s-%s-%s" // --- + csiAzureKeyVaultSecretMountPathTemplate = "/mnt/azure-key-vault/%s" + volumeNameMaxLength = 63 +) + +var ( + csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} + functionalPvPhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} +) + +// GetRadixDeployComponentVolumeMounts Gets list of v1.VolumeMount for radixv1.RadixCommonDeployComponent +func GetRadixDeployComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.VolumeMount, error) { + componentName := deployComponent.GetName() + volumeMounts := make([]corev1.VolumeMount, 0) + componentVolumeMounts, err := getRadixComponentVolumeMounts(deployComponent) + if err != nil { + return nil, err + } + volumeMounts = append(volumeMounts, componentVolumeMounts...) + secretRefsVolumeMounts := getRadixComponentSecretRefsVolumeMounts(deployComponent, componentName, radixDeploymentName) + volumeMounts = append(volumeMounts, secretRefsVolumeMounts...) + return volumeMounts, nil +} + +// GetVolumes Get volumes of a component by RadixVolumeMounts +func GetVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { + var volumes []corev1.Volume + volumeMountVolumes, err := getComponentVolumeMountVolumes(deployComponent, existingVolumes) + if err != nil { + return nil, err + } + volumes = append(volumes, volumeMountVolumes...) + + storageRefsVolumes, err := getComponentSecretRefsVolumes(ctx, kubeUtil, namespace, deployComponent, radixDeploymentName) + if err != nil { + return nil, err + } + volumes = append(volumes, storageRefsVolumes...) + + return volumes, nil +} + +// GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent Garbage collect volume-mount related secrets that are no longer in the spec +func GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent, excludeSecretNames []string) error { + secrets, err := listSecretsForVolumeMounts(ctx, kubeUtil, namespace, component) + if err != nil { + return err + } + for _, secret := range secrets { + if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { + continue + } + if err := kubeUtil.DeleteSecret(ctx, namespace, secret.Name); err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + return garbageCollectSecrets(ctx, kubeUtil, namespace, secrets, excludeSecretNames) +} + +// CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent Create or update CSI Azure volume resources for a DeployComponent - PersistentVolumes, PersistentVolumeClaims, PersistentVolume +// Returns actual volumes, with existing relevant PersistentVolumeClaimName and PersistentVolumeName +func CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { + componentName := deployComponent.GetName() + actualVolumes, err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredVolumes) + if err != nil { + return nil, err + } + return actualVolumes, nil +} + +// GarbageCollectCsiAzureVolumeResourcesForDeployComponent Garbage collect CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims +func GarbageCollectCsiAzureVolumeResourcesForDeployComponent(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string) error { + currentlyUsedPvcNames, err := getCurrentlyUsedPvcNames(ctx, kubeClient, radixDeployment) + if err != nil { + return err + } + var errs []error + if err = garbageCollectCsiAzurePvcs(ctx, kubeClient, namespace, currentlyUsedPvcNames); err != nil { + errs = append(errs, err) + } + if err = garbageCollectCsiAzurePvs(ctx, kubeClient, namespace, currentlyUsedPvcNames); err != nil { + errs = append(errs, err) + } + if err = garbageCollectOrphanedCsiAzurePvs(ctx, kubeClient, currentlyUsedPvcNames); err != nil { + errs = append(errs, err) + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} + +// CreateOrUpdateVolumeMountSecrets creates or updates secrets for volume mounts +func CreateOrUpdateVolumeMountSecrets(ctx context.Context, kubeUtil *kube.Kube, appName, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { + var volumeMountSecretsToManage []string + for _, volumeMount := range volumeMounts { + secretName, accountKey, accountName := getCsiAzureVolumeMountCredsSecrets(ctx, kubeUtil, namespace, componentName, volumeMount.Name) + volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) + err := createOrUpdateCsiAzureVolumeMountsSecrets(ctx, kubeUtil, appName, namespace, componentName, &volumeMount, secretName, accountName, accountKey) + if err != nil { + return nil, err + } + } + return volumeMountSecretsToManage, nil +} + +// GetCsiAzureVolumeMountType Gets the CSI Azure volume mount type +func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radixv1.MountType { + if radixVolumeMount.BlobFuse2 == nil { + //nolint:staticcheck + return radixVolumeMount.Type + } + switch radixVolumeMount.BlobFuse2.Protocol { + case radixv1.BlobFuse2ProtocolFuse2, "": // default protocol if not set + return radixv1.MountTypeBlobFuse2Fuse2CsiAzure + default: + return "unsupported" + } +} + +func getCsiAzurePvsForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + return slice.FindAll(pvList.Items, func(pv corev1.PersistentVolume) bool { + return pvIsForCsiDriver(pv) && pvIsForNamespace(pv, namespace) && (!onlyFunctional || pvIsFunctional(pv)) + }), nil +} + +func isKnownCsiAzureVolumeMount(volumeMount string) bool { + switch volumeMount { + case string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure): + return true + } + return false +} + +func getRadixComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.VolumeMount, error) { + if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { + return nil, nil + } + + var volumeMounts []corev1.VolumeMount + for _, volumeMount := range deployComponent.GetVolumeMounts() { + name, err := GetVolumeMountVolumeName(&volumeMount, deployComponent.GetName()) + if err != nil { + return nil, err + } + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: name, MountPath: volumeMount.Path}) + } + return volumeMounts, nil +} + +func getRadixComponentSecretRefsVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent, componentName, radixDeploymentName string) []corev1.VolumeMount { + secretRefs := deployComponent.GetSecretRefs() + var volumeMounts []corev1.VolumeMount + for _, azureKeyVault := range secretRefs.AzureKeyVaults { + k8sSecretTypeMap := make(map[corev1.SecretType]bool) + for _, keyVaultItem := range azureKeyVault.Items { + kubeSecretType := kube.GetSecretTypeForRadixAzureKeyVault(keyVaultItem.K8sSecretType) + if _, ok := k8sSecretTypeMap[kubeSecretType]; !ok { + k8sSecretTypeMap[kubeSecretType] = true + } + } + for kubeSecretType := range k8sSecretTypeMap { + volumeMountName := trimVolumeNameToValidLength(kube.GetAzureKeyVaultSecretRefSecretName(componentName, radixDeploymentName, azureKeyVault.Name, kubeSecretType)) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: volumeMountName, + ReadOnly: true, + MountPath: getCsiAzureKeyVaultSecretMountPath(azureKeyVault), + }) + } + } + return volumeMounts +} + +func getCsiAzureKeyVaultSecretMountPath(azureKeyVault radixv1.RadixAzureKeyVault) string { + if azureKeyVault.Path == nil || *(azureKeyVault.Path) == "" { + return fmt.Sprintf(csiAzureKeyVaultSecretMountPathTemplate, azureKeyVault.Name) + } + return *azureKeyVault.Path +} + +func getCsiAzureVolumeMountName(componentName string, volumeMount *radixv1.RadixVolumeMount) (string, error) { + // volumeName: --- + csiVolumeType, err := getCsiRadixVolumeTypeId(volumeMount) + if err != nil { + return "", err + } + if len(volumeMount.Name) == 0 { + return "", fmt.Errorf("name is empty for volume mount in the component %s", componentName) + } + csiAzureVolumeStorageName := getRadixVolumeMountStorage(volumeMount) + if len(csiAzureVolumeStorageName) == 0 { + return "", fmt.Errorf("storage is empty for volume mount %s in the component %s", volumeMount.Name, componentName) + } + if len(volumeMount.Path) == 0 { + return "", fmt.Errorf("path is empty for volume mount %s in the component %s", volumeMount.Name, componentName) + } + return trimVolumeNameToValidLength(fmt.Sprintf(csiVolumeNameTemplate, csiVolumeType, componentName, volumeMount.Name, csiAzureVolumeStorageName)), nil +} + +func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { + if radixVolumeMount.BlobFuse2 != nil { + switch radixVolumeMount.BlobFuse2.Protocol { + case radixv1.BlobFuse2ProtocolFuse2, "": + return csiVolumeTypeBlobFuse2ProtocolFuse2, nil + default: + return "", fmt.Errorf("unknown blobfuse2 protocol %s", radixVolumeMount.BlobFuse2.Protocol) + } + } + //nolint:staticcheck + if radixVolumeMount.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure { + return csiVolumeTypeBlobFuse2ProtocolFuse, nil + } + //nolint:staticcheck + return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) +} + +func getComponentSecretRefsVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { + azureKeyVaultVolumes, err := getComponentSecretRefsAzureKeyVaultVolumes(ctx, kubeUtil, namespace, deployComponent, radixDeploymentName) + if err != nil { + return nil, err + } + return azureKeyVaultVolumes, nil +} + +func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { + secretRef := deployComponent.GetSecretRefs() + var volumes []corev1.Volume + for _, azureKeyVault := range secretRef.AzureKeyVaults { + secretProviderClassName := kube.GetComponentSecretProviderClassName(radixDeploymentName, deployComponent.GetName(), radixv1.RadixSecretRefTypeAzureKeyVault, azureKeyVault.Name) + secretProviderClass, err := kubeutil.GetSecretProviderClass(ctx, namespace, secretProviderClassName) + if err != nil { + return nil, err + } + for _, secretObject := range secretProviderClass.Spec.SecretObjects { + volumeName := trimVolumeNameToValidLength(secretObject.SecretName) + volume := corev1.Volume{ + Name: volumeName, + } + provider := string(secretProviderClass.Spec.Provider) + switch provider { + case "azure": + volume.VolumeSource.CSI = &corev1.CSIVolumeSource{ + Driver: CsiVolumeSourceDriverSecretStore, + ReadOnly: pointers.Ptr(true), + VolumeAttributes: map[string]string{CsiVolumeSourceVolumeAttributeSecretProviderClass: secretProviderClass.Name}, + } + + useAzureIdentity := azureKeyVault.UseAzureIdentity != nil && *azureKeyVault.UseAzureIdentity + if !useAzureIdentity { + azKeyVaultName, azKeyVaultNameExists := secretProviderClass.Spec.Parameters[defaults.CsiSecretProviderClassParameterKeyVaultName] + if !azKeyVaultNameExists { + return nil, fmt.Errorf("missing Azure Key vault name in the secret provider class %s", secretProviderClass.Name) + } + credsSecretName := defaults.GetCsiAzureKeyVaultCredsSecretName(deployComponent.GetName(), azKeyVaultName) + volume.VolumeSource.CSI.NodePublishSecretRef = &corev1.LocalObjectReference{Name: credsSecretName} + } + default: + log.Ctx(ctx).Error().Msgf("Not supported provider %s in the secret provider class %s", provider, secretProviderClass.Name) + continue + } + volumes = append(volumes, volume) + } + } + return volumes, nil +} + +func getComponentVolumeMountVolumes(deployComponent radixv1.RadixCommonDeployComponent, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { + componentName := deployComponent.GetName() + existingVolumeSourcesMap := getVolumesSourcesByVolumeNamesMap(existingVolumes) + var volumes []corev1.Volume + var errs []error + for _, radixVolumeMount := range deployComponent.GetVolumeMounts() { + volume, err := createVolume(radixVolumeMount, componentName, existingVolumeSourcesMap) + if err != nil { + errs = append(errs, err) + continue + } + volumes = append(volumes, *volume) + } + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + return volumes, nil +} + +func createVolume(radixVolumeMount radixv1.RadixVolumeMount, componentName string, existingVolumeSourcesMap map[string]corev1.VolumeSource) (*corev1.Volume, error) { + volumeName, err := GetVolumeMountVolumeName(&radixVolumeMount, componentName) + if err != nil { + return nil, err + } + volumeSource, err := getOrCreateVolumeSource(volumeName, componentName, radixVolumeMount, existingVolumeSourcesMap) + if err != nil { + return nil, err + } + return &corev1.Volume{ + Name: volumeName, + VolumeSource: *volumeSource, + }, nil +} + +func getOrCreateVolumeSource(volumeName string, componentName string, radixVolumeMount radixv1.RadixVolumeMount, existingVolumeSourcesMap map[string]corev1.VolumeSource) (*corev1.VolumeSource, error) { + if existingVolumeSource, ok := existingVolumeSourcesMap[volumeName]; ok { + return &existingVolumeSource, nil + } + return getVolumeSource(componentName, &radixVolumeMount) +} + +func getVolumesSourcesByVolumeNamesMap(volumes []corev1.Volume) map[string]corev1.VolumeSource { + return slice.Reduce(volumes, make(map[string]corev1.VolumeSource), func(acc map[string]corev1.VolumeSource, volume corev1.Volume) map[string]corev1.VolumeSource { + if volume.PersistentVolumeClaim != nil { + acc[volume.Name] = volume.VolumeSource + } + return acc + }) +} + +func getVolumeSource(componentName string, volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { + switch { + case volumeMount.HasDeprecatedVolume(): + return getComponentVolumeMountDeprecatedVolumeSource(componentName, volumeMount) + case volumeMount.HasBlobFuse2(): + return getCsiAzureVolumeSource(componentName, volumeMount) + case volumeMount.HasEmptyDir(): + return getComponentVolumeMountEmptyDirVolumeSource(volumeMount.EmptyDir), nil + } + return nil, fmt.Errorf("missing configuration for volumeMount %s", volumeMount.Name) +} + +func getCsiAzureVolumeSource(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { + pvcName, err := getCsiAzurePvcName(componentName, radixVolumeMount) + if err != nil { + return nil, err + } + return &corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, nil +} + +func getComponentVolumeMountDeprecatedVolumeSource(componentName string, volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { + //nolint:staticcheck + if volumeMount.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure { + return getCsiAzureVolumeSource(componentName, volumeMount) + } + //nolint:staticcheck + return nil, fmt.Errorf("unsupported volume type %s", volumeMount.Type) +} + +func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolumeMount) *corev1.VolumeSource { + return &corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: &spec.SizeLimit, + }, + } +} + +// GetVolumeMountVolumeName Gets the volume name for a volume mount +func GetVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { + switch { + case volumeMount.HasDeprecatedVolume(): + return getVolumeMountDeprecatedVolumeName(volumeMount, componentName) + case volumeMount.HasBlobFuse2(): + return getCsiAzureVolumeMountName(componentName, volumeMount) + } + return fmt.Sprintf("radix-vm-%s", volumeMount.Name), nil +} + +func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { + //nolint:staticcheck + switch volumeMount.Type { + case radixv1.MountTypeBlobFuse2FuseCsiAzure: + return getCsiAzureVolumeMountName(componentName, volumeMount) + } + //nolint:staticcheck + return "", fmt.Errorf("unsupported volume type %s", volumeMount.Type) +} + +func getCsiAzurePvcName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { + volumeName, err := getCsiAzureVolumeMountName(componentName, radixVolumeMount) + if err != nil { + return "", err + } + return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(nameRandPartLength))), nil +} + +func getCsiAzurePvName() string { + return fmt.Sprintf(csiPersistentVolumeNameTemplate, uuid.New().String()) +} + +func createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, kubeUtil *kube.Kube, appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, accountName, accountKey []byte) error { + secret := corev1.Secret{ + Type: corev1.SecretTypeOpaque, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Labels: map[string]string{ + kube.RadixAppLabel: appName, + kube.RadixComponentLabel: componentName, + kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), + kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, + }, + }, + } + + // Will need to set fake data in order to apply the secret. The user then need to set data to real values + data := make(map[string][]byte) + data[defaults.CsiAzureCredsAccountKeyPart] = accountKey + data[defaults.CsiAzureCredsAccountNamePart] = accountName + + secret.Data = data + + _, err := kubeUtil.ApplySecret(ctx, namespace, &secret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret + if err != nil { + return err + } + + return nil +} + +func pvIsForCsiDriver(pv corev1.PersistentVolume) bool { + return pv.Spec.CSI != nil +} + +func pvIsForNamespace(pv corev1.PersistentVolume, namespace string) bool { + return pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Namespace == namespace +} + +func pvIsFunctional(pv corev1.PersistentVolume) bool { + // not Terminating or Released + return slice.Any(functionalPvPhases, func(phase corev1.PersistentVolumePhase) bool { return pv.Status.Phase == phase }) +} + +func getCsiAzurePvcs(ctx context.Context, kubeClient kubernetes.Interface, namespace string) (*corev1.PersistentVolumeClaimList, error) { + return kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) +} + +func buildPvc(appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + Labels: map[string]string{ + kube.RadixAppLabel: appName, + kube.RadixComponentLabel: componentName, + kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), + kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)}, + }, + VolumeName: pvName, + StorageClassName: pointers.Ptr(""), // use "" to avoid to use the "default" storage class + VolumeMode: pointers.Ptr(corev1.PersistentVolumeFilesystem), + }, + } +} + +func getVolumeCapacity(radixVolumeMount *radixv1.RadixVolumeMount) resource.Quantity { + requestsVolumeMountSize, err := resource.ParseQuantity(getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount)) + if err != nil { + return resource.MustParse("1Mi") + } + return requestsVolumeMountSize +} + +func buildCsiAzurePv(appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) *corev1.PersistentVolume { + identityClientId := getIdentityClientId(identity) + useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount) + csiVolumeCredSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) + pv := corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: getCsiAzurePvLabels(appName, namespace, componentName, radixVolumeMount), + }, + Spec: corev1.PersistentVolumeSpec{ + StorageClassName: "", + MountOptions: getCsiAzurePvMountOptionsForAzureBlob(radixVolumeMount), + Capacity: corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)}, + AccessModes: []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)}, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: k8s.KindPersistentVolumeClaim, + Namespace: namespace, + Name: pvcName, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: provisionerBlobCsiAzure, + VolumeHandle: getVolumeHandle(namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)), + VolumeAttributes: getCsiAzurePvAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), + }, + }, + }, + } + if !useAzureIdentity { + pv.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} + } + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. + return &pv +} + +func getVolumeHandle(namespace, componentName, pvName, storageName string) string { + // Specify a value the driver can use to uniquely identify the share in the cluster. + // https://github.com/kubernetes-csi/csi-driver-smb/blob/master/docs/driver-parameters.md#pvpvc-usage + return fmt.Sprintf("%s#%s#%s#%s", namespace, componentName, pvName, storageName) +} + +func getUseAzureIdentity(identity *radixv1.Identity, radixVolumeMount *radixv1.RadixVolumeMount) bool { + if len(getIdentityClientId(identity)) == 0 { + return false + } + return radixVolumeMount != nil && radixVolumeMount.BlobFuse2 != nil && + radixVolumeMount.BlobFuse2.UseAzureIdentity != nil && *radixVolumeMount.BlobFuse2.UseAzureIdentity +} + +func getIdentityClientId(identity *radixv1.Identity) string { + if identity != nil && identity.Azure != nil && len(identity.Azure.ClientId) > 0 { + return identity.Azure.ClientId + } + return "" +} + +func getCsiAzurePvLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { + return map[string]string{ + kube.RadixAppLabel: appName, + kube.RadixNamespace: namespace, + kube.RadixComponentLabel: componentName, + kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, + } +} + +func getCsiAzurePvAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId string) map[string]string { + attributes := make(map[string]string) + switch GetCsiAzureVolumeMountType(radixVolumeMount) { + case radixv1.MountTypeBlobFuse2FuseCsiAzure: + attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + attributes[csiVolumeMountAttributeProtocol] = csiVolumeAttributeProtocolParameterFuse + attributes[csiVolumeMountAttributeSecretNamespace] = namespace + case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: + attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + attributes[csiVolumeMountAttributeProtocol] = csiVolumeAttributeProtocolParameterFuse2 + if radixVolumeMount.BlobFuse2 != nil { + if len(radixVolumeMount.BlobFuse2.StorageAccount) > 0 { + attributes[csiVolumeAttributeStorageAccount] = radixVolumeMount.BlobFuse2.StorageAccount + } + if useAzureIdentity { + attributes[csiVolumeAttributeClientID] = clientId + attributes[csiVolumeAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup + if len(radixVolumeMount.BlobFuse2.SubscriptionId) > 0 { + attributes[csiVolumeAttributeSubscriptionId] = radixVolumeMount.BlobFuse2.SubscriptionId + } + if len(radixVolumeMount.BlobFuse2.TenantId) > 0 { + attributes[csiVolumeAttributeTenantId] = radixVolumeMount.BlobFuse2.TenantId + } + } else { + attributes[csiVolumeMountAttributeSecretNamespace] = namespace + } + } + } + attributes[csiVolumeMountAttributePvName] = pvName + attributes[csiVolumeMountAttributePvcName] = pvcName + attributes[csiVolumeMountAttributePvcNamespace] = namespace + // Do not specify the key storage.kubernetes.io/csiProvisionerIdentity in csi.volumeAttributes in PV specification. This key indicates dynamically provisioned PVs + // https://github.com/kubernetes-csi/external-provisioner/blob/master/pkg/controller/controller.go#L289C5-L289C21 + // It looks like this: storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com + return attributes +} + +func getCsiAzurePvMountOptionsForAzureBlob(radixVolumeMount *radixv1.RadixVolumeMount) []string { + mountOptions := []string{ + "--file-cache-timeout-in-seconds=120", + "--use-attr-cache=true", + "--cancel-list-on-mount-seconds=0", + "-o allow_other", + "-o attr_timeout=120", + "-o entry_timeout=120", + "-o negative_timeout=120", + } + gid := getRadixBlobFuse2VolumeMountGid(radixVolumeMount) + if len(gid) > 0 { + mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiMountOptionGid, gid)) + } else { + uid := getRadixBlobFuse2VolumeMountUid(radixVolumeMount) + if len(uid) > 0 { + mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiMountOptionUid, uid)) + } + } + if getVolumeMountAccessMode(radixVolumeMount) == corev1.ReadOnlyMany { + mountOptions = append(mountOptions, ReadOnlyMountOption) + } + if radixVolumeMount.BlobFuse2 != nil { + mountOptions = append(mountOptions, getStreamingMountOptions(radixVolumeMount.BlobFuse2.Streaming)...) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiMountOptionUseAdls, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) + } + return mountOptions +} + +func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []string { + var mountOptions []string + if streaming != nil && streaming.Enabled != nil && !*streaming.Enabled { + return nil + } + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%t", csiMountOptionStreamingEnabled, true)) + if streaming == nil { + return mountOptions + } + if streaming.StreamCache != nil { + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiMountOptionStreamingCache, *streaming.StreamCache)) + } + if streaming.BlockSize != nil { + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiMountOptionStreamingBlockSize, *streaming.BlockSize)) + } + if streaming.BufferSize != nil { + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiMountOptionStreamingBufferSize, *streaming.BufferSize)) + } + if streaming.MaxBuffers != nil { + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiMountOptionStreamingMaxBuffers, *streaming.MaxBuffers)) + } + if streaming.MaxBlocksPerFile != nil { + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiMountOptionStreamingMaxBlocksPerFile, *streaming.MaxBlocksPerFile)) + } + return mountOptions +} + +func getVolumeMountAccessMode(radixVolumeMount *radixv1.RadixVolumeMount) corev1.PersistentVolumeAccessMode { + //nolint:staticcheck + accessMode := radixVolumeMount.AccessMode + if radixVolumeMount.BlobFuse2 != nil { + accessMode = radixVolumeMount.BlobFuse2.AccessMode + } + switch strings.ToLower(accessMode) { + case strings.ToLower(string(corev1.ReadWriteOnce)): + return corev1.ReadWriteOnce + case strings.ToLower(string(corev1.ReadWriteMany)): + return corev1.ReadWriteMany + case strings.ToLower(string(corev1.ReadWriteOncePod)): + return corev1.ReadWriteOncePod + } + return corev1.ReadOnlyMany // default access mode +} + +func getRadixBlobFuse2VolumeMountUid(radixVolumeMount *radixv1.RadixVolumeMount) string { + if radixVolumeMount.BlobFuse2 != nil { + return radixVolumeMount.BlobFuse2.UID + } + //nolint:staticcheck + return radixVolumeMount.UID +} + +func getRadixBlobFuse2VolumeMountGid(radixVolumeMount *radixv1.RadixVolumeMount) string { + if radixVolumeMount.BlobFuse2 != nil { + return radixVolumeMount.BlobFuse2.GID + } + //nolint:staticcheck + return radixVolumeMount.GID +} + +func getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount *radixv1.RadixVolumeMount) string { + if radixVolumeMount.BlobFuse2 != nil { + return radixVolumeMount.BlobFuse2.Container + } + //nolint:staticcheck + return radixVolumeMount.Storage +} + +func getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { + if radixVolumeMount.BlobFuse2 != nil { + return radixVolumeMount.BlobFuse2.RequestsStorage + } + //nolint:staticcheck + return radixVolumeMount.RequestsStorage +} + +func deletePvc(ctx context.Context, kubeClient kubernetes.Interface, namespace, pvcName string) error { + if len(namespace) == 0 || len(pvcName) == 0 { + log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) + return nil + } + if err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return err + } + return nil +} + +func deletePv(ctx context.Context, kubeClient kubernetes.Interface, pvName string) error { + if len(pvName) == 0 { + log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") + return nil + } + if err := kubeClient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return err + } + return nil +} + +func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { + blobFuse2VolumeMountContainer := getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + if len(blobFuse2VolumeMountContainer) != 0 { + return blobFuse2VolumeMountContainer + } + //nolint:staticcheck + return radixVolumeMount.Storage +} + +func garbageCollectOrphanedCsiAzurePvs(ctx context.Context, kubeClient kubernetes.Interface, excludePvcNames map[string]any) error { + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + var errs []error + for _, pv := range pvList.Items { + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != k8s.KindPersistentVolumeClaim || pv.Spec.CSI == nil || !knownCSIDriver(pv.Spec.CSI.Driver) { + continue + } + if !(pv.Status.Phase == corev1.VolumeReleased || pv.Status.Phase == corev1.VolumeFailed) { + continue + } + if _, ok := excludePvcNames[pv.Spec.ClaimRef.Name]; ok { + continue + } + log.Ctx(ctx).Info().Msgf("Delete orphaned Csi Azure PersistantVolume %s of PersistantVolumeClaim %s", pv.Name, pv.Spec.ClaimRef.Name) + if err := deletePv(ctx, kubeClient, pv.Name); err != nil && !k8serrors.IsNotFound(err) { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} + +func knownCSIDriver(driver string) bool { + _, ok := csiVolumeProvisioners[driver] + return ok +} + +func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace, componentName string, identity *radixv1.Identity, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { + functionalPvList, err := getCsiAzurePvsForNamespace(ctx, kubeClient, namespace, true) + if err != nil { + return nil, err + } + pvcByNameMap, err := getComponentPvcByNameMap(ctx, kubeClient, namespace, componentName) + if err != nil { + return nil, err + } + radixVolumeMountsByNameMap := getRadixVolumeMountsByNameMap(radixDeployment, componentName) + var errs []error + var volumes []corev1.Volume + for _, volume := range desiredVolumes { + if volume.PersistentVolumeClaim == nil { + volumes = append(volumes, volume) + continue + } + radixVolumeMount, existsRadixVolumeMount := radixVolumeMountsByNameMap[volume.Name] + if !existsRadixVolumeMount { + continue + } + processedVolume, err := createOrUpdateCsiAzureVolumeResourcesForVolume(ctx, kubeClient, radixDeployment, namespace, componentName, identity, volume, radixVolumeMount, functionalPvList, pvcByNameMap) + if err != nil { + errs = append(errs, err) + continue + } + if processedVolume != nil { + volumes = append(volumes, *processedVolume) + } + } + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + return volumes, nil +} + +func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { + if volume.PersistentVolumeClaim == nil { + return &volume, nil + } + appName := radixDeployment.Spec.AppName + pvcName := volume.PersistentVolumeClaim.ClaimName + existingPvc := pvcByNameMap[pvcName] + + pvName := getCsiAzurePvName() + existingPv, pvExists := getCsiAzureComponentPvByPvcName(persistentVolumes, pvcName) + if pvExists { + radixVolumeMountPv := buildCsiAzurePv(appName, namespace, componentName, existingPv.GetName(), pvcName, radixVolumeMount, identity) + if !matchComponentVolumeMountLabels(namespace, appName, componentName, radixVolumeMount, existingPv, existingPvc) || + !EqualPersistentVolumes(existingPv, radixVolumeMountPv) { + pvExists = false + newPvcName, err := getCsiAzurePvcName(componentName, radixVolumeMount) + if err != nil { + return nil, err + } + pvcName = newPvcName + } + } else { + existingPv, pvExists = getCsiAzureComponentPvByRadixVolumeMountContent(namespace, appName, componentName, radixVolumeMount, persistentVolumes, pvcName, existingPvc, identity) + } + if pvExists { + pvName = existingPv.GetName() + pvcName = existingPv.Spec.ClaimRef.Name + } + + existingPvc, pvcExists := pvcByNameMap[pvcName] + if !pvExists && pvcExists { + pvName = existingPvc.Spec.VolumeName + } + needToCreatePvc := !pvcExists + needToReCreatePv := pvExists && !pvcExists && len(existingPv.Spec.StorageClassName) > 0 // HACK: always re-create PV if it uses SC and PVC is missing + radixVolumeMountPvc := buildPvc(appName, namespace, componentName, pvName, pvcName, radixVolumeMount) + needToReCreatePvc := pvcExists && (existingPvc.Spec.VolumeName != pvName || !EqualPersistentVolumeClaims(existingPvc, radixVolumeMountPvc)) + + if needToReCreatePv || needToReCreatePvc { + newPvcName, err := getCsiAzurePvcName(componentName, radixVolumeMount) + if err != nil { + return nil, err + } + pvcName = newPvcName + + pvExists = false + pvName = getCsiAzurePvName() + } + + if !pvExists || needToReCreatePv { + log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) + pv := buildCsiAzurePv(appName, namespace, componentName, pvName, pvcName, radixVolumeMount, identity) + if _, err := kubeClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { + return nil, err + } + } + + if needToCreatePvc || needToReCreatePvc { + radixVolumeMountPvc.SetName(pvcName) + radixVolumeMountPvc.Spec.VolumeName = pvName + log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", radixVolumeMountPvc.GetName(), namespace, pvName) + if _, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, radixVolumeMountPvc, metav1.CreateOptions{}); err != nil { + return nil, err + } + } + volume.PersistentVolumeClaim.ClaimName = pvcName // in case it was updated with new name + return &volume, nil +} + +func getComponentPvcByNameMap(ctx context.Context, kubeClient kubernetes.Interface, namespace string, componentName string) (map[string]*corev1.PersistentVolumeClaim, error) { + pvcList, err := getCsiAzurePvcs(ctx, kubeClient, namespace) + if err != nil { + return nil, err + } + pvcs := slice.FindAll(pvcList.Items, func(pvc corev1.PersistentVolumeClaim) bool { + return pvc.GetLabels()[kube.RadixComponentLabel] == componentName + }) + return GetPersistentVolumeClaimMap(&pvcs), nil +} + +func getCurrentlyUsedPvcNames(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment) (map[string]any, error) { + namespace := radixDeployment.GetNamespace() + currentlyUsedPvcNames := make(map[string]any) + existingDeployments, err := kubeClient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + for _, deployment := range existingDeployments.Items { + currentlyUsedPvcNames = appendPvcNamesFromVolumes(currentlyUsedPvcNames, deployment.Spec.Template.Spec.Volumes) + } + existingJobs, err := kubeClient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + for _, job := range existingJobs.Items { + currentlyUsedPvcNames = appendPvcNamesFromVolumes(currentlyUsedPvcNames, job.Spec.Template.Spec.Volumes) + } + return currentlyUsedPvcNames, nil +} + +func appendPvcNamesFromVolumes(pvcMap map[string]any, volumes []corev1.Volume) map[string]any { + return slice.Reduce(volumes, pvcMap, func(acc map[string]any, volume corev1.Volume) map[string]any { + if volume.PersistentVolumeClaim != nil && len(volume.PersistentVolumeClaim.ClaimName) > 0 { + acc[volume.PersistentVolumeClaim.ClaimName] = struct{}{} + } + return acc + }) +} + +func garbageCollectCsiAzurePvs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, excludePvcNames map[string]any) error { + pvs, err := getCsiAzurePvsForNamespace(ctx, kubeClient, namespace, false) + if err != nil { + return err + } + for _, pv := range pvs { + if _, ok := excludePvcNames[pv.Spec.ClaimRef.Name]; ok { + continue + } + log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pv.Name, namespace) + if err = deletePv(ctx, kubeClient, pv.Name); err != nil { + return err + } + } + return nil +} +func garbageCollectCsiAzurePvcs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, excludePvcNames map[string]any) error { + pvcList, err := getCsiAzurePvcs(ctx, kubeClient, namespace) + if err != nil { + return err + } + var errs []error + for _, pvc := range pvcList.Items { + if _, ok := excludePvcNames[pvc.Name]; ok { + continue + } + log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolumeClaim %s in namespace %s", pvc.Name, namespace) + if err := deletePvc(ctx, kubeClient, namespace, pvc.Name); err != nil { + errs = append(errs, err) + continue + } + pvName := pvc.Spec.VolumeName + log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pvName, namespace) + if err := deletePv(ctx, kubeClient, pvName); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} + +func getCsiAzureComponentPvByPvcName(persistentVolumes []corev1.PersistentVolume, pvcName string) (*corev1.PersistentVolume, bool) { + pv, ok := slice.FindFirst(persistentVolumes, func(pv corev1.PersistentVolume) bool { + return pv.Spec.PersistentVolumeSource.CSI != nil && pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Name == pvcName + }) + return &pv, ok +} + +func getCsiAzureComponentPvByRadixVolumeMountContent(appName string, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, pvc *corev1.PersistentVolumeClaim, identity *radixv1.Identity) (*corev1.PersistentVolume, bool) { + for _, pv := range persistentVolumes { + if pv.Spec.PersistentVolumeSource.CSI == nil || + pv.Spec.ClaimRef == nil || + !matchComponentVolumeMountLabels(namespace, appName, componentName, radixVolumeMount, &pv, pvc) { + continue + } + radixVolumeMountPv := buildCsiAzurePv(appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) + if EqualPersistentVolumes(&pv, radixVolumeMountPv) { + return &pv, true + } + } + return nil, false +} + +func matchComponentVolumeMountLabels(namespace, appName, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClaim) bool { + pvLabels := pv.GetLabels() + if _, ok := pvLabels[kube.RadixAppLabel]; ok && matchComponentVolumeMountPvLabels(pvLabels, namespace, appName, componentName, radixVolumeMount) { + return true + } + return pvc != nil && matchComponentVolumeMountPvcLabels(pvc.GetLabels(), appName, componentName, radixVolumeMount) +} + +func matchComponentVolumeMountPvLabels(labels map[string]string, namespace, appName, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) bool { + return labels[kube.RadixNamespace] == namespace && + labels[kube.RadixAppLabel] == appName && + labels[kube.RadixComponentLabel] == componentName && + labels[kube.RadixVolumeMountNameLabel] == radixVolumeMount.Name +} + +func matchComponentVolumeMountPvcLabels(labels map[string]string, appName, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) bool { + return labels[kube.RadixAppLabel] == appName && + labels[kube.RadixComponentLabel] == componentName && + labels[kube.RadixVolumeMountNameLabel] == radixVolumeMount.Name && + labels[kube.RadixMountTypeLabel] == string(GetCsiAzureVolumeMountType(radixVolumeMount)) +} + +func getRadixVolumeMountsByNameMap(radixDeployment *radixv1.RadixDeployment, componentName string) map[string]*radixv1.RadixVolumeMount { + volumeMountsByNameMap := make(map[string]*radixv1.RadixVolumeMount) + for _, component := range radixDeployment.Spec.Components { + if findCsiAzureVolumeForComponent(volumeMountsByNameMap, component.VolumeMounts, componentName, &component) { + break + } + } + for _, component := range radixDeployment.Spec.Jobs { + if findCsiAzureVolumeForComponent(volumeMountsByNameMap, component.VolumeMounts, componentName, &component) { + break + } + } + return volumeMountsByNameMap +} + +func findCsiAzureVolumeForComponent(volumeMountsByNameMap map[string]*radixv1.RadixVolumeMount, volumeMounts []radixv1.RadixVolumeMount, componentName string, component radixv1.RadixCommonDeployComponent) bool { + if !strings.EqualFold(componentName, component.GetName()) { + return false + } + for _, radixVolumeMount := range volumeMounts { + if radixVolumeMount.BlobFuse2 == nil && !isKnownCsiAzureVolumeMount(string(GetCsiAzureVolumeMountType(&radixVolumeMount))) { + continue + } + radixVolumeMount := radixVolumeMount + volumeMountName, err := getCsiAzureVolumeMountName(componentName, &radixVolumeMount) + if err != nil { + return false + } + volumeMountsByNameMap[volumeMountName] = &radixVolumeMount + } + return true +} + +func trimVolumeNameToValidLength(volumeName string) string { + if len(volumeName) <= volumeNameMaxLength { + return volumeName + } + + randString := strings.ToLower(commonUtils.RandStringStrSeed(nameRandPartLength, volumeName)) + return fmt.Sprintf("%s-%s", volumeName[:63-nameRandPartLength-1], randString) +} + +func getCsiAzureVolumeMountCredsSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, componentName, volumeMountName string) (string, []byte, []byte) { + secretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, volumeMountName) + accountKey := []byte(defaults.SecretDefaultData) + accountName := []byte(defaults.SecretDefaultData) + if kubeUtil.SecretExists(ctx, namespace, secretName) { + oldSecret, _ := kubeUtil.GetSecret(ctx, namespace, secretName) + accountKey = oldSecret.Data[defaults.CsiAzureCredsAccountKeyPart] + accountName = oldSecret.Data[defaults.CsiAzureCredsAccountNamePart] + } + return secretName, accountKey, accountName +} + +func getLabelSelectorForCsiAzureVolumeMountSecret(component radixv1.RadixCommonDeployComponent) string { + return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) +} + +func listSecretsForVolumeMounts(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent) ([]*corev1.Secret, error) { + csiAzureVolumeMountSecret := getLabelSelectorForCsiAzureVolumeMountSecret(component) + csiSecrets, err := kubeUtil.ListSecretsWithSelector(ctx, namespace, csiAzureVolumeMountSecret) + if err != nil { + return nil, err + } + return csiSecrets, nil +} + +func garbageCollectSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace string, secrets []*corev1.Secret, excludeSecretNames []string) error { + for _, secret := range secrets { + if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { + continue + } + if err := kubeUtil.DeleteSecret(ctx, namespace, secret.GetName()); err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + return nil +} + +// GetExistingJobAuxComponentVolumes Get existing job aux component volumes +func GetExistingJobAuxComponentVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace, jobComponentName string) ([]corev1.Volume, error) { + jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobComponentName) + jobAuxKubeDeployment, err := kubeUtil.KubeClient().AppsV1().Deployments(namespace).Get(ctx, jobAuxKubeDeploymentName, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return jobAuxKubeDeployment.Spec.Template.Spec.Volumes, nil +} diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go new file mode 100644 index 000000000..4b143ec32 --- /dev/null +++ b/pkg/apis/volumemount/volumemount_test.go @@ -0,0 +1,901 @@ +//nolint:staticcheck +package volumemount + +import ( + "context" + "fmt" + "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-operator/pkg/apis/internal" + "github.com/equinor/radix-operator/pkg/apis/kube" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func (suite *TestSuite) Test_NoVolumeMounts() { + suite.T().Run("app", func(t *testing.T) { + t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories { + + component := utils.NewDeployCommonComponentBuilder(factory). + WithName("app"). + BuildComponent() + + volumeMounts, _ := GetRadixDeployComponentVolumeMounts(component, "") + assert.Equal(t, 0, len(volumeMounts)) + } + }) +} + +func (suite *TestSuite) Test_ValidBlobCsiAzureVolumeMounts() { + scenarios := []volumeMountTestScenario{ + { + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storagename1", Path: "TestPath1"}, + expectedVolumeName: "csi-az-blob-app-volume1-storagename1", + }, + { + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storagename2", Path: "TestPath2"}, + expectedVolumeName: "csi-az-blob-app-volume2-storagename2", + }, + { + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "blobstoragename-with-long-name", Path: "TestPath2"}, + expectedVolumeName: "csi-az-blob-app-volume-with-long-name-blobstoragename-wit-12345", + }, + } + suite.T().Run("One Blob CSI Azure volume mount ", func(t *testing.T) { + t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories { + t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) + component := utils.NewDeployCommonComponentBuilder(factory).WithName("app"). + WithVolumeMounts(scenarios[0].radixVolumeMount). + BuildComponent() + + volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") + assert.Nil(t, err) + assert.Equal(t, 1, len(volumeMounts), "Unexpected volume count") + if len(volumeMounts) > 0 { + mount := volumeMounts[0] + assert.Less(t, len(volumeMounts[0].Name), 64, "Volume name is too long") + assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name, "Mismatching volume names") + assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath, "Mismatching volume paths") + } + } + }) + suite.T().Run("Multiple Blob CSI Azure volume mount ", func(t *testing.T) { + t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories { + t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) + component := utils.NewDeployCommonComponentBuilder(factory). + WithName("app"). + WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount, scenarios[2].radixVolumeMount). + BuildComponent() + + volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") + assert.Equal(t, 3, len(volumeMounts), "Unexpected volume count") + assert.Nil(t, err) + for idx, testCase := range scenarios { + if len(volumeMounts) > 0 { + volumeMountName := volumeMounts[idx].Name + assert.Less(t, len(volumeMountName), 64) + if len(volumeMountName) > 60 { + assert.True(t, internal.EqualTillPostfix(testCase.expectedVolumeName, volumeMountName, nameRandPartLength), "Mismatching volume name prefixes %s and %s", volumeMountName, testCase.expectedVolumeName) + } else { + assert.Equal(t, testCase.expectedVolumeName, volumeMountName, "Mismatching volume names") + } + assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath, "Mismatching volume paths") + } + } + } + }) +} + +func (suite *TestSuite) Test_FailBlobCsiAzureVolumeMounts() { + scenarios := []volumeMountTestScenario{ + { + name: "Missed volume mount name", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Storage: "storagename1", Path: "TestPath1"}, + expectedError: "name is empty for volume mount in the component app", + }, + { + name: "Missed volume mount storage", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Path: "TestPath1"}, + expectedError: "storage is empty for volume mount volume1 in the component app", + }, + { + name: "Missed volume mount path", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storagename1"}, + expectedError: "path is empty for volume mount volume1 in the component app", + }, + } + suite.T().Run("Failing Blob CSI Azure volume mount", func(t *testing.T) { + t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories { + for _, testCase := range scenarios { + t.Logf("Test case %s for component %s", testCase.name, factory.GetTargetType()) + component := utils.NewDeployCommonComponentBuilder(factory). + WithName("app"). + WithVolumeMounts(testCase.radixVolumeMount). + BuildComponent() + + _, err := GetRadixDeployComponentVolumeMounts(component, "") + assert.NotNil(t, err) + assert.Equal(t, testCase.expectedError, err.Error()) + } + } + }) +} + +func (suite *TestSuite) Test_GetNewVolumes() { + namespace := "some-namespace" + componentName := "some-component" + suite.T().Run("No volumes in component", func(t *testing.T) { + t.Parallel() + testEnv := getTestEnv() + component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts().BuildComponent() + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) + assert.Nil(t, err) + assert.Len(t, volumes, 0) + }) + scenarios := []volumeMountTestScenario{ + { + name: "Blob CSI Azure volume", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-volume1-storage1", + expectedPvcName: "pvc-csi-az-blob-some-component-volume1-storage1-12345", + }, + { + name: "Blob CSI Azure volume with long names", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "blobstoragesame-with-long-name", Path: "path1", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-volume-with-long-name-blobstor-12345", + expectedPvcName: "pvc-csi-az-blob-some-component-volume-with-long-name-blobstor-einhp-12345", + }, + } + suite.T().Run("CSI Azure volumes", func(t *testing.T) { + t.Parallel() + testEnv := getTestEnv() + for _, scenario := range scenarios { + t.Logf("Scenario %s", scenario.name) + component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) + assert.Nil(t, err) + assert.Len(t, volumes, 1, "Unexpected volume count") + volume := volumes[0] + if len(volume.Name) > 60 { + assert.True(t, internal.EqualTillPostfix(scenario.expectedVolumeName, volume.Name, nameRandPartLength), "Mismatching volume name prefixes %s and %s", scenario.expectedVolumeName, volume.Name) + } else { + assert.Equal(t, scenario.expectedVolumeName, volume.Name, "Mismatching volume names") + } + assert.Less(t, len(volume.Name), 64, "Volume name is too long") + assert.NotNil(t, volume.PersistentVolumeClaim, "PVC is nil") + assert.True(t, internal.EqualTillPostfix(scenario.expectedPvcName, volume.PersistentVolumeClaim.ClaimName, nameRandPartLength), "Mismatching PVC name prefixes %s and %s", scenario.expectedPvcName, volume.PersistentVolumeClaim.ClaimName) + } + }) + suite.T().Run("Unsupported volume type", func(t *testing.T) { + t.Parallel() + testEnv := getTestEnv() + mounts := []radixv1.RadixVolumeMount{ + {Type: "unsupported-type", Name: "volume1", Container: "storage1", Path: "path1"}, + } + component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(mounts...).BuildComponent() + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) + assert.Len(t, volumes, 0, "Unexpected volume count") + assert.NotNil(t, err) + assert.Equal(t, "unsupported volume type unsupported-type", err.Error()) + }) +} + +func (suite *TestSuite) Test_GetCsiVolumesWithExistingPvcs() { + const ( + namespace = "any-app-some-env" + componentName = "some-component" + ) + props := getPropsCsiBlobFuse2Volume1Storage1(nil) + scenarios := []pvcTestScenario{ + { + volumeMountTestScenario: volumeMountTestScenario{ + name: "Blob CSI Azure BlobFuse2 Fuse2 volume", + radixVolumeMount: radixv1.RadixVolumeMount{Name: "volume1", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Container: "storage1", GID: "1000"}, Path: "path1"}, + expectedVolumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", + }, + pvc: createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pv: createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + { + volumeMountTestScenario: volumeMountTestScenario{ + name: "Changed container name", + radixVolumeMount: radixv1.RadixVolumeMount{Name: "volume1", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Container: "storage2", GID: "1000"}, Path: "path1"}, + expectedVolumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage2", + }, + pvc: createExpectedPvc(modifyProps(props, func(props *expectedPvcPvProperties) { + props.blobStorageName = "storage2" + props.pvcName = "pvc-csi-blobfuse2-fuse2-some-component-volume1-storage2-12345" + }), func(pvc *corev1.PersistentVolumeClaim) {}), + pv: createExpectedPv(modifyProps(props, func(props *expectedPvcPvProperties) { + props.blobStorageName = "storage2" + }), func(pv *corev1.PersistentVolume) {}), + }, + } + + suite.T().Run("CSI Azure volumes with existing PVC", func(t *testing.T) { + t.Parallel() + for _, scenario := range scenarios { + t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) + testEnv := getTestEnv() + _, err := testEnv.kubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = testEnv.kubeClient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = testEnv.kubeClient.CoreV1().PersistentVolumes().Create(context.Background(), &scenario.pv, metav1.CreateOptions{}) + require.NoError(t, err) + + component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", []corev1.Volume{{ + Name: props.volumeName, + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: scenario.pvc.Name}}, + }}) + assert.Nil(t, err) + assert.Len(t, volumes, 1) + assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name, "Mismatching volume names") + assert.NotNil(t, volumes[0].PersistentVolumeClaim, "PVC is nil") + assert.True(t, internal.EqualTillPostfix(scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName, nameRandPartLength), "Mismatching PVC name prefixes %s and %s", scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName) + } + }) + + suite.T().Run("CSI Azure volumes with no existing PVC", func(t *testing.T) { + t.Parallel() + for _, scenario := range scenarios { + t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) + testEnv := getTestEnv() + _, err := testEnv.kubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) + require.NoError(t, err) + component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) + assert.Nil(t, err) + assert.Len(t, volumes, 1, "Unexpected volume count") + assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name, "Mismatching volume names") + assert.NotNil(t, volumes[0].PersistentVolumeClaim, "Unexpected PVC") + assert.True(t, internal.EqualTillPostfix(scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName, nameRandPartLength), "Matching PVC name prefixes %s and %s", scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName) + } + }) +} + +func (suite *TestSuite) Test_GetVolumesForComponent() { + const ( + appName = "any-app" + environment = "some-env" + componentName = "some-component" + ) + namespace := fmt.Sprintf("%s-%s", appName, environment) + scenarios := []pvcTestScenario{ + { + volumeMountTestScenario: volumeMountTestScenario{ + name: "Blob CSI Azure volume, Status phase: Bound", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-blob-volume1-storage1", + expectedPvcName: "pvc-csi-az-blob-some-component-blob-volume1-storage1-12345", + }, + pvc: createPvc(namespace, componentName, radixv1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), + }, + { + volumeMountTestScenario: volumeMountTestScenario{ + name: "Blob CSI Azure volume, Status phase: Pending", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", + expectedPvcName: "pvc-csi-az-blob-some-component-blob-volume2-storage2-12345", + }, + pvc: createPvc(namespace, componentName, radixv1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), + }, + } + + suite.T().Run("No volumes", func(t *testing.T) { + t.Parallel() + testEnv := getTestEnv() + for _, factory := range suite.radixCommonDeployComponentFactories { + t.Logf("Test case for component %s", factory.GetTargetType()) + + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{}) + deployComponent := radixDeployment.Spec.Components[0] + + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, radixDeployment.GetNamespace(), &deployComponent, radixDeployment.GetName(), nil) + + assert.Nil(t, err) + assert.Len(t, volumes, 0, "No volumes should be returned") + } + }) + suite.T().Run("Exists volume", func(t *testing.T) { + t.Parallel() + testEnv := getTestEnv() + for _, factory := range suite.radixCommonDeployComponentFactories { + for _, scenario := range scenarios { + t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) + + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{scenario.radixVolumeMount}) + deployComponent := radixDeployment.Spec.Components[0] + + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, radixDeployment.GetNamespace(), &deployComponent, radixDeployment.GetName(), nil) + + assert.Nil(t, err) + assert.Len(t, volumes, 1, "Unexpected volume count") + assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name, "Mismatching volume names") + assert.NotNil(t, volumes[0].PersistentVolumeClaim, "PVC is nil") + assert.True(t, internal.EqualTillPostfix(scenario.expectedPvcName, volumes[0].PersistentVolumeClaim.ClaimName, nameRandPartLength), "Mismatching PVC name prefixes %s and %s", scenario.expectedPvcName, volumes[0].PersistentVolumeClaim.ClaimName) + } + } + }) +} + +func (suite *TestSuite) Test_GetRadixDeployComponentVolumeMounts() { + const ( + appName = "any-app" + environment = "some-env" + componentName = "some-component" + ) + scenarios := []volumeMountTestScenario{ + { + name: "Blob CSI Azure volume, Status phase: Bound", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-blob-volume1-storage1", + expectedPvcName: "pvc-csi-az-blob-some-component-blob-volume1-storage1-12345", + }, + { + name: "Blob CSI Azure volume, Status phase: Pending", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", + expectedPvcName: "pvc-csi-az-blob-some-component-blob-volume2-storage2-12345", + }, + { + name: "Blob CSI Azure volume, Status phase: Pending", + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume-with-long-name", Storage: "storage-with-long-name", Path: "path2", GID: "1000"}, + expectedVolumeName: "csi-az-blob-some-component-blob-volume-with-long-name-sto-12345", + expectedPvcName: "pvc-csi-az-blob-some-component-blob-volume-with-long-name-12345", + }, + } + + suite.T().Run("No volumes", func(t *testing.T) { + t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories { + t.Logf("Test case for component %s", factory.GetTargetType()) + + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{}) + deployComponent := radixDeployment.Spec.Components[0] + + volumes, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") + + assert.Nil(t, err) + assert.Len(t, volumes, 0, "No volumes should be returned") + } + }) + suite.T().Run("Exists volume", func(t *testing.T) { + t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories { + for _, scenario := range scenarios { + t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) + + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{scenario.radixVolumeMount}) + deployComponent := radixDeployment.Spec.Components[0] + + volumeMounts, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") + + assert.Nil(t, err) + assert.Len(t, volumeMounts, 1) + volumeMountName := volumeMounts[0].Name + if len(volumeMountName) > 60 { + assert.True(t, internal.EqualTillPostfix(scenario.expectedVolumeName, volumeMountName, nameRandPartLength), "Mismatching volume name prefixes %s and %s", scenario.expectedVolumeName, volumeMountName) + } else { + assert.Equal(t, scenario.expectedVolumeName, volumeMountName) + } + assert.Less(t, len(volumeMountName), 64, "Volume name is too long") + assert.Equal(t, scenario.radixVolumeMount.Path, volumeMounts[0].MountPath, "Mismatching volume paths") + } + } + }) +} + +func (suite *TestSuite) Test_CreateOrUpdateCsiAzureResources() { + var scenarios []deploymentVolumesTestScenario + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, nil), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.ClaimRef.Name = scenarioProps.expectedNewPvcName + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createRandomPvc(props, props.namespace, props.componentName) + expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }) + existingPv := createRandomPv(props, props.namespace, props.componentName) + expectedPv := createExpectedPv(props, nil) + return deploymentVolumesTestScenario{ + name: "Set readonly volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(func(props *expectedPvcPvProperties) { + props.readOnly = false + })), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + expectedPv := modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props) + }) + expectedPvc := modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteMany volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(modifyProps(props, func(props *expectedPvcPvProperties) { props.readOnly = false })) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + existingPv = modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(modifyProps(props, func(props *expectedPvcPvProperties) { props.readOnly = false })) + }) + existingPvc = modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }) + return deploymentVolumesTestScenario{ + name: "Set ReadOnlyMany volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &radixv1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has disabled streaming", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &radixv1.RadixVolumeMountStreaming{ + Enabled: pointers.Ptr(false), + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + volume := createTestVolume(props, func(v *corev1.Volume) {}) + existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef.Name = volume.PersistentVolumeClaim.ClaimName }) + expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + matchPvAndPvc(&expectedPv, &expectedPvc) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume with class name, when creating new PVC", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRandomVolumeMount(func(vm *radixv1.RadixVolumeMount) { vm.Name = anotherVolumeMountName }), + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + volume, + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + expectedPvc, + pvcForAnotherComponent, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + expectedPvs: []corev1.PersistentVolume{ + expectedPv, + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume without class name, when creating new PVC", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + expectedPvc := createRandomPvc(props, props.namespace, componentName1) + expectedPv := createRandomPv(props, props.namespace, componentName1) + matchPvAndPvc(&expectedPv, &expectedPvc) + return deploymentVolumesTestScenario{ + name: "Do not change existing PVC with class name, when creating new PersistentVolume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + pvForAnotherComponent, + }, + expectedPvs: []corev1.PersistentVolume{ + pvForAnotherComponent, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + + suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { + for _, factory := range suite.radixCommonDeployComponentFactories { + for _, scenario := range scenarios { + t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) + testEnv := getTestEnv() + radixDeployment := buildRd(appName1, envName1, componentName1, scenario.radixVolumeMounts) + putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) + desiredVolumes := getDesiredDeployment(componentName1, scenario.volumes).Spec.Template.Spec.Volumes + + deployComponent := radixDeployment.Spec.Components[0] + actualVolumes, err := CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName1, envName1), &deployComponent, desiredVolumes) + require.NoError(t, err) + assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") + + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) + require.NoError(t, err) + assert.Len(t, existingPvcs, len(scenario.expectedPvcs), "PVC-s count is not equal") + assert.True(t, equalPersistentVolumeClaims(&scenario.expectedPvcs, &existingPvcs), "PVC-s are not equal") + assert.Len(t, existingPvs, len(scenario.expectedPvs), "PV-s count is not equal") + assert.True(t, equalPersistentVolumes(&scenario.expectedPvs, &existingPvs), "PV-s are not equal") + } + } + }) +}