fix: Improve options to configure aws rds secret and region

docs/functions.md

@@ -964,7 +964,7 @@ Arguments:
   | instanceID | Yes      | string | ID of RDS instance you want to create snapshot of |
   | dbEngine   | No       | string | Required in case of RDS Aurora instance. Supported DB Engines: `aurora` `aurora-mysql` and `aurora-postgresql` |
   | credentialsSource | No | string | Source for aws credentials. Supported sources: `profile`, `secret`, `serviceaccount`. Default value is `profile` |
-  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be referenced in the Actionset |
+  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be referenced in the Actionset or `objects` in the blueprint phase |
   | region | No | string | AWS region to use. Derived from profile or serviceaccount if not set |
 
 ::: tip NOTE
@@ -1029,7 +1029,7 @@ Arguments:
   | podAnnotations       | No       | map[string]string       | custom annotations for the temporary pod that gets created |
   | podLabels            | No       | map[string]string       | custom labels for the temporary pod that gets created |
   | credentialsSource | No | string | Source for aws credentials. Supported sources: `profile`, `secret`, `serviceaccount`. Default value is `profile` |
-  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be [referenced in the Actionset](templates.html#secrets) |
+  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be [referenced in the Actionset](templates.html#secrets) or `objects` in the blueprint phase |
   | region | No | string | AWS region to use. Derived from profile or serviceaccount if not set |
 
 
@@ -1148,7 +1148,7 @@ Arguments:
   | podAnnotations       | No       | map[string]string       | custom annotations for the temporary pod that gets created |
   | podLabels            | No       | map[string]string       | custom labels for the temporary pod that gets created |
   | credentialsSource | No | string | Source for aws credentials. Supported sources: `profile`, `secret`, `serviceaccount`. Default value is `profile` |
-  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be referenced in the Actionset |
+  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be referenced in the Actionset or `objects` in the blueprint phase |
   | region | No | string | AWS region to use. Derived from profile or serviceaccount if not set |
 
 
@@ -1215,7 +1215,7 @@ Arguments:
   | ---------- | :------: | ------ | ----------- |
   | snapshotID | No       | string | ID of the RDS snapshot |
   | credentialsSource | No | string | Source for aws credentials. Supported sources: `profile`, `secret`, `serviceaccount`. Default value is `profile` |
-  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be referenced in the Actionset |
+  | credentialsSecret | No | string | Secret to get credentials from. Only used with `credentialsSource: secret`. Secret with this name should be referenced in the Actionset or `objects` in the blueprint phase |
   | region | No | string | AWS region to use. Derived from profile or serviceaccount if not set |
 
 ::: tip NOTE

examples/aws-rds/postgresql/README.md

@@ -41,31 +41,81 @@ aws rds create-db-instance \
 aws rds wait db-instance-available --db-instance-identifier=test-postgresql-instance
 ```
 
+## Create DB secret
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: dbcreds
+  namespace: pgtestrds
+type: Opaque
+stringData:
+  username: <username>
+  password: <password>
+```
+
+This secret needs to be referenced in the configmap.
+
+
+### Configure a secret to access RDS
+
+By default the blueprints in this example are using credentials from the profile to access RDS
+resources.
+
+If you want to export your backups to a different region or a different object store (recommended),
+you need to provide alternative credential configuration.
+
+Here we use k8s secret with AWS credentials.
+You need to make sure credentials provided in this secret can be used to access RDS operations.
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: rds-secret
+  namespace: pgtestrds
+type: secrets.kanister.io/aws
+stringData:
+  aws_access_key_id: "<your access key id>"
+  aws_secret_access_key: "<you secret>"
+  role: ""
+```
+
+This secret needs to be referenced in the configmap.
+
+
 ## Create configmap
 
 Create a configmap which contains information to connect to the RDS DB instance
 
+**NOTE** If you want to export your backups to a different region or a different object store (recommended) you need to also set the region where your RDS instance is deployed with `aws.region` parameter below. You don't need this setting otherwise.
+
 ```yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   annotations:
     kasten.io/config: dataservice
   name: dbconfig
+  namespace: pgtestrds
 data:
   postgres.instanceid: test-rds-postgresql
   postgres.host: test-rds-postgresql.example.ap-south-1.rds.amazonaws.com
   postgres.databases: |
     - postgres
     - template1
-  postgres.secret: dbcreds # name of K8s secret in the same namespace
+  postgres.secret: dbcreds # name of K8s secret in the same namespace with postgres credentials
+  aws.secret: rds-secret # name of K8s secret in the same namespace with aws credentials
+  aws.region: <aws region>
 ```
 
 ## Integrating with Kanister
 
 ### Create Profile
 
-Create Profile CR if not created already
+Create Profile CR if not created already.
 
 ```bash
 $ kanctl create profile s3compliant --access-key <aws-access-key-id> \
@@ -82,33 +132,6 @@ data operations such as backup should go. This is stored as a `profiles.cr.kanis
 requires a Profile reference to complete the action. This CR (`profiles.cr.kanister.io`)
 can be shared between Kanister-enabled application instances.
 
-### Configure a secret to access RDS
-
-By default the blueprints in this example are using credentials from the profile to access RDS
-resources.
-
-If you want to export your backups to a different region or a different object store (recommended),
-you need to provide alternative credential configuration.
-
-Here we use k8s secret with AWS credentials.
-You need to make sure credentials provided in this secret can be used to access RDS operations.
-
-```yaml
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: rds-secret
-  namespace: pgtestrds
-type: secrets.kanister.io/aws
-data:
-  aws_access_key_id: "<your access key id>"
-  aws_secret_access_key: "<you secret>"
-  role: ""
-```
-
-This secret needs to be referenced in the acitonset.
-
 ### Create Blueprint
 
 There are two ways that you can use to backup and restore RDS instance data:
@@ -150,14 +173,13 @@ Create actionset file:
 
 > Use correct blueprint name (one of `rds-postgres-dump-bp` or `rds-postgres-snapshot-bp`) you have created earlier
 > If you have deployed your application which uses RDS instance in namespace other than `pgtestrds`, you need to modify the commands used below to use the correct namespace
-> Please make sure `region` option corresponds to the AWS region where your RDS is deployed.
 
 ```yaml
 apiVersion: cr.kanister.io/v1alpha1
 kind: ActionSet
 metadata:
   name: rds-backup
-  namespace: kasten-io
+  namespace: kanister
 spec:
   actions:
   - name: backup
@@ -170,19 +192,11 @@ spec:
     profile:
       name: <your profile>
       namespace: pgtestrds
-    secrets:
-      aws:
-        name: rds-secret
-        namespace: pgtestrds
-    options:
-      region: <rds region>
 ```
 
 Where:
 - dbconfig is a configmap holding RDS infromation
   Please see pgtest/deploy/config.yaml for configmap format
-- rds-secret is an AWS secret with access to RDS resources
-
 
 Apply actionset:
 

examples/aws-rds/postgresql/rds-postgres-dump-blueprint.yaml

@@ -15,18 +15,27 @@ actions:
     phases:
     - func: CreateRDSSnapshot
       name: createSnapshot
+      objects:
+        awssecret:
+          kind: Secret
+          name: '{{ index .Object.data "aws.secret" }}'
+          namespace: "{{ .Object.metadata.namespace }}"
       args:
         instanceID: '{{ index .Object.data "postgres.instanceid" }}'
         credentialsSource: secret
-        credentialsSecret: aws
-        region: '{{ .Options.region }}'
+        credentialsSecret: awssecret
+        region: '{{ index .Object.data "aws.region" }}'
     - func: ExportRDSSnapshotToLocation
       name: exportSnapshot
       objects:
         dbsecret:
           kind: Secret
           name: '{{ index .Object.data "postgres.secret" }}'
           namespace: "{{ .Object.metadata.namespace }}"
+        awssecret:
+          kind: Secret
+          name: '{{ index .Object.data "aws.secret" }}'
+          namespace: "{{ .Object.metadata.namespace }}"
       args:
         namespace: "{{ .Object.metadata.namespace }}"
         instanceID: "{{ .Phases.createSnapshot.Output.instanceID }}"
@@ -39,15 +48,20 @@ actions:
         backupArtifactPrefix: test-postgresql-instance/postgres
         dbSubnetGroup: "{{ .Phases.createSnapshot.Output.dbSubnetGroup }}"
         credentialsSource: secret
-        credentialsSecret: aws
-        region: '{{ .Options.region }}'
+        credentialsSecret: awssecret
+        region: '{{ index .Object.data "aws.region" }}'
     - func: DeleteRDSSnapshot
       name: deleteSnapshot
+      objects:
+        awssecret:
+          kind: Secret
+          name: '{{ index .Object.data "aws.secret" }}'
+          namespace: "{{ .Object.metadata.namespace }}"
       args:
         snapshotID: "{{ .Phases.createSnapshot.Output.snapshotID }}"
         credentialsSource: secret
-        credentialsSecret: aws
-        region: '{{ .Options.region }}'
+        credentialsSecret: awssecret
+        region: '{{ index .Object.data "aws.region" }}'
   restore:
     inputArtifactNames:
     - backupInfo
@@ -59,6 +73,10 @@ actions:
           kind: Secret
           name: '{{ index .Object.data "postgres.secret" }}'
           namespace: "{{ .Object.metadata.namespace }}"
+        awssecret:
+          kind: Secret
+          name: '{{ index .Object.data "aws.secret" }}'
+          namespace: "{{ .Object.metadata.namespace }}"
       args:
         namespace: "{{ .Object.metadata.namespace }}"
         backupArtifactPrefix: test-postgresql-instance/postgres
@@ -70,8 +88,8 @@ actions:
         dbEngine: "PostgreSQL"
         dbSubnetGroup: "{{ .ArtifactsIn.backupInfo.KeyValue.dbSubnetGroup }}"
         credentialsSource: secret
-        credentialsSecret: aws
-        region: '{{ .Options.region }}'
+        credentialsSecret: awssecret
+        region: '{{ index .Object.data "aws.region" }}'
   delete:
     phases:
     - func: KubeTask

pkg/app/rds_postgres.go

@@ -214,6 +214,8 @@ func (pdb *RDSPostgresDB) Install(ctx context.Context, ns string) error {
 			"postgres.databases":  makeYamlList(pdb.databases),
 			"postgres.user":       pdb.username,
 			"postgres.secret":     pdb.secretName,
+			"aws.secret":          pdb.secretName, // Using same secret for postgres and aws
+			"aws.region":          pdb.region,
 		},
 	}
 	_, err = pdb.cli.CoreV1().ConfigMaps(ns).Create(ctx, dbconfig, metav1.CreateOptions{})

pkg/function/rds_utils.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/kanisterio/kanister/pkg/aws"
 	"github.com/kanisterio/kanister/pkg/aws/rds"
+	"github.com/kanisterio/kanister/pkg/log"
 	"github.com/kanisterio/kanister/pkg/param"
 	"github.com/kanisterio/kanister/pkg/secrets"
 )
@@ -71,8 +72,8 @@ func getAwsConfig(ctx context.Context, credentialsSource awsCredentialsSource, t
 		// Get aws config from profile
 		return getAWSConfigFromProfile(ctx, profile, credentialsSource.region)
 	case CredentialSourceSecret:
-		if secret, ok := tp.Secrets[credentialsSource.secret]; ok {
-			return getAWSConfigFromSecret(ctx, secret, credentialsSource.region)
+		if secret, ok := getParamSecret(tp, credentialsSource.secret); ok {
+			return getAWSConfigFromSecret(ctx, *secret, credentialsSource.region)
 		}
 		return nil, "", errkit.New("Cannot find secret in actionset secrets", "secret", credentialsSource.secret)
 	case CredentialSourceServiceAccount:
@@ -82,6 +83,22 @@ func getAwsConfig(ctx context.Context, credentialsSource awsCredentialsSource, t
 	}
 }
 
+// getParamSecret tries to get secret with a specified name in actionset or phase secrets
+func getParamSecret(tp param.TemplateParams, secretName string) (*corev1.Secret, bool) {
+	// Actionset secrets
+	if secret, ok := tp.Secrets[secretName]; ok {
+		return &secret, ok
+	}
+	// Technically tp.CurrentPhase should always be set
+	if tp.CurrentPhase == nil {
+		log.Info().Print("WARNING: CurrentPhase is not set in phase execution!")
+		return nil, false
+	} else if secret, ok := tp.CurrentPhase.Secrets[secretName]; ok {
+		return &secret, ok
+	}
+	return nil, false
+}
+
 func getAWSConfigFromProfile(ctx context.Context, profile *param.Profile, region string) (*awssdk.Config, string, error) {
 	// Validate profile secret
 	config := make(map[string]string)
@@ -103,11 +120,6 @@ func getAWSConfigFromProfile(ctx context.Context, profile *param.Profile, region
 }
 
 func getAWSConfigFromSecret(ctx context.Context, secret corev1.Secret, region string) (*awssdk.Config, string, error) {
-	err := secrets.ValidateAWSCredentials(&secret)
-	if err != nil {
-		return nil, "", errkit.Wrap(err, "Invalid AWS credential")
-	}
-
 	config := map[string]string{
 		aws.AccessKeyID:     string(secret.Data[secrets.AWSAccessKeyID]),
 		aws.SecretAccessKey: string(secret.Data[secrets.AWSSecretAccessKey]),

pkg/param/param.go

@@ -57,6 +57,7 @@ type TemplateParams struct {
 	RepositoryServer *RepositoryServer
 	Options          map[string]string
 	Object           map[string]interface{}
+	CurrentPhase     *Phase
 	Phases           map[string]*Phase
 	DeferPhase       *Phase
 	PodOverride      crv1alpha1.JSONMap

pkg/phase.go

@@ -78,6 +78,8 @@ func (p *Phase) Exec(ctx context.Context, bp crv1alpha1.Blueprint, action string
 			return nil, err
 		}
 	}
+	// To simplify usage of the phase secrets in phase functions
+	tp.CurrentPhase = tp.Phases[p.name]
 	// Execute the function
 	return p.f.Exec(ctx, tp, p.args)
 }