From 676ad8995f444d7f6a614542e561c77c5e120356 Mon Sep 17 00:00:00 2001 From: Mulder Date: Thu, 9 Jan 2025 08:28:21 -0800 Subject: [PATCH] DBC22-2470: Improve Disaster Recovery Process --- .../templates/PostgresCluster.yaml | 18 +++ .../crunchy-postgres/values-dev-recovery.yaml | 126 +++++++++++++++ .../crunchy-postgres/values-dev.yaml | 16 ++ .../crunchy-postgres/values-prod.yaml | 16 ++ .../crunchy-postgres/values-test.yaml | 16 ++ .../crunchy-postgres/values-uat.yaml | 15 ++ infrastructure/main/values-dev-recovery.yaml | 147 ++++++++++++++++++ 7 files changed, 354 insertions(+) create mode 100644 infrastructure/crunchy-postgres/values-dev-recovery.yaml create mode 100644 infrastructure/main/values-dev-recovery.yaml diff --git a/infrastructure/crunchy-postgres/templates/PostgresCluster.yaml b/infrastructure/crunchy-postgres/templates/PostgresCluster.yaml index 2d55211ee..7159ff96d 100644 --- a/infrastructure/crunchy-postgres/templates/PostgresCluster.yaml +++ b/infrastructure/crunchy-postgres/templates/PostgresCluster.yaml @@ -72,6 +72,24 @@ spec: databases: - {{ template "crunchy-postgres.fullname" . }} + {{ if .Values.dataSource.enabled }} + dataSource: + pgbackrest: + configuration: + - secret: + name: {{ .Values.dataSource.secretName }} + global: + repo2-path: {{ .Values.dataSource.repo.path }} + repo2-s3-uri-style: {{ .Values.dataSource.repo.s3UriStyle }} + repo: + name: {{ .Values.dataSource.repo.name }} + s3: + bucket: {{ .Values.dataSource.repo.s3.bucket }} + endpoint: {{ .Values.dataSource.repo.s3.endpoint }} + region: {{ .Values.dataSource.repo.s3.region }} + stanza: {{ .Values.dataSource.stanza }} + {{ end }} + backups: pgbackrest: {{ if .Values.pgBackRest.image }} diff --git a/infrastructure/crunchy-postgres/values-dev-recovery.yaml b/infrastructure/crunchy-postgres/values-dev-recovery.yaml new file mode 100644 index 000000000..6300a4697 --- /dev/null +++ b/infrastructure/crunchy-postgres/values-dev-recovery.yaml @@ -0,0 +1,126 @@ +#use this helm chart if you need to recover dev. It has dataSource set to enabled and s3 backup set to false. +fullnameOverride: dev-drivebc + +#crunchyImage: artifacts.developer.gov.bc.ca/bcgov-docker-local/crunchy-postgres-gis:ubi8-15.2-3.3-0 + +postgresVersion: 15 +postGISVersion: '3.3' + +instances: + name: ha # high availability + replicas: 1 + dataVolumeClaimSpec: + storage: 5Gi + storageClassName: netapp-block-standard + requests: + cpu: 50m + memory: 300Mi + replicaCertCopy: + requests: + cpu: 1m + memory: 16Mi + +# If we need to restore the cluster from a backup, we need to set the following values +# assuming restore from repo2 (s3), adjust as needed if your S3 repo is different +dataSource: + enabled: true + # should have the same name and contain the same keys as the pgbackrest secret + secretName: s3-pgbackrest + repo: + name: repo2 + path: "/db/habackup" + s3UriStyle: path + s3: + bucket: "" + endpoint: "" + region: "ca-central-1" #this can be whatever if using BC Gov object storage + stanza: db + +pgBackRest: + image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default + retention: "2" # Set to 2 days for PVC backups + # If retention-full-type set to 'count' then the oldest backups will expire when the number of backups reach the number defined in retention + # If retention-full-type set to 'time' then the number defined in retention will take that many days worth of full backups before expiration + retentionFullType: time + repos: + schedules: + full: 0 8 * * * + incremental: 0 0,4,12,16,20 * * * + volume: + accessModes: "ReadWriteOnce" + storage: 5Gi + storageClassName: netapp-file-backup + repoHost: + requests: + cpu: 5m + memory: 16Mi + sidecars: + pgbackrest: + requests: + cpu: 5m + memory: 90Mi + pgbackrestConfig: + requests: + cpu: 1m + memory: 16Mi + s3: + enabled: false + retention: "7" # Set to 7 days for S3 storage. + retentionFullType: time + createS3Secret: true #Will create the secret if it doesn't already exist. NOTE: Once the secret is set, you must change it in OpenShift. + # the s3 secret name + s3Secret: dev-drivebc-s3-pgbackrest + # the path start with /, it will be created under bucket if it doesn't exist + s3Path: "/db/habackup2" + # s3UriStyle is host or path + s3UriStyle: path + # bucket specifies the S3 bucket to use, + bucket: "" + # endpoint specifies the S3 endpoint to use. + endpoint: "" + # region specifies the S3 region to use. If your S3 storage system does not + # use "region", fill this in with a random value. + region: "ca-central-1" + # key is the S3 key. This is stored in a Secret. + # Please DO NOT push this value to GitHub + key: "s3keyValue" + # keySecret is the S3 key secret. This is stored in a Secret. + # Please DO NOT push this value to GitHub + keySecret: "s3SecretValue" + # setting the below to be one plus of the default schedule + # to avoid conflicts + fullSchedule: "0 9 * * *" + incrementalSchedule: "30 0,4,12,16,20 * * *" + +patroni: + postgresql: + pg_hba: "host all all 0.0.0.0/0 md5" + parameters: + shared_buffers: 16MB # default is 128MB; a good tuned default for shared_buffers is 25% of the memory allocated to the pod + wal_buffers: "64kB" # this can be set to -1 to automatically set as 1/32 of shared_buffers or 64kB, whichever is larger + min_wal_size: 32MB + max_wal_size: 64MB # default is 1GB + max_slot_wal_keep_size: 128MB # default is -1, allowing unlimited wal growth when replicas fall behind + +proxy: + pgBouncer: + image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default + replicas: 1 + requests: + cpu: 2m + memory: 16Mi + pgbouncerConfig: + requests: + cpu: 1m + memory: 16Mi + + +# Postgres Cluster resource values: +pgmonitor: + enabled: true #Can be true or false + namespace: a781ec #The high level namespace of your project without the -tools -dev, etc part. + exporter: + image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default + requests: + cpu: 10m + memory: 64Mi diff --git a/infrastructure/crunchy-postgres/values-dev.yaml b/infrastructure/crunchy-postgres/values-dev.yaml index 112507f45..682546202 100644 --- a/infrastructure/crunchy-postgres/values-dev.yaml +++ b/infrastructure/crunchy-postgres/values-dev.yaml @@ -19,6 +19,22 @@ instances: cpu: 1m memory: 16Mi +# If we need to restore the cluster from a backup, we need to set the following values +# assuming restore from repo2 (s3), adjust as needed if your S3 repo is different +dataSource: + enabled: false + # should have the same name and contain the same keys as the pgbackrest secret + secretName: s3-pgbackrest + repo: + name: repo2 + path: "/db/habackup" + s3UriStyle: path + s3: + bucket: "" + endpoint: "" + region: "ca-central-1" #this can be whatever if using BC Gov object storage + stanza: db + pgBackRest: image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default retention: "2" # Set to 2 days for PVC backups diff --git a/infrastructure/crunchy-postgres/values-prod.yaml b/infrastructure/crunchy-postgres/values-prod.yaml index 0c175dc19..5972f8855 100644 --- a/infrastructure/crunchy-postgres/values-prod.yaml +++ b/infrastructure/crunchy-postgres/values-prod.yaml @@ -19,6 +19,22 @@ instances: cpu: 1m memory: 16Mi +# If we need to restore the cluster from a backup, we need to set the following values +# assuming restore from repo2 (s3), adjust as needed if your S3 repo is different +dataSource: + enabled: false + # should have the same name and contain the same keys as the pgbackrest secret + secretName: s3-pgbackrest + repo: + name: repo2 + path: "/db/habackup" + s3UriStyle: path + s3: + bucket: "" + endpoint: "" + region: "ca-central-1" #this can be whatever if using BC Gov object storage + stanza: db + pgBackRest: image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default retention: "2" # Set to 2 days for PVC backups diff --git a/infrastructure/crunchy-postgres/values-test.yaml b/infrastructure/crunchy-postgres/values-test.yaml index d1bce3acb..3e6bf1c2c 100644 --- a/infrastructure/crunchy-postgres/values-test.yaml +++ b/infrastructure/crunchy-postgres/values-test.yaml @@ -19,6 +19,22 @@ instances: cpu: 1m memory: 16Mi +# If we need to restore the cluster from a backup, we need to set the following values +# assuming restore from repo2 (s3), adjust as needed if your S3 repo is different +dataSource: + enabled: false + # should have the same name and contain the same keys as the pgbackrest secret + secretName: s3-pgbackrest + repo: + name: repo2 + path: "/db/habackup" + s3UriStyle: path + s3: + bucket: "" + endpoint: "" + region: "ca-central-1" #this can be whatever if using BC Gov object storage + stanza: db + pgBackRest: image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default retention: "2" # Set to 2 days for PVC backups diff --git a/infrastructure/crunchy-postgres/values-uat.yaml b/infrastructure/crunchy-postgres/values-uat.yaml index 305df80b2..ade0528b0 100644 --- a/infrastructure/crunchy-postgres/values-uat.yaml +++ b/infrastructure/crunchy-postgres/values-uat.yaml @@ -19,6 +19,21 @@ instances: cpu: 1m memory: 16Mi +# If we need to restore the cluster from a backup, we need to set the following values +# assuming restore from repo2 (s3), adjust as needed if your S3 repo is different +dataSource: + enabled: false + # should have the same name and contain the same keys as the pgbackrest secret + secretName: s3-pgbackrest + repo: + name: repo2 + path: "/db/habackup" + s3UriStyle: path + s3: + bucket: "" + endpoint: "" + region: "ca-central-1" #this can be whatever if using BC Gov object storage + stanza: db pgBackRest: image: # it's not necessary to specify an image as the images specified in the Crunchy Postgres Operator will be pulled by default diff --git a/infrastructure/main/values-dev-recovery.yaml b/infrastructure/main/values-dev-recovery.yaml new file mode 100644 index 000000000..3a48cd8d6 --- /dev/null +++ b/infrastructure/main/values-dev-recovery.yaml @@ -0,0 +1,147 @@ +#Use this chart if you need to recover dev into a different namespace. Should be identical to the main helm chart, except the URL's might be slightly different. +nameOverride: dev-drivebc +fullnameOverride: dev-drivebc + +django: + fullnameOverride: dev-django + nameOverride: dev-django + replicaCount: 1 + image: + repository: ghcr.io/bcgov/drivebc-django + tag: latest-dev + deployment: + resources: + requests: + cpu: 20m + memory: 200Mi + env: + postgresSecret: dev-drivebc-pguser-dev-drivebc + djangoConfigMap: dev-drivebc-django + djangoSecret: dev-drivebc-django + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 2 + targetCPUUtilizationPercentage: 75 + route: + host: dev-drivebc.apps.golddr.devops.gov.bc.ca + iprestricted: false #Set to true if you want to limit IP's that can access publicly available pages to the the addresses in the ipallowlist + iprestrictedAdminPages: false #Set to true if you want to limit IP's that can access backend admin pages + mediapvc: + storage: 1Gi + apppvc: + storage: 1Gi + podDisruptionBudget: + enabled: false + minAvailable: + + +tasks: + fullnameOverride: dev-tasks + nameOverride: dev-tasks + replicaCount: 1 + image: + repository: ghcr.io/bcgov/drivebc-django + tag: latest-dev + deployment: + resources: + requests: + cpu: 200m + memory: 250Mi + volumes: + claimName: dev-django-app-images + env: + postgresSecret: dev-drivebc-pguser-dev-drivebc + djangoConfigMap: dev-drivebc-django + djangoSecret: dev-drivebc-django + +redis: + fullnameOverride: dev-redis + nameOverride: dev-redis + replicaCount: 1 + image: + repository: ghcr.io/bcgov/drivebc-redis + tag: latest-dev + deployment: + resources: + requests: + cpu: 5m + memory: 16Mi + +static: + fullnameOverride: dev-static + nameOverride: dev-static + replicaCount: 1 + release: + image: + repository: ghcr.io/bcgov/drivebc-static + tag: latest-dev + deployment: + resources: + requests: + cpu: 5m + memory: 50Mi + env: + staticConfigMap: dev-drivebc-static + volumes: + imagesClaimName: dev-django-app-images + + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 2 + targetCPUUtilizationPercentage: 75 + route: + host: dev-drivebc.apps.golddr.devops.gov.bc.ca + iprestricted: false #Set to true if you want to limit IP's the the addresses in the ipallowlist + logpvc: + storage: 1Gi + podDisruptionBudget: + enabled: false + minAvailable: + +openshiftjobs: + fullnameOverride: dev-openshiftjobs + nameOverride: dev-openshiftjobs + + image: + repository: ghcr.io/bcgov/drivebc-openshiftjobs + tag: latest-dev + + cronjobs: + analyzeuploadlogs: + name: analyzeuploadlogs + schedule: '0 9 * * *' #NOTE: This is in UTC + deployment: + resources: + requests: + cpu: 50m + memory: 150Mi + env: + s3Secret: dev-drivebc-cronjob-s3bucket + volumes: + logs: dev-static-log-storage + + ziplogs: + name: ziplogs + schedule: '30 * * * *' + deployment: + resources: + requests: + cpu: 50m + memory: 100Mi + volumes: + logs: dev-static-log-storage + + backupmediapvc: + name: backupmediapvc + schedule: '0 8 * * *' + deployment: + resources: + requests: + cpu: 50m + memory: 100Mi + env: + s3Secret: dev-drivebc-cronjob-s3bucket + volumes: + media: dev-django \ No newline at end of file