From def9c3356d7f5c4af4c6d03fa5ae0d6be1844c08 Mon Sep 17 00:00:00 2001 From: binbin21avery <38284249+binbin21avery@users.noreply.github.com> Date: Thu, 25 Jan 2024 11:02:00 +0800 Subject: [PATCH 1/2] Create backup functionality. --- .../dataprotection/backup-info-collector.sh | 29 ++++++++++++++++ .../dataprotection/halo-basebackup-backup.sh | 13 ++++++++ .../templates/actionset-halobasebackup.yaml | 33 +++++++++++++++++++ .../halo/templates/backuppolicytemplate.yaml | 25 ++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 addons/halo/dataprotection/backup-info-collector.sh create mode 100644 addons/halo/dataprotection/halo-basebackup-backup.sh create mode 100644 addons/halo/templates/actionset-halobasebackup.yaml create mode 100644 addons/halo/templates/backuppolicytemplate.yaml diff --git a/addons/halo/dataprotection/backup-info-collector.sh b/addons/halo/dataprotection/backup-info-collector.sh new file mode 100644 index 000000000..3fabe21fc --- /dev/null +++ b/addons/halo/dataprotection/backup-info-collector.sh @@ -0,0 +1,29 @@ +function get_current_time() { + curr_time=$(psql -U ${DP_DB_USER} -d halo0root -t -c "SELECT now() AT TIME ZONE 'UTC'") + echo $curr_time +} + +function stat_and_save_backup_info() { + export PATH="$PATH:$DP_DATASAFED_BIN_PATH" + export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" + START_TIME=$1 + STOP_TIME=$2 + if [ -z $STOP_TIME ]; then + STOP_TIME=$(get_current_time) + fi + START_TIME=$(date -d "${START_TIME}" -u '+%Y-%m-%dT%H:%M:%SZ') + STOP_TIME=$(date -d "${STOP_TIME}" -u '+%Y-%m-%dT%H:%M:%SZ') + TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}') + echo "{\"totalSize\":\"$TOTAL_SIZE\",\"timeRange\":{\"start\":\"${START_TIME}\",\"end\":\"${STOP_TIME}\"}}" >"${DP_BACKUP_INFO_FILE}" +} + +# if the script exits with a non-zero exit code, touch a file to indicate that the backup failed, +# the sync progress container will check this file and exit if it exists +function handle_exit() { + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "failed with exit code $exit_code" + touch "${DP_BACKUP_INFO_FILE}.exit" + exit 1 + fi +} \ No newline at end of file diff --git a/addons/halo/dataprotection/halo-basebackup-backup.sh b/addons/halo/dataprotection/halo-basebackup-backup.sh new file mode 100644 index 000000000..e728ddcd9 --- /dev/null +++ b/addons/halo/dataprotection/halo-basebackup-backup.sh @@ -0,0 +1,13 @@ +set -e +set -o pipefail +export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" +export PGPASSWORD=${DP_DB_PASSWORD} + +trap handle_exit EXIT + +START_TIME=`get_current_time` +echo ${DP_DB_PASSWORD} | pg_basebackup -Ft -Pv -c fast -Xf -D - -h ${DP_DB_HOST} -U ${DP_DB_USER} -W | datasafed push -z zstd-fastest - "/${DP_BACKUP_NAME}.tar.zst" + +# stat and save the backup information +stat_and_save_backup_info $START_TIME \ No newline at end of file diff --git a/addons/halo/templates/actionset-halobasebackup.yaml b/addons/halo/templates/actionset-halobasebackup.yaml new file mode 100644 index 000000000..e78de7acc --- /dev/null +++ b/addons/halo/templates/actionset-halobasebackup.yaml @@ -0,0 +1,33 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: ActionSet +metadata: + name: pg-basebackup + labels: + clusterdefinition.kubeblocks.io/name: halo + {{- include "halo.labels" . | nindent 4 }} +spec: + backupType: Full + env: + - name: DATA_DIR + value: {{ .Values.dataMountPath }} + - name: DP_DB_USER + value: halo + - name: DP_BACKUP_BASE_PATH + value: /u01/app/halo/product/dbms/14/bin + - name: DP_DB_HOST + value: localhost + backup: + preBackup: [] + postBackup: [] + backupData: + image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + command: + - bash + - -c + - | + {{- .Files.Get "dataprotection/backup-info-collector.sh" | nindent 8 -}} + {{- .Files.Get "dataprotection/halo-basebackup-backup.sh" | nindent 8 -}} + syncProgress: + enabled: true + intervalSeconds: 5 + diff --git a/addons/halo/templates/backuppolicytemplate.yaml b/addons/halo/templates/backuppolicytemplate.yaml new file mode 100644 index 000000000..1c65815eb --- /dev/null +++ b/addons/halo/templates/backuppolicytemplate.yaml @@ -0,0 +1,25 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: halo-backup-policy-template + labels: + clusterdefinition.kubeblocks.io/name: halo + {{- include "halo.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: halo + backupPolicies: + - componentDefRef: halo + backupMethods: + - name: pg-basebackup + snapshotVolumes: false + actionSetName: pg-basebackup + targetVolumes: + volumeMounts: + - name: data + mountPath: {{ .Values.dataMountPath }} + schedules: + - backupMethod: pg-basebackup + enabled: false + cronExpression: "*/5 * * * *" + retentionPeriod: 7d + From b7441386d3fc00241ca0bca181edeb0ad72cda8e Mon Sep 17 00:00:00 2001 From: binbin21avery <38284249+binbin21avery@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:42:06 +0800 Subject: [PATCH 2/2] edit values.yaml with debug=true --- addons/halo/values.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/addons/halo/values.yaml b/addons/halo/values.yaml index b46c6194d..594cbf309 100644 --- a/addons/halo/values.yaml +++ b/addons/halo/values.yaml @@ -18,7 +18,10 @@ image: ## pullSecrets: ## - myRegistryKeySecretName ## - + pullSecrets: [] + ## Set to true if you would like to see extra information on logs + ## + debug: true ## Set PostgreSQL preload extension shared libraries. ## @param postgresqlSharedPreloadLibraries Shared preload libraries (comma-separated list) ##