K8SPXC-1366: Improve parallel backup prevention

e2e-tests/demand-backup-parallel/conf/backup.yml

@@ -0,0 +1,7 @@
+apiVersion: pxc.percona.com/v1
+kind: PerconaXtraDBClusterBackup
+metadata:
+  name:
+spec:
+  pxcCluster: demand-backup-parallel
+  storageName: minio

e2e-tests/demand-backup-parallel/conf/cr.yml

@@ -0,0 +1,88 @@
+apiVersion: pxc.percona.com/v1
+kind: PerconaXtraDBCluster
+metadata:
+  name: demand-backup-parallel
+  finalizers:
+    - percona.com/delete-pxc-pods-in-order
+  # annotations:
+  #   percona.com/issue-vault-token: "true"
+spec:
+  tls:
+    SANs:
+      - "minio-service.#namespace"
+  secretsName: my-cluster-secrets
+  vaultSecretName: some-name-vault
+  pause: false
+  pxc:
+    size: 3
+    image: -pxc
+    configuration: |
+      [mysqld]
+      wsrep_log_conflicts
+      log_error_verbosity=3
+      wsrep_debug=1
+      [sst]
+      xbstream-opts=--decompress
+      [xtrabackup]
+      compress=lz4
+    resources:
+      requests:
+        memory: 0.1G
+        cpu: 100m
+      limits:
+        memory: "2G"
+        cpu: "1"
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 2Gi
+    affinity:
+      antiAffinityTopologyKey: "kubernetes.io/hostname"
+  haproxy:
+    enabled: true
+    size: 2
+    image: -haproxy
+    resources:
+      requests:
+        memory: 0.1G
+        cpu: 100m
+      limits:
+        memory: 1G
+        cpu: 700m
+    affinity:
+      antiAffinityTopologyKey: "kubernetes.io/hostname"
+  pmm:
+    enabled: false
+    image: perconalab/pmm-client:1.17.1
+    serverHost: monitoring-service
+    serverUser: pmm
+  backup:
+    activeDeadlineSeconds: 3600
+    allowParallel: false
+    backoffLimit: 3
+    image: -backup
+    storages:
+      pvc:
+        type: filesystem
+        volume:
+          persistentVolumeClaim:
+            accessModes: [ "ReadWriteOnce" ]
+            resources:
+              requests:
+                storage: 1Gi
+      minio:
+        type: s3
+        resources:
+          requests:
+            memory: 0.5G
+            cpu: 500m
+          limits:
+            memory: "2G"
+            cpu: "1"
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing/prefix/subfolder
+          endpointUrl: http://minio-service.#namespace:9000/
+        verifyTLS: false

e2e-tests/demand-backup-parallel/run

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# This test checks if spec.backup.allowParallel=false works as expected.
+
+set -o errexit
+
+test_dir=$(realpath $(dirname $0))
+. ${test_dir}/../functions
+
+set_debug
+
+function run_backup() {
+	local name=$1
+	yq eval ".metadata.name = \"${name}\"" ${test_dir}/conf/backup.yml \
+		| kubectl_bin apply -f -
+}
+
+function check_active_backup_count() {
+	active_backup_count=$(kubectl_bin get pxc-backup | grep -E 'Starting|Running' | wc -l)
+	if [[ ${active_backup_count} -gt 1 ]]; then
+		log "There are ${active_backup_count} active backups. 'allowParallel: false' doesn't work properly"
+		exit 1
+	fi
+}
+
+create_infra ${namespace}
+
+start_minio
+
+log "creating PXC client"
+kubectl_bin apply -f ${conf_dir}/client.yml
+
+log "creating cluster secrets"
+kubectl_bin apply -f ${conf_dir}/secrets.yml
+
+cluster="demand-backup-parallel"
+log "create PXC cluster: ${cluster}"
+apply_config ${test_dir}/conf/cr.yml
+
+desc 'creating backups'
+run_backup backup1
+run_backup backup2
+run_backup backup3
+run_backup backup4
+
+wait_cluster_consistency ${cluster} 3 2
+sleep 5
+check_active_backup_count
+
+for i in $(seq 0 3); do
+	sleep 5
+	check_active_backup_count
+	holder=$(kubectl_bin get lease pxc-${cluster}-backup-lock -o jsonpath={.spec.holderIdentity})
+	log "Backup lock holder: ${holder}"
+	wait_backup ${holder}
+done
+
+# explicitly check all backups to ensure all succeeded
+wait_backup backup1
+wait_backup backup2
+wait_backup backup3
+wait_backup backup4
+
+log "test passed"

e2e-tests/functions

@@ -60,6 +60,11 @@ set_debug() {
 	fi
 }
 
+log() {
+       echo "[$(date +%Y-%m-%dT%H:%M:%S%z)]" $*
+}
+
+
 HELM_VERSION=$(helm version -c | $sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/')
 if [ "${HELM_VERSION:0:2}" == "v2" ]; then
 	HELM_ARGS="--name"
@@ -98,17 +103,19 @@ wait_cluster_consistency() {
 	local i=0
 	local max=36
 	sleep 7 # wait for two reconcile loops ;)  3 sec x 2 times + 1 sec = 7 seconds
+	echo -n "waiting for pxc/${cluster_name} to be ready"
 	until [[ "$(kubectl_bin get pxc "${cluster_name}" -o jsonpath='{.status.state}')" == "ready" &&
 	"$(kubectl_bin get pxc "${cluster_name}" -o jsonpath='{.status.pxc.ready}')" == "${cluster_size}" &&
 	"$(kubectl_bin get pxc "${cluster_name}" -o jsonpath='{.status.'$(get_proxy_engine ${cluster_name})'.ready}')" == "${proxy_size}" ]]; do
-		echo 'waiting for cluster readyness'
+		echo -n .
 		sleep 20
 		if [[ $i -ge $max ]]; then
 			echo "Something went wrong waiting for cluster consistency!"
 			exit 1
 		fi
 		let i+=1
 	done
+	echo
 }
 
 create_namespace() {
@@ -235,7 +242,7 @@ wait_backup() {
 
 	set +o xtrace
 	retry=0
-	echo -n $backup
+	echo -n "waiting for pxc-backup/${backup} to reach ${status} state"
 	until kubectl_bin get pxc-backup/$backup -o jsonpath='{.status.state}' 2>/dev/null | grep $status; do
 		sleep 1
 		echo -n .

e2e-tests/run-pr.csv

@@ -5,6 +5,7 @@ custom-users,8.0
 demand-backup-cloud,8.0
 demand-backup-encrypted-with-tls,8.0
 demand-backup,8.0
+demand-backup-parallel,8.0
 haproxy,5.7
 haproxy,8.0
 init-deploy,5.7

e2e-tests/run-release.csv

@@ -5,6 +5,7 @@ cross-site
 custom-users
 default-cr
 demand-backup
+demand-backup-parallel
 demand-backup-cloud
 demand-backup-encrypted-with-tls
 haproxy

pkg/apis/pxc/v1/pxc_backup_types.go

@@ -162,6 +162,7 @@ type PXCBackupState string
 
 const (
 	BackupNew       PXCBackupState = ""
+	BackupWaiting   PXCBackupState = "Waiting"
 	BackupStarting  PXCBackupState = "Starting"
 	BackupRunning   PXCBackupState = "Running"
 	BackupFailed    PXCBackupState = "Failed"

pkg/controller/pxc/upgrade.go

@@ -168,12 +168,10 @@ func (r *ReconcilePerconaXtraDBCluster) smartUpdate(ctx context.Context, sfs api
 	}
 
 	if cr.HAProxyEnabled() && cr.Status.HAProxy.Status != api.AppStateReady {
-		log.V(1).Info("Waiting for HAProxy to be ready before smart update")
 		return nil
 	}
 
 	if cr.ProxySQLEnabled() && cr.Status.ProxySQL.Status != api.AppStateReady {
-		log.V(1).Info("Waiting for ProxySQL to be ready before smart update")
 		return nil
 	}