Timeout stream keep alive for Upgrades, Restores and Migrations

config/crd/bases/eda.ansible.com_edas.yaml

@@ -2473,6 +2473,10 @@ spec:
               bundle_cacert_secret:
                 description: Secret where the trusted Certificate Authority Bundle is stored
                 type: string
+              force_drop_db:
+                description: Force drop the database before restoring. USE WITH CAUTION!
+                type: boolean
+                default: false
           status:
             description: Status defines the observed state of EDA
             properties:

config/manifests/bases/eda-server-operator.clusterserviceversion.yaml

@@ -649,6 +649,11 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Force drop database before restore
+        path: force_drop_db
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
       statusDescriptors:
       - description: Admin password for the instance deployed
         displayName: Admin Password

roles/backup/tasks/postgres.yml

@@ -99,11 +99,27 @@
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Dumping data from database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
       PGPASSWORD='{{ eda_postgres_pass }}' {{ pgdump }} > {{ backup_dir }}/eda.db
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"

roles/eda/tasks/deploy_eda.yml

@@ -1,4 +1,18 @@
 ---
+- name: Wait for {{ deployment_type }}restore to complete
+  kubernetes.core.k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ deployment_type }}restore"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: restore_status_check
+  until:
+    # yamllint disable-line rule:line-length
+    - (restore_status_check.resources | length == 0) or (restore_status_check.resources | selectattr('spec.deployment_name', 'equalto', ansible_operator_meta.name) | map(attribute='status') | selectattr('restoreComplete', 'defined') | map(attribute='restoreComplete') | list | length > 0)
+  delay: 10
+  retries: 8640
+  ignore_errors: yes
+  changed_when: false
+
 - name: Apply ConfigMap resources
   k8s:
     apply: yes

roles/restore/tasks/init.yml

@@ -41,7 +41,7 @@
         backup_pvc: "{{ this_backup['resources'][0]['status']['backupClaim'] }}"
         backup_dir: "{{ this_backup['resources'][0]['status']['backupDirectory'] }}"
   when:
-    - backup_name != '' or backup_name is defined
+    - backup_name is defined and backup_name != ''
 
 # Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
 - name: Check provided PVC exists

roles/restore/tasks/postgres.yml

@@ -80,16 +80,55 @@
       -p {{ eda_postgres_port }}
   no_log: "{{ no_log }}"
 
+- name: Set drop db command
+  set_fact:
+    pg_drop_db: >-
+      echo 'DROP DATABASE {{ eda_postgres_database }} WITH (FORCE);' | PGPASSWORD='{{ eda_postgres_pass }}' psql
+      -U {{ eda_postgres_user }}
+      -h {{ resolvable_db_host }}
+      -d postgres
+      -p {{ eda_postgres_port }}
+  no_log: "{{ no_log }}"
+
+- name: Set create db command
+  set_fact:
+    pg_create_db: >-
+      echo 'CREATE DATABASE {{ eda_postgres_database }} WITH OWNER = {{ eda_postgres_user }};' | PGPASSWORD='{{ eda_postgres_pass }}' psql
+      -U {{ eda_postgres_user }}
+      -h {{ resolvable_db_host }}
+      -d postgres
+      -p {{ eda_postgres_port }}
+  no_log: "{{ no_log }}"
+
 - name: Restore database dump to the new postgresql container
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data from old database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
+      if {{ force_drop_db }}; then
+        {{ pg_drop_db }}
+        {{ pg_create_db }}
+      fi
       cat {{ backup_dir }}/eda.db | PGPASSWORD='{{ eda_postgres_pass }}' {{ pg_restore }}
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"