Timeout stream keep alive for Upgrades, Restores and Migrations

bundle/manifests/pulp-operator.clusterserviceversion.yaml

@@ -857,6 +857,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:Secret
+      - displayName: Force drop database before restore
+        path: force_drop_db
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
       statusDescriptors:
       - displayName: Conditions
         path: conditions

config/crd/bases/pulpproject_v1beta1_pulp_crd.yaml

@@ -866,6 +866,10 @@ spec:
                         type: object
                     type: object
                 type: object
+              force_drop_db:
+                description: Force drop the database before restoring. USE WITH CAUTION!
+                type: boolean
+                default: false
             type: object
           status:
             properties:

config/manifests/bases/pulp-operator.clusterserviceversion.yaml

@@ -489,6 +489,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:Secret
+      - displayName: Force drop database before restore
+        path: force_drop_db
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
       statusDescriptors:
       - displayName: Conditions
         path: conditions

playbooks/pulp.yml

@@ -59,12 +59,51 @@
             operator: pulp
       when:
         - (_rh_ops_secret is not defined) or not (_rh_ops_secret['resources'] | length)
+
   roles:
     - common
     - postgres
     - { role: "pulp-web", when: [ ingress_type | lower != 'route' ] }
-    - pulp-api
-    - pulp-content
-    - pulp-worker
-    - pulp-routes
-    - pulp-status
+
+  tasks:
+
+    # If file storage is used, this role will create the PVC,
+    # which is needed by the restore management pod
+    - include_role:
+        name: pulp-content
+
+    - name: Pre-load role variables for all roles
+      include_vars: "{{ item }}"
+      loop:
+        - /opt/ansible/roles/pulp-api/defaults/main.yml
+        - /opt/ansible/roles/pulp-content/defaults/main.yml
+        - /opt/ansible/roles/pulp-worker/defaults/main.yml
+
+    - name: Wait for {{ kind }}Restore to complete check
+      kubernetes.core.k8s_info:
+        api_version: "{{ api_version }}"
+        kind: "{{ kind }}Restore"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+
+    - name: Wait for {{ kind }}Restore to complete
+      kubernetes.core.k8s_info:
+        api_version: "{{ api_version }}"
+        kind: "{{ kind }}Restore"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      register: restore_status_check
+      until:
+        # yamllint disable-line rule:line-length
+        - (restore_status_check.resources | length == 0) or (restore_status_check.resources | selectattr('spec.deployment_name', 'equalto', ansible_operator_meta.name) | map(attribute='status') | selectattr('restoreComplete', 'defined') | map(attribute='restoreComplete') | list | length > 0)
+      delay: 10
+      retries: 8640
+      ignore_errors: yes
+      changed_when: false
+
+    - name: Include roles
+      include_role:
+        name: "{{ item }}"
+      with_items:
+        - pulp-api
+        - pulp-worker
+        - pulp-routes
+        - pulp-status

roles/backup/tasks/postgres.yml

@@ -92,13 +92,35 @@
       -U {{ postgres_user }}
       -d {{ postgres_database }}
       -p {{ postgres_port }}
+      -F custom
   no_log: "{{ no_log }}"
 
 - name: Write pg_dump to backup on PVC
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-backup-manager"
-    command: >-
-      bash -c "PGPASSWORD={{ postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/pulp.db"
+    command: |
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Dumping data from database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
+      set -e -o pipefail
+      PGPASSWORD='{{ postgres_pass }}' {{ pgdump }} > {{ _backup_dir }}/pulp.db
+      set +e +o pipefail
+      echo 'Successful'
+      "
   register: data_migration
   no_log: "{{ no_log }}"
+  failed_when: "'Successful' not in data_migration.stdout"

roles/common/defaults/main.yml

@@ -4,3 +4,16 @@ image_pull_secret: ''
 image_pull_secrets: []
 operator_service_account_name: '{{ lookup("env","OPERATOR_SA_NAME") | default("pulp-operator-sa",true) }}'
 bundle_cacert_secret: ''
+
+is_k8s: false
+is_openshift: false
+
+
+# Secret to lookup that provide the admin password
+#
+admin_password_secret: ''
+
+admin_password_name: '{{ ansible_operator_meta.name }}-admin-password'
+
+# signing_service.yml
+__gpg_init_container_image: "quay.io/centos/centos:stream9"

roles/common/tasks/check_k8s_or_openshift.yml

@@ -0,0 +1,38 @@
+- name: Get information about the cluster
+  set_fact:
+    api_groups: "{{ lookup('k8s', cluster_info='api_groups') }}"
+  when:
+  - not is_openshift
+  - not is_k8s
+
+- name: Determine the cluster type
+  set_fact:
+    is_openshift: "{{ True if 'route.openshift.io' in api_groups else False }}"
+    is_k8s: "{{ False if 'route.openshift.io' in api_groups else True }}"
+  when:
+  - not is_openshift
+  - not is_k8s
+
+# Indicate what kind of cluster we are in (OpenShift or Kubernetes).
+- debug:
+    msg: "CLUSTER TYPE: is_openshift={{ is_openshift }}; is_k8s={{ is_k8s }}"
+
+- block:
+  - k8s_status:
+      api_version: "{{ api_version }}"
+      kind: "{{ kind }}"
+      name: "{{ ansible_operator_meta.name }}"
+      namespace: "{{ ansible_operator_meta.namespace }}"
+      conditions:
+        - type: "{{ deployment_type|capitalize }}-API-Ready"
+          message: Cannot determine what type of cluster we are in
+          reason: FailedToIdentifyClusterType
+          status: "False"
+          lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
+
+  - fail:
+      msg: "Cannot determine what type of cluster we are in"
+
+  when:
+  - not is_openshift
+  - not is_k8s

roles/common/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+- name: Set apiVersion and kind variables
+  set_fact:
+    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
+    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
 
 - name: Fail execution if image_pull_secret or image_pull_secrets are defined but as NoneType ('image_pull_secret[s]:')
   fail:
@@ -49,3 +53,18 @@
 - name: Set Bundle Certificate Authority
   include_tasks: set_bundle_cacert.yml
   when: bundle_cacert_secret | length
+
+- name: Check if k8s or Openshift
+  include_tasks: check_k8s_or_openshift.yml
+
+- name: Configure Admin Password from secret
+  include_tasks:
+    file: admin_password_configuration.yml
+
+- name: Configure DB fields encryption key from secret
+  include_tasks:
+    file: db_fields_encryption_configuration.yml
+
+- name: Configure Signing Service
+  include_tasks: signing_service.yml
+  when: signing_secret is defined

roles/postgres/tasks/main.yml

@@ -1,8 +1,4 @@
 ---
-- name: Set apiVersion and kind variables
-  set_fact:
-    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
-    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
 
 - k8s_status:
     api_version: "{{ api_version }}"
@@ -136,6 +132,17 @@
     postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}"
   no_log: "{{ no_log }}"
 
+- name: Getting raw pulp_settings
+  set_fact:
+    raw_pulp_settings: "{{ raw_spec['pulp_settings'] | default({}) }}"
+  no_log: "{{ no_log }}"
+  when: pulp_settings is defined
+
+- name: Combining pulp_settings
+  set_fact:
+    pulp_combined_settings: "{{ default_settings|combine(raw_pulp_settings, recursive=True) if pulp_settings is defined and pulp_settings is not none else default_settings }}"
+  no_log: "{{ no_log }}"
+
 - name: Set database as managed
   set_fact:
     managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"

roles/postgres/tasks/migrate_data.yml

@@ -88,7 +88,7 @@
 
 - name: Set pg_restore command
   set_fact:
-    psql_restore: >-
+    pg_restore: >-
       pg_restore --clean --if-exists
       -U {{ postgres_user }}
       -d {{ postgres_database }}
@@ -111,14 +111,29 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data from old database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD={{ migrant_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ postgres_pass }} {{ psql_restore }}
+      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
-  failed_when: "'Successful' not in data_migration['stdout']"
 
 - name: Set flag signifying that this instance has been migrated
   set_fact:

roles/postgres/tasks/upgrade_postgres.yml

@@ -125,7 +125,7 @@
 
 - name: Set full resolvable host name for old postgres pod
   set_fact:
-    resolvable_db_host: "{{ ansible_operator_meta.name }}-postgres-svc.{{ ansible_operator_meta.namespace }}.svc.cluster.local"  # yamllint disable-line rule:line-length
+    resolvable_db_host: "{{ ansible_operator_meta.name }}-postgres-svc.{{ ansible_operator_meta.namespace }}.svc"  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
 
 
@@ -189,11 +189,27 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data to new PostgreSQL {{ supported_postgres_version }} Database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD={{ postgres_pass }} {{ pgdump }} | PGPASSWORD={{ postgres_pass }} {{ pg_restore }}
+      PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   no_log: "{{ no_log }}"
   register: data_migration
   failed_when: "'Successful' not in data_migration.stdout"

roles/postgres/templates/postgres.yaml.j2

@@ -93,6 +93,13 @@ spec:
               value: '{{ postgres_initdb_args }}'
             - name: POSTGRES_HOST_AUTH_METHOD
               value: '{{ postgres_host_auth_method }}'
+{% if 'resources' in old_pg_config and old_pg_config['resources'] and 'password' in old_pg_config['resources'][0]['data'] %}
+            - name: PGPASSWORD_OLD
+              valueFrom:
+                secretKeyRef:
+                  name: '{{ old_pg_config['resources'][0]['metadata']['name'] }}'
+                  key: password
+{% endif %}
           ports:
             - containerPort: {{ postgres_port | default('5432')}}
               name: postgres

roles/pulp-api/defaults/main.yml

@@ -14,12 +14,6 @@ api:
 # see: https://github.com/operator-framework/operator-sdk/issues/1770
 raw_spec: "{{ vars['_pulp_pulpproject_org_pulp']['spec'] }}"
 
-# Secret to lookup that provide the admin password
-#
-admin_password_secret: ''
-
-admin_password_name: '{{ ansible_operator_meta.name }}-admin-password'
-
 # Set content host
 content_host: '{{ ansible_operator_meta.name }}-content-svc'
 content_port: '24816'
@@ -74,9 +68,6 @@ default_azure_settings:
   AZURE_OVERWRITE_FILES: "True"
   DEFAULT_FILE_STORAGE: "storages.backends.azure_storage.AzureStorage"
 
-is_k8s: false
-is_openshift: false
-
 container_auth_public_key_name: 'container_auth_public_key.pem'
 container_auth_private_key_name: 'container_auth_private_key.pem'
 
@@ -94,7 +85,5 @@ keycloak_protocol_available: false
 keycloak_port_available: false
 keycloak_realm_available: false
 
-__gpg_init_container_image: "quay.io/centos/centos:stream9"
-
 gunicorn_timeout: 90
 gunicorn_api_workers: 2