Refactor to use include_roles and wait for content PVC to be created …

…when restoring

playbooks/pulp.yml

@@ -59,12 +59,49 @@
             operator: pulp
       when:
         - (_rh_ops_secret is not defined) or not (_rh_ops_secret['resources'] | length)
+
   roles:
     - common
     - postgres
     - { role: "pulp-web", when: [ ingress_type | lower != 'route' ] }
-    - pulp-api
-    - pulp-content
-    - pulp-worker
-    - pulp-routes
-    - pulp-status
+    - pulp-content  # If file storage is used, this role will create the PVC which is needed by the restore management pod
+
+  tasks:
+
+    # Hack to import variables ahead of time since we not longer statically include all roles
+    # Eventually, these roles should be re-written, but this is safer for now
+    - name: Pre-load role variables for all roles
+      include_vars: "{{ item }}"
+      loop:
+        - /opt/ansible/roles/pulp-api/defaults/main.yml
+        - /opt/ansible/roles/pulp-content/defaults/main.yml
+        - /opt/ansible/roles/pulp-worker/defaults/main.yml
+
+    - name: Wait for {{ kind }}Restore to complete check
+      kubernetes.core.k8s_info:
+        api_version: "{{ api_version }}"
+        kind: "{{ kind }}Restore"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+
+    - name: Wait for {{ kind }}Restore to complete
+      kubernetes.core.k8s_info:
+        api_version: "{{ api_version }}"
+        kind: "{{ kind }}Restore"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      register: restore_status_check
+      until:
+        # yamllint disable-line rule:line-length
+        - (restore_status_check.resources | length == 0) or (restore_status_check.resources | selectattr('spec.deployment_name', 'equalto', ansible_operator_meta.name) | map(attribute='status') | selectattr('restoreComplete', 'defined') | map(attribute='restoreComplete') | list | length > 0)
+      delay: 10
+      retries: 8640
+      ignore_errors: yes
+      changed_when: false
+
+    - name: Include roles
+      include_role:
+        name: "{{ item }}"
+      with_items:
+        - pulp-api
+        - pulp-worker
+        - pulp-routes
+        - pulp-status

roles/backup/tasks/postgres.yml

@@ -92,6 +92,7 @@
       -U {{ postgres_user }}
       -d {{ postgres_database }}
       -p {{ postgres_port }}
+      -F custom
   no_log: "{{ no_log }}"
 
 - name: Write pg_dump to backup on PVC
@@ -116,7 +117,7 @@
       trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
       echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD='{{ postgres_pass }}' {{ pgdump }} > {{ _backup_dir }}/tower.db
+      PGPASSWORD='{{ postgres_pass }}' {{ pgdump }} > {{ _backup_dir }}/pulp.db
       set +e +o pipefail
       echo 'Successful'
       "

roles/common/defaults/main.yml

@@ -4,3 +4,6 @@ image_pull_secret: ''
 image_pull_secrets: []
 operator_service_account_name: '{{ lookup("env","OPERATOR_SA_NAME") | default("pulp-operator-sa",true) }}'
 bundle_cacert_secret: ''
+
+is_k8s: false
+is_openshift: false

roles/common/tasks/check_k8s_or_openshift.yml

@@ -0,0 +1,38 @@
+- name: Get information about the cluster
+  set_fact:
+    api_groups: "{{ lookup('k8s', cluster_info='api_groups') }}"
+  when:
+  - not is_openshift
+  - not is_k8s
+
+- name: Determine the cluster type
+  set_fact:
+    is_openshift: "{{ True if 'route.openshift.io' in api_groups else False }}"
+    is_k8s: "{{ False if 'route.openshift.io' in api_groups else True }}"
+  when:
+  - not is_openshift
+  - not is_k8s
+
+# Indicate what kind of cluster we are in (OpenShift or Kubernetes).
+- debug:
+    msg: "CLUSTER TYPE: is_openshift={{ is_openshift }}; is_k8s={{ is_k8s }}"
+
+- block:
+  - k8s_status:
+      api_version: "{{ api_version }}"
+      kind: "{{ kind }}"
+      name: "{{ ansible_operator_meta.name }}"
+      namespace: "{{ ansible_operator_meta.namespace }}"
+      conditions:
+        - type: "{{ deployment_type|capitalize }}-API-Ready"
+          message: Cannot determine what type of cluster we are in
+          reason: FailedToIdentifyClusterType
+          status: "False"
+          lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
+
+  - fail:
+      msg: "Cannot determine what type of cluster we are in"
+
+  when:
+  - not is_openshift
+  - not is_k8s

roles/common/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+- name: Set apiVersion and kind variables
+  set_fact:
+    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
+    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
 
 - name: Fail execution if image_pull_secret or image_pull_secrets are defined but as NoneType ('image_pull_secret[s]:')
   fail:
@@ -49,3 +53,6 @@
 - name: Set Bundle Certificate Authority
   include_tasks: set_bundle_cacert.yml
   when: bundle_cacert_secret | length
+
+- name: Check if k8s or Openshift
+  include_tasks: check_k8s_or_openshift.yml

roles/postgres/tasks/main.yml

@@ -1,8 +1,4 @@
 ---
-- name: Set apiVersion and kind variables
-  set_fact:
-    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
-    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
 
 - k8s_status:
     api_version: "{{ api_version }}"
@@ -136,6 +132,17 @@
     postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}"
   no_log: "{{ no_log }}"
 
+- name: Getting raw pulp_settings
+  set_fact:
+    raw_pulp_settings: "{{ raw_spec['pulp_settings'] | default({}) }}"
+  no_log: "{{ no_log }}"
+  when: pulp_settings is defined
+
+- name: Combining pulp_settings
+  set_fact:
+    pulp_combined_settings: "{{ default_settings|combine(raw_pulp_settings, recursive=True) if pulp_settings is defined and pulp_settings is not none else default_settings }}"
+  no_log: "{{ no_log }}"
+
 - name: Set database as managed
   set_fact:
     managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"

roles/pulp-api/defaults/main.yml

@@ -74,9 +74,6 @@ default_azure_settings:
   AZURE_OVERWRITE_FILES: "True"
   DEFAULT_FILE_STORAGE: "storages.backends.azure_storage.AzureStorage"
 
-is_k8s: false
-is_openshift: false
-
 container_auth_public_key_name: 'container_auth_public_key.pem'
 container_auth_private_key_name: 'container_auth_private_key.pem'
 

roles/pulp-api/tasks/main.yml

@@ -1,19 +1,5 @@
 ---
 
-- name: Wait for {{ deployment_type }}restore to complete
-  kubernetes.core.k8s_info:
-    api_version: "{{ api_version }}"
-    kind: "{{ deployment_type }}restore"
-    namespace: "{{ ansible_operator_meta.namespace }}"
-  register: restore_status_check
-  until:
-    # yamllint disable-line rule:line-length
-    - (restore_status_check.resources | length == 0) or (restore_status_check.resources | selectattr('spec.deployment_name', 'equalto', ansible_operator_meta.name) | map(attribute='status') | selectattr('restoreComplete', 'defined') | map(attribute='restoreComplete') | list | length > 0)
-  delay: 10
-  retries: 8640
-  ignore_errors: yes
-  changed_when: false
-
 - set_fact:
     object_storage_secret: "{{ object_storage_s3_secret }}"
   when:
@@ -24,58 +10,6 @@
   when:
     - object_storage_azure_secret is defined
 
-- set_fact:
-    is_file_storage: false
-  when:
-    - object_storage_secret is defined
-
-- name: pulp-file-storage
-  block:
-    - name: "Creating {{ deployment_type|capitalize }}-api PVC resource"
-      k8s_status:
-        api_version: "{{ api_version }}"
-        kind: "{{ kind }}"
-        name: "{{ ansible_operator_meta.name }}"
-        namespace: "{{ ansible_operator_meta.namespace }}"
-        conditions:
-          - type: "{{ deployment_type|capitalize }}-API-Ready"
-            message: "Creating {{ deployment_type|capitalize }}-api PVC resource"
-            reason: CreatingPVC
-            status: "False"
-            lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
-
-    - name: pulp-file-storage persistent volume claim
-      k8s:
-        state: "{{ deployment_state }}"
-        definition: "{{ lookup('template', 'templates/' + item + '.pvc.yaml.j2') | from_yaml }}"
-      with_items:
-        - pulp-file-storage
-
-    - name: "Removing ownerReferences from {{ ansible_operator_meta.name}}-file-storage PVC"
-      k8s_status:
-        api_version: "{{ api_version }}"
-        kind: "{{ kind }}"
-        name: "{{ ansible_operator_meta.name }}"
-        namespace: "{{ ansible_operator_meta.namespace }}"
-        conditions:
-          - type: "{{ deployment_type|capitalize }}-API-Ready"
-            message: "Removing ownerReferences from {{ ansible_operator_meta.name}}-file-storage PVC"
-            reason: RemovingPVCOwnerReferences
-            status: "False"
-            lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
-
-    - name: Remove ownerReferences from pulp-file-storage pvc to avoid garbage collection
-      k8s:
-        definition:
-          apiVersion: v1
-          kind: PersistentVolumeClaim
-          metadata:
-            name: '{{ ansible_operator_meta.name }}-file-storage'
-            namespace: '{{ ansible_operator_meta.namespace }}'
-            ownerReferences: null
-
-  when: is_file_storage
-
 - include_tasks:
     file: s3-storage-configuration.yml
   when:
@@ -183,45 +117,6 @@
   with_items:
     - pulp-api
 
-- name: Get information about the cluster
-  set_fact:
-    api_groups: "{{ lookup('k8s', cluster_info='api_groups') }}"
-  when:
-  - not is_openshift
-  - not is_k8s
-
-- name: Determine the cluster type
-  set_fact:
-    is_openshift: "{{ True if 'route.openshift.io' in api_groups else False }}"
-    is_k8s: "{{ False if 'route.openshift.io' in api_groups else True }}"
-  when:
-  - not is_openshift
-  - not is_k8s
-
-# Indicate what kind of cluster we are in (OpenShift or Kubernetes).
-- debug:
-    msg: "CLUSTER TYPE: is_openshift={{ is_openshift }}; is_k8s={{ is_k8s }}"
-
-- block:
-  - k8s_status:
-      api_version: "{{ api_version }}"
-      kind: "{{ kind }}"
-      name: "{{ ansible_operator_meta.name }}"
-      namespace: "{{ ansible_operator_meta.namespace }}"
-      conditions:
-        - type: "{{ deployment_type|capitalize }}-API-Ready"
-          message: Cannot determine what type of cluster we are in
-          reason: FailedToIdentifyClusterType
-          status: "False"
-          lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
-
-  - fail:
-      msg: "Cannot determine what type of cluster we are in"
-
-  when:
-  - not is_openshift
-  - not is_k8s
-
 - name: Set default pulp-api image
   set_fact:
     _default_image: "{{ _image }}:{{ _image_version }}"

roles/pulp-content/tasks/create-content-pvc.yml

@@ -0,0 +1,52 @@
+---
+- set_fact:
+    is_file_storage: false
+  when:
+    - object_storage_secret is defined
+
+- name: pulp-file-storage
+  block:
+    - name: "Creating {{ deployment_type|capitalize }}-api PVC resource"
+      k8s_status:
+        api_version: "{{ api_version }}"
+        kind: "{{ kind }}"
+        name: "{{ ansible_operator_meta.name }}"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        conditions:
+          - type: "{{ deployment_type|capitalize }}-API-Ready"
+            message: "Creating {{ deployment_type|capitalize }}-api PVC resource"
+            reason: CreatingPVC
+            status: "False"
+            lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
+
+    - name: pulp-file-storage persistent volume claim
+      k8s:
+        state: "{{ deployment_state }}"
+        definition: "{{ lookup('template', 'templates/' + item + '.pvc.yaml.j2') | from_yaml }}"
+      with_items:
+        - pulp-file-storage
+
+    - name: "Removing ownerReferences from {{ ansible_operator_meta.name}}-file-storage PVC"
+      k8s_status:
+        api_version: "{{ api_version }}"
+        kind: "{{ kind }}"
+        name: "{{ ansible_operator_meta.name }}"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        conditions:
+          - type: "{{ deployment_type|capitalize }}-API-Ready"
+            message: "Removing ownerReferences from {{ ansible_operator_meta.name}}-file-storage PVC"
+            reason: RemovingPVCOwnerReferences
+            status: "False"
+            lastTransitionTime: "{{ lookup('pipe', 'date --iso-8601=seconds') }}"
+
+    - name: Remove ownerReferences from pulp-file-storage pvc to avoid garbage collection
+      k8s:
+        definition:
+          apiVersion: v1
+          kind: PersistentVolumeClaim
+          metadata:
+            name: '{{ ansible_operator_meta.name }}-file-storage'
+            namespace: '{{ ansible_operator_meta.namespace }}'
+            ownerReferences: null
+
+  when: is_file_storage

roles/pulp-content/tasks/main.yml

@@ -38,6 +38,9 @@
     _node_affinity: "{{ raw_spec['affinity']['node_affinity'] | default({}) }}"
   when: affinity is defined and affinity.node_affinity is defined
 
+- name: Create Content PVC if file-storage is enabled
+  include_tasks: create-content-pvc.yml
+
 - k8s_status:
     api_version: "{{ api_version }}"
     kind: "{{ kind }}"

roles/pulp-web/tasks/main.yml

@@ -1,8 +1,4 @@
 ---
-- name: Set apiVersion and kind variables
-  set_fact:
-    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
-    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
 
 - name: Getting raw pulp_settings
   set_fact: