From 504016d12d5cca3244f3c6d1918bd32a2340a7b7 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 18 Jan 2024 19:54:06 +0000
Subject: [PATCH 01/24] #310 Allow STS deployment of ODF

---
 automation-generators/aws/openshift/preprocessor.py             | 2 --
 .../ocs-storage-class/tasks/aws-ipi-configure-ocs.yml           | 2 +-
 .../40-configure-infra/openshift-mcg/tasks/main.yml             | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/automation-generators/aws/openshift/preprocessor.py b/automation-generators/aws/openshift/preprocessor.py
index 119b8fd44..8a954ffca 100644
--- a/automation-generators/aws/openshift/preprocessor.py
+++ b/automation-generators/aws/openshift/preprocessor.py
@@ -119,8 +119,6 @@ def preprocessor(attributes=None, fullConfig=None, moduleVariables=None):
                 elif os['storage_name'] not in nfs_server_names:
                     g.appendError(msg="'"+ os['storage_name'] + "' is not an existing nfs_server name (Found nfs_server: ["+ ','.join(nfs_server_names) +"] )")
             if "storage_type" in os and os['storage_type']=='ocs':
-                if "credentials_mode" in ge['infrastructure']:
-                    g.appendError(msg='Installation of ODF using temporary cloud credentials (credentials_mode property) is not supported. Please choose elastic storage or install using permanent credentials.')
                 if "ocs_storage_label" not in os:
                     g.appendError(msg='ocs_storage_label must be specified when storage_type is ocs')
                 if "ocs_storage_size_gb" not in os:
diff --git a/automation-roles/40-configure-infra/ocs-storage-class/tasks/aws-ipi-configure-ocs.yml b/automation-roles/40-configure-infra/ocs-storage-class/tasks/aws-ipi-configure-ocs.yml
index 19d925805..a678a60ad 100644
--- a/automation-roles/40-configure-infra/ocs-storage-class/tasks/aws-ipi-configure-ocs.yml
+++ b/automation-roles/40-configure-infra/ocs-storage-class/tasks/aws-ipi-configure-ocs.yml
@@ -67,7 +67,7 @@
      -o custom-columns='name:metadata.name,phase:status.phase' | \
      grep -i ready | wc -l
   register: _ocs_csv_status
-  retries: 30
+  retries: 60
   delay: 30
   until: _ocs_csv_status.stdout == "1"
   vars:
diff --git a/automation-roles/40-configure-infra/openshift-mcg/tasks/main.yml b/automation-roles/40-configure-infra/openshift-mcg/tasks/main.yml
index 2b3f6775a..e6fef0332 100644
--- a/automation-roles/40-configure-infra/openshift-mcg/tasks/main.yml
+++ b/automation-roles/40-configure-infra/openshift-mcg/tasks/main.yml
@@ -36,7 +36,7 @@
         --no-headers \
         -o custom-columns='phase:status.phase'
       register: _storage_cluster_status
-      retries: 30
+      retries: 60
       delay: 30
       until: _storage_cluster_status.stdout == "Ready"
       vars:

From 72589de2a26dbb880afab33a11ee8a6e5688cf35 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 18 Jan 2024 21:11:41 +0100
Subject: [PATCH 02/24] #572 Allow custom non-mco script

---
 .../tasks/non-mco-apply-node-settings.yml     | 11 ++++++++
 .../templates/cloud-pak-node-fix.j2           |  6 ++++
 .../30-reference/configuration/cp4d-assets.md |  3 ++
 .../30-reference/process/install-cloud-pak.md | 28 +++++++++++++++++--
 4 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/tasks/non-mco-apply-node-settings.yml b/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/tasks/non-mco-apply-node-settings.yml
index d8ae61c4d..dbae82a78 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/tasks/non-mco-apply-node-settings.yml
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/tasks/non-mco-apply-node-settings.yml
@@ -24,6 +24,17 @@
     oc set data -n kube-system cm/cloud-pak-node-fix-scripts \
         --from-file={{ status_dir }}/openshift/cloud-pak-node-fix-timer.sh 
 
+- name: Check if custom script {{ config_dir }}/assets/apply-custom-node-settings.sh exists
+  stat:
+    path: "{{ config_dir }}/assets/apply-custom-node-settings.sh"
+  register: _custom_node_settings_script
+
+- name: Put apply-custom-node-settings.sh script into config map
+  shell:
+    oc set data -n kube-system cm/cloud-pak-node-fix-scripts \
+        --from-file={{ config_dir }}/assets/apply-custom-node-settings.sh
+  when: _custom_node_settings_script.stat.exists
+
 - name: Generate ServiceAccount for DaemonSet
   template:
     src: cloud-pak-node-fix-sa.j2
diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2 b/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2
index 8c0fb8937..c30a43aab 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2
@@ -81,6 +81,12 @@ if [ -e ${NODE_FIX_DIR}/cp4d-apply-crio-config.sh ];then
   fi
 fi
 
+if [ -e ${NODE_FIX_DIR}/apply-custom-node-settings.sh ];then
+  echo "Calling custom node update script ${NODE_FIX_DIR}/apply-custom-node-settings.sh" >> /tmp/cloud-pak-node-fix.log
+  source ${NODE_FIX_DIR}/apply-custom-node-settings.sh >> /tmp/cloud-pak-node-fix.log
+  echo "Custom script updated node settings (0=no, 1=yes): $NODE_UDPATED" >> /tmp/cloud-pak-node-fix.log
+fk
+
 if [ $NODE_UPDATED -eq 1 ];then
   echo "Restarting kubelet and crio daemons" >> /tmp/cloud-pak-node-fix.log
   systemctl restart kubelet
diff --git a/docs/src/30-reference/configuration/cp4d-assets.md b/docs/src/30-reference/configuration/cp4d-assets.md
index ef304bec8..38e9fdf2e 100644
--- a/docs/src/30-reference/configuration/cp4d-assets.md
+++ b/docs/src/30-reference/configuration/cp4d-assets.md
@@ -2,6 +2,9 @@
 
 The Cloud Pak Deployer can implement demo assets and accelerators as part of the deployment process to standardize standing up fully-featured demo environments, or to test patches or new versions of the Cloud Pak using pre-defined assets.
 
+## Node changes for ROKS and Satellite clusters
+If you put a script named `apply-custom-node-settings.sh` in the `CONFIG_DIR/assets` directory, it will be run as part of applying the node settings. This way you can override the existing node settings applied by the deployer or update the compute nodes with new settings. For more information regarding the `apply-custom-node-settings.sh` script, go to [Prepare OpenShift cluster on IBM Cloud and IBM Cloud Satellite](../process/install-cloud-pak.md#prepare-openshift-cluster-on-ibm-cloud-and-ibm-cloud-satellite).
+
 ## `cp4d_asset`
 A `cp4d_asset` entry defines one or more assets to be deployed for a specific Cloud Pak for Data instance (OpenShift project). In the configuration, a directory relative to the configuration directory (`CONFIG_DIR`) is specified. For example, if the directory where the configuration is stored is `$HOME/cpd-config/sample` and you specify `assets` as the asset directory, all assets under `/cpd-config/sample/assets` are processed.
 
diff --git a/docs/src/30-reference/process/install-cloud-pak.md b/docs/src/30-reference/process/install-cloud-pak.md
index 88f22d31c..e47dbc095 100644
--- a/docs/src/30-reference/process/install-cloud-pak.md
+++ b/docs/src/30-reference/process/install-cloud-pak.md
@@ -33,7 +33,7 @@ If an image registry has been specified for the Cloud Pak using the `image_regis
 
 ## Install Cloud Pak for Data and cartridges
 
-### Prepare OpenShift cluster for Cloud Pak for Data installation
+### Prepare OpenShift cluster for Cloud Pak installation
 Cloud Pak for Data requires a number of cluster-wide settings:
 
 * Create an `ImageContentSourcePolicy` if images must be pulled from a private registry
@@ -46,7 +46,7 @@ For all OpenShift clusters, except ROKS on IBM Cloud, these settings are applied
 
 To avoid having to reload the nodes more than once, the Machine Config Operator is paused before the settings are applied. After all setup, the Machine Config Operator is released and the deployment process will then wait until all nodes are ready with the configuration applied.
 
-#### Prepare OpenShift cluster on IBM Cloud
+#### Prepare OpenShift cluster on IBM Cloud and IBM Cloud Satellite
 As mentioned before, ROKS on IBM Cloud does not include the Machine Config Operator and would normally require the compute nodes to be reloaded (classic ROKS) or replaced (ROKS on VPC) to make the changes effective. While implementing this process, we have experienced intermittent reliability issues where replacement of nodes never finished or the cluster ended up in a unusable state. To avoid this, the process is applying the settings in a different manner.
 
 On every node, a cron job is created which starts every 5 minutes. It runs a script that checks if any of the cluster-wide settings must be (re-)applied, then updates the local system and restarts the `crio` and `kubelet` daemons. If no settings are to be adjusted, the daemons will not be restarted and therefore the cron job has minimal or no effect on the running applications.
@@ -57,6 +57,30 @@ Compute node changes that are made by the cron job:
 **CRI-O**: `pids_limit` and `default_ulimit` changes are made to the `/etc/crio/crio.conf` file.
 **Pull secret**: The registry and credentials are appended to the `/.docker/config.json` configuration.
 
+There are scenarios, especially on IBM Cloud Satellite, where custom changes must be applied to the compute nodes. This is possible by adding the `apply-custom-node-settings.sh` to the `assets` directory within the `CONFIG_DIR` directory. Once Kubelet, CRI-O and other changes have been applied, this script (if existing) is run to apply any additional configuration changes to the compute node.
+
+By setting the `NODE_UPDATED` script variable to `1` you can tell the deployer to restart the `crio` and `kubelet` daemons.
+
+**WARNING:** You should never set the `NODE_UPDATED` script variable to `0` as this will cause previous changes to the pull secret, ImageContentSourcePolicy and others not to become effective.
+
+Sample script:
+```bash
+#!/bin/bash
+
+#
+# This is a sample script that will cause the crio and kubelet daemons to be restarted once by checking
+# file /tmp/apply-custom-node-settings-run. If the file doesn't exist, it creates it and sets NODE_UPDATED to 1.
+# The deployer will observe that the node has been updated and restart the daemons.
+#
+
+if [ ! -e /tmp/apply-custom-node-settings-run ];then
+    touch /tmp/apply-custom-node-settings-run
+    NODE_UPDATED=1
+fi
+
+exit 0
+```
+
 ### Mirror images to the private registry
 If a private image registry is specified, and if the IBM Cloud Pak entitlement key is available in the vault (`cp_entitlement_key` secret), the Cloud Pak case files for the Foundational Services, the Cloud Pak control plane and cartridges are downloaded to a subdirectory of the status directory that was specified. Then all images defined for the cartridges are mirrored from the entitled registry to the private image registry. Dependent on network speed and how many cartridges have been configured, the mirroring can take a very long time (12+ hours). All images which have already been mirrored to the private registry are skipped by the mirroring process.
 

From 5b17df3df7f80005b48df7a9ca71d399e7e03c41 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 19 Jan 2024 09:47:47 +0000
Subject: [PATCH 03/24] #572 Add warning not to use exit + fix typo

---
 .../cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2    | 2 +-
 docs/src/30-reference/process/install-cloud-pak.md            | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2 b/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2
index c30a43aab..d122e9c95 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-ocp-mco-resume/templates/cloud-pak-node-fix.j2
@@ -85,7 +85,7 @@ if [ -e ${NODE_FIX_DIR}/apply-custom-node-settings.sh ];then
   echo "Calling custom node update script ${NODE_FIX_DIR}/apply-custom-node-settings.sh" >> /tmp/cloud-pak-node-fix.log
   source ${NODE_FIX_DIR}/apply-custom-node-settings.sh >> /tmp/cloud-pak-node-fix.log
   echo "Custom script updated node settings (0=no, 1=yes): $NODE_UDPATED" >> /tmp/cloud-pak-node-fix.log
-fk
+fi
 
 if [ $NODE_UPDATED -eq 1 ];then
   echo "Restarting kubelet and crio daemons" >> /tmp/cloud-pak-node-fix.log
diff --git a/docs/src/30-reference/process/install-cloud-pak.md b/docs/src/30-reference/process/install-cloud-pak.md
index e47dbc095..703d69da0 100644
--- a/docs/src/30-reference/process/install-cloud-pak.md
+++ b/docs/src/30-reference/process/install-cloud-pak.md
@@ -63,6 +63,8 @@ By setting the `NODE_UPDATED` script variable to `1` you can tell the deployer t
 
 **WARNING:** You should never set the `NODE_UPDATED` script variable to `0` as this will cause previous changes to the pull secret, ImageContentSourcePolicy and others not to become effective.
 
+**WARNING:** Do not end the script with the `exit` command; this will stop the calling script from running and therefore not restart the daemons.
+
 Sample script:
 ```bash
 #!/bin/bash
@@ -77,8 +79,6 @@ if [ ! -e /tmp/apply-custom-node-settings-run ];then
     touch /tmp/apply-custom-node-settings-run
     NODE_UPDATED=1
 fi
-
-exit 0
 ```
 
 ### Mirror images to the private registry

From e18c1f0d2e237267ec71ac191ff89a96813c538b Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 19 Jan 2024 09:48:42 +0000
Subject: [PATCH 04/24] #618 Fix preprocessor for cp4d

---
 automation-generators/generic/cp4d/preprocessor.py         | 7 +++++--
 .../lint-config/scripts/generatorPreProcessor.py           | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/automation-generators/generic/cp4d/preprocessor.py b/automation-generators/generic/cp4d/preprocessor.py
index cb6265531..7c232629f 100644
--- a/automation-generators/generic/cp4d/preprocessor.py
+++ b/automation-generators/generic/cp4d/preprocessor.py
@@ -226,9 +226,7 @@ def preprocessor(attributes=None, fullConfig=None, moduleVariables=None):
 
     g('project').isRequired()
     g('openshift_cluster_name').expandWith('openshift[*]',remoteIdentifier='name')
-    openshift_cluster_name=g('openshift_cluster_name').getExpandedAttributes()['openshift_cluster_name']
     g('cp4d_version').isRequired()
-    g('openshift_storage_name').expandWithSub('openshift', remoteIdentifier='name', remoteValue=openshift_cluster_name, listName='openshift_storage',listIdentifier='storage_name')
     g('cartridges').isRequired()
     g('use_case_files').isOptional().mustBeOneOf([True, False])
     g('sequential_install').isOptional().mustBeOneOf([True, False])
@@ -237,6 +235,11 @@ def preprocessor(attributes=None, fullConfig=None, moduleVariables=None):
     g('cp4d_entitlement').isOptional().mustBeOneOf(['cpd-enterprise', 'cpd-standard'])
     g('cp4d_production_license').isOptional().mustBeOneOf([True, False])
 
+    # Expand storage if no errors yet
+    if len(g.getErrors()) == 0:
+        openshift_cluster_name=g('openshift_cluster_name').getExpandedAttributes()['openshift_cluster_name']
+        g('openshift_storage_name').expandWithSub('openshift', remoteIdentifier='name', remoteValue=openshift_cluster_name, listName='openshift_storage',listIdentifier='storage_name')
+
     # Now that we have reached this point, we can check the attribute details if the previous checks passed
     if len(g.getErrors()) == 0:
         fc = g.getFullConfig()
diff --git a/automation-roles/10-validation/lint-config/scripts/generatorPreProcessor.py b/automation-roles/10-validation/lint-config/scripts/generatorPreProcessor.py
index c974de15a..5b6790b6c 100644
--- a/automation-roles/10-validation/lint-config/scripts/generatorPreProcessor.py
+++ b/automation-roles/10-validation/lint-config/scripts/generatorPreProcessor.py
@@ -151,7 +151,7 @@ def expandWith(self, matchPattern, remoteIdentifier='name'):
                 self.attributesDict[ self.recentCheck.get('pathToCheck') ]=listOfMatches[0]
             else:
                 #print(listOfMatches)
-                self.appendError(msg="Can't expand, result of given path ("+ matchPatternCombined +") not unique, found:" + ','.join(listOfMatches))
+                self.appendError(msg="Can't expand attribute "+self.recentCheck.get('pathToCheck')+", resource to infer from: " + matchPatternCombined + ") not unique, found: " + ','.join(listOfMatches))
         return self
 
     # matchPattern (string): should look like 'vpc[*].name'

From 829458ac95a50fe61b267222c1f3b0db6b8786e7 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 19 Jan 2024 09:49:15 +0000
Subject: [PATCH 05/24] #618 Cluster components not applied for 2nd cluster

---
 .../cpfs/cp-fs-cluster-components/tasks/main.yml | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
index c5b352ecd..0ecdc2132 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
@@ -10,16 +10,8 @@
     dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-apply-cluster-components.sh"
     mode: u+rwx
 
-- name: Apply cluster components if not already done in an earlier step
-  block:
-
-  - name: Run shell script to apply cluster components, logs are in {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-apply-cluster-components.log
-    shell: |
-      {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
-
-  - set_fact:
-      _cp_fs_cluster_compontents_run: True
-
+- name: Run shell script to apply cluster components, logs are in {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-apply-cluster-components.log
+  shell: |
+    {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
   when: 
-  - not _p_preview
-  - not (_cp_fs_cluster_compontents_run | default(False))
\ No newline at end of file
+  - not _p_preview
\ No newline at end of file

From 026a69b6f73e8f515c1daba9b8a6f2783add773f Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 19 Jan 2024 10:35:14 +0000
Subject: [PATCH 06/24] #618 Show cluster name that changes are made to

---
 .../cp4d/cp4d-cluster/tasks/install-cp4d.yml              | 4 ++--
 .../cp4d/cp4d-prepare-openshift/tasks/main.yml            | 4 ++--
 .../cpfs/cp-fs-cluster-components/tasks/main.yml          | 8 ++++----
 .../templates/apply-cluster-components.j2                 | 2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml
index c866e5ae5..27695d44b 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml
@@ -20,7 +20,7 @@
   - (current_cp4d_cluster.image_registry_name | default("")) != ""
   - not (cpd_skip_mirror | bool)
 
-- name: Migrate to private topology if upgrading to CP4D 4.7.0 or higher
+- name: Migrate to private topology on OpenShift cluster {{ current_cp4d_cluster.openshift_cluster_name }} if upgrading to CP4D 4.7.0 or higher
   include_role: 
     name: cp4d-migrate-private-topology
   vars:
@@ -30,7 +30,7 @@
   - _installed_ibmcpd_version < "4.7.0"
   - current_cp4d_cluster.cp4d_version >= "4.7.0"
 
-- name: Activate license service and certificate manager for CP4D 4.7.0 and higher
+- name: Activate license service and certificate manager on OpenShift cluster {{ current_cp4d_cluster.openshift_cluster_name }}
   include_role:
     name: cp-fs-cluster-components
   vars:
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-prepare-openshift/tasks/main.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-prepare-openshift/tasks/main.yml
index 4a651d086..f89417075 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-prepare-openshift/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-prepare-openshift/tasks/main.yml
@@ -8,7 +8,7 @@
   fail: msg="cloud_platform {{ cloud_platform }} is not implemented, current implemented cloud platforms are {{ implemented_cloud_platform_types }} "
   when: "cloud_platform not in implemented_cloud_platform_types"
 
-- name: Retrieve or detect cloud infra
+- name: Retrieve or detect cloud infrastructure type for OpenShift cluster {{ current_cp4d_cluster.openshift_cluster_name }}
   include_role:
     name: retrieve-cloud-infra-type
   vars:
@@ -19,5 +19,5 @@
   - cloud_platform == 'existing-ocp'
   - _storage_type == 'pwx'
 
-- name: Prepare cluster-wide configuration for Cloud Pak for Data
+- name: Prepare Cloud Pak for Data cluster-wide configuration on cluster {{ current_cp4d_cluster.openshift_cluster_name }}
   include_tasks: cp4d-prepare-openshift.yml
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
index 0ecdc2132..beca10d6c 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
@@ -4,14 +4,14 @@
     that:
       - _p_cp4d_version is defined
 
-- name: Generate apply-cluster-components script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
+- name: Generate apply-cluster-components script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
   template:
     src: apply-cluster-components.j2
-    dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-apply-cluster-components.sh"
+    dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.sh"
     mode: u+rwx
 
-- name: Run shell script to apply cluster components, logs are in {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-apply-cluster-components.log
+- name: Run shell script to apply cluster components on OpenShift cluster {{ current_cp4d_cluster.openshift_cluster_name }}, logs are in {{ status_dir }}/log/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.log
   shell: |
-    {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
+    {{ status_dir }}/cp4d/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
   when: 
   - not _p_preview
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2 b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2
index 325870b9b..ab1221c4e 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2
@@ -9,4 +9,4 @@ apply-cluster-components \
     --migrate_from_cs_ns={{ foundational_services_project }} \
 {% endif -%}   
     --cert_manager_ns={{ cert_manager_project }} \
-    --licensing_ns={{ cs_control_project }} 2>&1 | tee {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-apply-cluster-components.log 2>&1
+    --licensing_ns={{ cs_control_project }} 2>&1 | tee {{ status_dir }}/log/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.log 2>&1

From 1b1f47b679cfb80eaee57ce73f134f66870442f1 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 19 Jan 2024 15:53:49 +0000
Subject: [PATCH 07/24] #622 Also move case files when applying cluster
 components

---
 .../cp-fs-cluster-components/tasks/main.yml     | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
index beca10d6c..64905921b 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
@@ -4,6 +4,23 @@
     that:
       - _p_cp4d_version is defined
 
+- name: Delete offline directory
+  file:
+    path: /tmp/work/offline
+    state: absent
+
+- name: Create offline directory
+  file:
+    path: /tmp/work/offline
+    state: absent
+
+- name: If air-gapped, copy case files from {{ status_dir }}/cp4d/offline to /tmp/work/offline
+  copy:
+    src: "{{ status_dir }}/cp4d/offline"
+    dest: /tmp/work/
+    remote_src: True
+  when: (cpd_airgap | default(False) | bool)
+
 - name: Generate apply-cluster-components script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.sh
   template:
     src: apply-cluster-components.j2

From 9dcf9b2f4f933c67e25fdc37ce93333896a09283 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Sat, 20 Jan 2024 16:32:14 +0000
Subject: [PATCH 08/24] #624 Default to ibm-licensing project

---
 .../cp4d/cp4d-variables/tasks/main.yml               |  3 +++
 .../tasks/set-license-service-project.yml            | 12 ++++++++++++
 .../cp4d-variables/vars/vars-cp4d-installation.yml   |  2 +-
 .../templates/apply-cluster-components.j2            |  2 +-
 4 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/set-license-service-project.yml

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
index b2dba790b..768a69ce9 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
@@ -20,6 +20,9 @@
     foundational_services_project: "{{ _p_current_cp4d_cluster.operators_project | default('cpd-operators') }}"
   when: _p_current_cp4d_cluster.cp4d_version >= "4.7.0"
 
+# Set the license server project to the correct value, dependent if the license service is already installed in cs-control
+- include_tasks: set-license-service-project.yml
+
 - debug:
     var: implemented_cloud_platform_types
 
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/set-license-service-project.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/set-license-service-project.yml
new file mode 100644
index 000000000..c42cc68af
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/set-license-service-project.yml
@@ -0,0 +1,12 @@
+---
+- name: Check if the license service runs in the {{ cs_control_project }} project
+  shell: |
+    oc get deploy ibm-licensing-service-instance \
+      -n {{ cs_control_project }}
+  failed_when: False
+  register: _get_license_service_instance
+
+- name: Set the license server project to {{ cs_control_project }} if it already runs there
+  set_fact:
+    license_service_project: "{{ cs_control_project }}"
+  when: _get_license_service_instance.rc == 0
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml
index a0706d1ac..95990472a 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml
@@ -9,7 +9,7 @@ cp4d_login_username: admin
 cp4d_repo_url: cp.icr.io/cp/cpd
 cp4d_repo_username: cp
 
-license_service_project: cs-control
+license_service_project: ibm-licensing
 scheduling_service_project: cpd-scheduler
 cert_manager_project: ibm-cert-manager
 cs_control_project: cs-control
diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2 b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2
index ab1221c4e..b51843efa 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/templates/apply-cluster-components.j2
@@ -9,4 +9,4 @@ apply-cluster-components \
     --migrate_from_cs_ns={{ foundational_services_project }} \
 {% endif -%}   
     --cert_manager_ns={{ cert_manager_project }} \
-    --licensing_ns={{ cs_control_project }} 2>&1 | tee {{ status_dir }}/log/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.log 2>&1
+    --licensing_ns={{ license_service_project }} 2>&1 | tee {{ status_dir }}/log/{{ current_cp4d_cluster.openshift_cluster_name }}-{{ current_cp4d_cluster.project }}-apply-cluster-components.log 2>&1

From f44d2ea0c59233d47120ecf7ab120384cd6c69d8 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Mon, 22 Jan 2024 13:02:40 +0000
Subject: [PATCH 09/24] #622 Remove duplicate code

---
 .../tasks/create-catalog-source-olm-utils.yml                | 5 -----
 .../cpfs/cp-fs-cluster-components/tasks/main.yml             | 5 -----
 2 files changed, 10 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-catalog-source/tasks/create-catalog-source-olm-utils.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-catalog-source/tasks/create-catalog-source-olm-utils.yml
index f5243d062..9528dd4a5 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-catalog-source/tasks/create-catalog-source-olm-utils.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-catalog-source/tasks/create-catalog-source-olm-utils.yml
@@ -13,11 +13,6 @@
     path: /tmp/work/offline
     state: absent
 
-- name: Create offline directory
-  file:
-    path: /tmp/work/offline
-    state: absent
-
 - name: If air-gapped, copy case files from {{ status_dir }}/cp4d/offline to /tmp/work/offline
   copy:
     src: "{{ status_dir }}/cp4d/offline"
diff --git a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
index 64905921b..dba7e8680 100644
--- a/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cpfs/cp-fs-cluster-components/tasks/main.yml
@@ -9,11 +9,6 @@
     path: /tmp/work/offline
     state: absent
 
-- name: Create offline directory
-  file:
-    path: /tmp/work/offline
-    state: absent
-
 - name: If air-gapped, copy case files from {{ status_dir }}/cp4d/offline to /tmp/work/offline
   copy:
     src: "{{ status_dir }}/cp4d/offline"

From 924a9de9a6531e174d72efd089f8952b80ab2594 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Mon, 22 Jan 2024 16:55:30 +0000
Subject: [PATCH 10/24] #626 Remove pip upgrade from Dockerfile

---
 Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 1364e80aa..9faa0d4d9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,7 +23,6 @@ RUN yum install -y yum-utils python38 python38-pip && \
     yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo && \
     yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
     yum install -y tar sudo unzip wget jq skopeo httpd-tools git hostname bind-utils iproute procps-ng && \
-    pip3 install --upgrade pip && \
     pip3 install jmespath pyyaml argparse python-benedict pyvmomi psutil && \
     alternatives --set python /usr/bin/python3 && \
     yum install -y vault && \

From 9e4cf84041075b625df5bac69d3234dfbc9c0250 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 25 Jan 2024 17:36:23 +0000
Subject: [PATCH 11/24] #625 Download case files when --skip-mirror-images

---
 .../cp4d/cp4d-case-save/tasks/main.yml        | 59 +++++++++++++++++++
 .../templates/mirror-images-case-save.j2      | 10 ++++
 playbooks/playbook-env-download.yml           | 15 ++++-
 3 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/tasks/main.yml
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/templates/mirror-images-case-save.j2

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/tasks/main.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/tasks/main.yml
new file mode 100644
index 000000000..59616005f
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/tasks/main.yml
@@ -0,0 +1,59 @@
+---
+- include_role:
+    name: cp4d-variables
+
+- name: Create /tmp/work directory
+  file:
+    path: /tmp/work
+    state: directory
+
+- name: Generate mirror-images command preview script to save case files
+  set_fact:
+    _mirror_images_case_save: "{{ lookup('template', 'mirror-images-case-save.j2') }}" 
+
+- name: Show mirror-images command to save case files
+  debug:
+    var: _mirror_images_case_save
+
+- name: Write script to "{{ status_dir }}/cp4d/mirror-images-case-save.sh"
+  copy:
+    content: "{{ _mirror_images_case_save }}"
+    dest: "{{ status_dir }}/cp4d/mirror-images-case-save.sh"
+    mode: u+rwx
+
+- name: Run mirror-images command to identify case files, logs are in {{ status_dir }}/log/mirror-images-case-save.log
+  shell: |
+    {{ status_dir }}/cp4d/mirror-images-case-save.sh > {{ status_dir }}/log/mirror-images-case-save.log 2>&1
+
+- name: Copy preview script to {{ status_dir }}/cp4d/mirror-images-case-save-ibm-pak.sh
+  copy:
+    content: "{{ lookup('file', '/tmp/work/preview.sh') }}"
+    dest: "{{ status_dir }}/cp4d/mirror-images-case-save-ibm-pak.sh"
+    mode: u+rwx
+
+- name: Comment out the mirroring of the images
+  replace:
+    path: "{{ status_dir }}/cp4d/mirror-images-case-save-ibm-pak.sh"
+    regexp: "{{ item }}"
+    replace: '#\1'
+  with_items:
+  - '(oc image mirror.*)'
+  - '(oc ibm-pak generate.*)'
+  - '(mv /tmp/work.*)'
+
+- name: Download case files, logs are in {{ status_dir }}/cp4d/mirror-images-case-save-ibm-pak.log
+  shell: |
+    {{ status_dir }}/cp4d/mirror-images-case-save-ibm-pak.sh > {{ status_dir }}/log/mirror-images-case-save-ibm-pak.log
+  args:
+    chdir: /tmp/work
+
+- name: Create {{ status_dir }}/cp4d/offline/{{ _p_current_cp4d_cluster.cp4d_version }} directory
+  file:
+    path: "{{ status_dir }}/cp4d/offline/{{ _p_current_cp4d_cluster.cp4d_version }}"
+    state: directory
+
+- name: Copy case files to {{ status_dir }}/cp4d/offline/{{ _p_current_cp4d_cluster.cp4d_version }}
+  copy:
+    src: "/opt/ansible/.ibm-pak"
+    dest: "{{ status_dir }}/cp4d/offline/{{ _p_current_cp4d_cluster.cp4d_version }}/"
+    remote_src: True
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/templates/mirror-images-case-save.j2 b/automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/templates/mirror-images-case-save.j2
new file mode 100644
index 000000000..7a21282b6
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-case-save/templates/mirror-images-case-save.j2
@@ -0,0 +1,10 @@
+mirror-images \
+    --release={{ _p_current_cp4d_cluster.cp4d_version }} \
+    --target_registry=127.0.0.1:12443 \
+    --preview=True \
+    --components={% for c in _cartridges_to_install -%}
+{%- if ((c.state | default('installed')) == 'installed') or (cpd_test_cartridges | default(False) | bool) -%}
+{%- if not loop.first -%},{% endif -%}
+{{ c.olm_utils_name }}
+{%- endif -%}
+{%- endfor -%}
\ No newline at end of file
diff --git a/playbooks/playbook-env-download.yml b/playbooks/playbook-env-download.yml
index b72512bef..5662e3228 100644
--- a/playbooks/playbook-env-download.yml
+++ b/playbooks/playbook-env-download.yml
@@ -145,7 +145,7 @@
 
   tasks:
   - block:
-    # Mirror images to portable or private registry
+
     - block:
       - name: Get IBM Entitled Registry key
         include_role: 
@@ -160,6 +160,19 @@
           msg: "Entitlement key must be stored in vault secret ibm_cp_entitlement_key."
         when: ibm_cp_entitlement_key == ""
 
+      when:
+      - not (cpd_skip_mirror | bool)
+
+    - name: Save case files
+      include_role:
+        name: cp4d-case-save
+      loop: "{{ all_config.cp4d | default([]) }}"
+      loop_control:
+        loop_var: _p_current_cp4d_cluster
+
+    # Mirror images to portable or private registry
+    - block:
+
       # Mirror to portable image registry
       - name: Mirror images to portable image registry
         include_role:

From 4e90a1c7ddda57a814ab605355261646ac302f6a Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 25 Jan 2024 17:59:19 +0000
Subject: [PATCH 12/24] #629 Rescue copy of list-components output

---
 .../tasks/list-components-olm-utils.yml       | 38 +++++++++++++------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml
index c5e163718..33925e263 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml
@@ -43,13 +43,20 @@
     shell: |
       {{ _list_components_command }}
 
-  - name: Copy file to {{ _cp4d_components_file }}
-    copy:
-      src: /tmp/work/components.csv
-      dest: "{{ _cp4d_components_file }}"
-      remote_src: True
-      force: True
-      mode: u+rwx,g+rwx,o+rwx
+  # Try to copy the list-components output file. In certain situations (Windows, SELinux), the command may fail and has to be re-run without remote_src
+  - block:
+    - name: Copy file to {{ _cp4d_components_file }}
+      copy:
+        src: /tmp/work/components.csv
+        dest: "{{ _cp4d_components_file }}"
+        remote_src: True
+        force: True
+        mode: u+rwx,g+rwx,o+rwx
+    rescue:
+    - name: Rescue copy file to {{ _cp4d_components_file }}
+      copy:
+        src: /tmp/work/components.csv
+        dest: "{{ _cp4d_components_file }}"
 
   when:
   - not (cpd_airgap | bool)
@@ -70,11 +77,18 @@
     head -1 {{ _cp4d_components_file }}
   register: _csv_column_headers
 
-- name: Copy file to _cp4d_components_file
-  copy:
-    src: "{{ _cp4d_components_file }}"
-    dest: "{{ _cp4d_components_no_header_file }}"
-    remote_src: True
+# Try to copy the list-components output file. In certain situations (Windows, SELinux), the command may fail and has to be re-run without remote_src
+- block:
+  - name: Copy file to _cp4d_components_file
+    copy:
+      src: "{{ _cp4d_components_file }}"
+      dest: "{{ _cp4d_components_no_header_file }}"
+      remote_src: True
+  rescue:
+  - name: Copy file to _cp4d_components_file
+    copy:
+      src: "{{ _cp4d_components_file }}"
+      dest: "{{ _cp4d_components_no_header_file }}"
 
 - name: Remove first line from file
   lineinfile:

From c5a415006b731ffcfebe4d0b25d5800cebacf006 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 25 Jan 2024 18:12:54 +0000
Subject: [PATCH 13/24] #628 Do not install Python

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9faa0d4d9..1cedbbcfc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,7 +19,7 @@ LABEL authors="Arthur Laimbock, \
 USER 0
 
 # Install required packages, including HashiCorp Vault client
-RUN yum install -y yum-utils python38 python38-pip && \
+RUN yum install -y yum-utils && \
     yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo && \
     yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
     yum install -y tar sudo unzip wget jq skopeo httpd-tools git hostname bind-utils iproute procps-ng && \

From ae35ad1099185dd6059f8ea23acc12c83fe35b3e Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 25 Jan 2024 19:46:48 +0000
Subject: [PATCH 14/24] #623 Deploy knative eventing

---
 .../cp4d/cp4d-cluster/tasks/install-cp4d.yml    |  3 +++
 .../tasks/openshift-install-knative.yml         | 17 +++++++++++++++++
 .../templates/deploy-knative-eventing.j2        |  5 +++++
 .../cp4d/cp4d-variables/tasks/main.yml          | 11 +++++++++++
 .../vars/vars-cp4d-installation.yml             |  1 +
 5 files changed, 37 insertions(+)
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml
index 27695d44b..2bcac1a71 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/install-cp4d.yml
@@ -39,6 +39,9 @@
     _p_preview: False
   when: current_cp4d_cluster.cp4d_version >= "4.7.0"
 
+- name: Prepare KNative for OpenShift
+  include_tasks: openshift-install-knative.yml
+
 - name: Prepare OpenShift project {{ current_cp4d_cluster.project }} for Cloud Pak for Data
   include_tasks: openshift-prepare-project.yml
 
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml
new file mode 100644
index 000000000..ffb16de91
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml
@@ -0,0 +1,17 @@
+---
+
+# Prepare CP4D for versions >= 4.8.0
+- block:
+  - name: Generate deploy KNative eventing script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deployer-knative-eventing.sh
+    template:
+      src: deploy-knative-eventing.j2
+      dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deployer-knative-eventing.sh"
+      mode: u+rwx
+
+  - name: Run script to deploy KNative eventing, output can be found in {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.log
+    shell: |
+      {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.sh
+
+  when: 
+  - current_cp4d_cluster.cp4d_version >= '4.8.0'
+  - (_knative_eventing_dependency | default(False))
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2 b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2
new file mode 100644
index 000000000..c95f541a8
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2
@@ -0,0 +1,5 @@
+set -o pipefail
+deploy-native-eventing \
+    --release={{ current_cp4d_cluster.cp4d_version }} \
+    --block_storage_class={{ ocp_storage_class_block }} \
+    2>&1 | tee {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.log
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
index 768a69ce9..4f44eeaae 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
@@ -181,6 +181,17 @@
 - include_tasks: check-db2u-kubelet.yml
   when: _db2u_dependency
 
+- name: Determine if any of the cartridges has a knative eventing dependency
+  set_fact:
+    _knative_eventing_dependency: "{{ _knative_eventing_dependency or (item.knative_eventing_dependency | default(False)) }}"
+  when: (item.state | default('installed')) == 'installed'
+  loop: "{{ _cartridges_to_install }}"
+  no_log: True
+
+- name: Show if there is a knative eventing dependency
+  debug:
+    var: _knative_eventing_dependency
+
 - name: Determine cartridge dependencies
   set_fact:
     _cartridge_dependencies: >-
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml
index 95990472a..1ff8bf40e 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/vars/vars-cp4d-installation.yml
@@ -312,6 +312,7 @@ cartridge_cr:
     cr_status_attribute: watsonAssistantStatus
     cr_status_completed: Completed
     force_sequential_install: True
+    knative_eventing_dependency: True
   - name: watson-discovery
     olm_utils_name: watson_discovery
     cr_preprocessing_script: cp4d-pre-wd.yml

From 2e5a2012d3f517a12b6ad59ad8a23130f7725afb Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 26 Jan 2024 09:11:48 +0000
Subject: [PATCH 15/24] #623 Install knative eventing

---
 .../tasks/openshift-install-knative.yml       |  4 +--
 .../templates/deploy-knative-eventing.j2      |  2 +-
 .../tasks/check-db2u-dependency.yml           | 15 ++++++++++
 .../tasks/check-knative-eventing.yml          | 14 ++++++++++
 .../cp4d/cp4d-variables/tasks/main.yml        | 28 +++----------------
 5 files changed, 36 insertions(+), 27 deletions(-)
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-db2u-dependency.yml
 create mode 100644 automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-knative-eventing.yml

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml
index ffb16de91..787cbab8d 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-install-knative.yml
@@ -2,10 +2,10 @@
 
 # Prepare CP4D for versions >= 4.8.0
 - block:
-  - name: Generate deploy KNative eventing script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deployer-knative-eventing.sh
+  - name: Generate deploy KNative eventing script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.sh
     template:
       src: deploy-knative-eventing.j2
-      dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deployer-knative-eventing.sh"
+      dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.sh"
       mode: u+rwx
 
   - name: Run script to deploy KNative eventing, output can be found in {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.log
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2 b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2
index c95f541a8..4d41c7c82 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/deploy-knative-eventing.j2
@@ -1,5 +1,5 @@
 set -o pipefail
-deploy-native-eventing \
+deploy-knative-eventing \
     --release={{ current_cp4d_cluster.cp4d_version }} \
     --block_storage_class={{ ocp_storage_class_block }} \
     2>&1 | tee {{ status_dir }}/log/{{ current_cp4d_cluster.project }}-deploy-knative-eventing.log
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-db2u-dependency.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-db2u-dependency.yml
new file mode 100644
index 000000000..c2f422144
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-db2u-dependency.yml
@@ -0,0 +1,15 @@
+---
+# Determine if any cartridges with a db2u dependency are installed
+- set_fact:
+    _db2u_dependency: False
+
+- name: Determine if any of the cartridges has a db2u dependency
+  set_fact:
+    _db2u_dependency: "{{ _db2u_dependency or (item.db2u_dependency | default(False)) }}"
+  when: (item.state | default('installed')) == 'installed'
+  loop: "{{ _cartridges_to_install }}"
+  no_log: True
+
+- name: Show if there is a db2u dependency
+  debug:
+    var: _db2u_dependency
\ No newline at end of file
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-knative-eventing.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-knative-eventing.yml
new file mode 100644
index 000000000..141bf118c
--- /dev/null
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/check-knative-eventing.yml
@@ -0,0 +1,14 @@
+---
+# Determine if any cartridges with a Knative dependency are installed
+- set_fact:
+    _knative_eventing_dependency: False
+
+- name: Determine if any of the cartridges has a Knative eventing dependency
+  set_fact:
+    _knative_eventing_dependency: "{{ _knative_eventing_dependency or (item.knative_eventing_dependency | default(False)) }}"
+  when: (item.state | default('installed')) == 'installed'
+  loop: "{{ _cartridges_to_install }}"
+
+- name: Show if there is a Knative eventing dependency
+  debug:
+    var: _knative_eventing_dependency
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
index 4f44eeaae..396b120ba 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/main.yml
@@ -160,20 +160,8 @@
   debug:
     var: _sequential_install
 
-# Determine if any cartridges with a db2u dependency are installed
-- set_fact:
-    _db2u_dependency: False
-
-- name: Determine if any of the cartridges has a db2u dependency
-  set_fact:
-    _db2u_dependency: "{{ _db2u_dependency or (item.db2u_dependency | default(False)) }}"
-  when: (item.state | default('installed')) == 'installed'
-  loop: "{{ _cartridges_to_install }}"
-  no_log: True
-
-- name: Show if there is a db2u dependency
-  debug:
-    var: _db2u_dependency
+# check if any of the cartridges have a db2u dependency
+- include_tasks: check-db2u-dependency.yml
 
 - include_tasks: check-db2u-node-tuning.yml
   when: _db2u_dependency
@@ -181,16 +169,8 @@
 - include_tasks: check-db2u-kubelet.yml
   when: _db2u_dependency
 
-- name: Determine if any of the cartridges has a knative eventing dependency
-  set_fact:
-    _knative_eventing_dependency: "{{ _knative_eventing_dependency or (item.knative_eventing_dependency | default(False)) }}"
-  when: (item.state | default('installed')) == 'installed'
-  loop: "{{ _cartridges_to_install }}"
-  no_log: True
-
-- name: Show if there is a knative eventing dependency
-  debug:
-    var: _knative_eventing_dependency
+# check if any of the cartridges have a knative-eventing dependency
+- include_tasks: check-knative-eventing.yml
 
 - name: Determine cartridge dependencies
   set_fact:

From d6f0c819b4853d13b5769b8d3fbe70cfb3ff7148 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Fri, 26 Jan 2024 09:43:34 +0000
Subject: [PATCH 16/24] #625 Fix message in the log

---
 .../cp4d/cp4d-variables/tasks/list-components-olm-utils.yml   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml
index 33925e263..fbff6a452 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-variables/tasks/list-components-olm-utils.yml
@@ -79,13 +79,13 @@
 
 # Try to copy the list-components output file. In certain situations (Windows, SELinux), the command may fail and has to be re-run without remote_src
 - block:
-  - name: Copy file to _cp4d_components_file
+  - name: Copy file to {{ _cp4d_components_no_header_file }}
     copy:
       src: "{{ _cp4d_components_file }}"
       dest: "{{ _cp4d_components_no_header_file }}"
       remote_src: True
   rescue:
-  - name: Copy file to _cp4d_components_file
+  - name: Copy file to {{ _cp4d_components_no_header_file }}
     copy:
       src: "{{ _cp4d_components_file }}"
       dest: "{{ _cp4d_components_no_header_file }}"

From 2b3a47bb8287d4e01b5d76132a130e737c1ca074 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Sat, 27 Jan 2024 08:54:31 +0000
Subject: [PATCH 17/24] #623 Also delete Knative eventing resources

---
 scripts/cp4d/cp4d-delete-instance.sh | 38 ++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/scripts/cp4d/cp4d-delete-instance.sh b/scripts/cp4d/cp4d-delete-instance.sh
index 5a95183f1..ec2d655ee 100755
--- a/scripts/cp4d/cp4d-delete-instance.sh
+++ b/scripts/cp4d/cp4d-delete-instance.sh
@@ -149,6 +149,44 @@ delete_operator_ns ibm-common-services
 # Delete operators in new operators namespace
 delete_operator_ns ${CP4D_PROJECT}-operators
 
+# Delete operators in ibm-knative-events
+delete_operator_ns ibm-knative-events
+
+KNATIVE_EVENTING=knative-eventing
+oc get project ${KNATIVE_EVENTING} > /dev/null 2>&1
+if [ $? -eq 0 ];then
+    log "Deleting everything in the ${KNATIVE_EVENTING} project"
+
+    log "Getting Custom Resources in OpenShift project ${KNATIVE_EVENTING}..."
+    oc get --no-headers -n $KNATIVE_EVENTING $(oc api-resources --namespaced=true --verbs=list -o name | grep ibm | awk '{printf "%s%s",sep,$0;sep=","}')  --ignore-not-found -o=custom-columns=KIND:.kind,NAME:.metadata.name --sort-by='kind' > ${temp_dir}/knative-eventing-resources.out
+
+    log "Delete all Custom Resources"
+    while read -r line;do
+        read -r CR CR_NAME <<< "${line}"
+        log "Deleting $CR $CR_NAME"
+        oc delete -n ${KNATIVE_EVENTING} ${CR} ${CR_NAME} --wait=false --ignore-not-found
+        oc patch -n ${KNATIVE_EVENTING} ${CR}/${CR_NAME} --type=merge -p '{"metadata": {"finalizers":null}}' 2> /dev/null
+    done < ${temp_dir}/knative-eventing-resources.out
+
+    log "Deleting ${KNATIVE_EVENTING} project"
+    oc delete ns ${KNATIVE_EVENTING} --ignore-not-found --wait=false
+    wait_ns_deleted ${KNATIVE_EVENTING}
+else
+    echo "Project ${KNATIVE_EVENTING} does not exist, skipping"
+fi
+
+KNATIVE_SERVING=knative-serving
+oc get project ${KNATIVE_SERVING} > /dev/null 2>&1
+if [ $? -eq 0 ];then
+    log "Deleting everything in the ${KNATIVE_SERVING} project"
+
+    log "Deleting ${KNATIVE_SERVING} project"
+    oc delete ns ${KNATIVE_SERVING} --ignore-not-found --wait=false
+    wait_ns_deleted ${KNATIVE_SERVING}
+else
+    echo "Project ${KNATIVE_SERVING} does not exist, skipping"
+fi
+
 IBM_SCHEDULING=ibm-scheduling
 oc get project ${IBM_SCHEDULING} > /dev/null 2>&1
 if [ $? -eq 0 ];then

From 37bdbdbccc13ff091575fd071c2f552b851e851d Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Sun, 28 Jan 2024 06:41:12 +0000
Subject: [PATCH 18/24] #631 Add Watson Assistant instance handling

---
 .../generic/cp4d/preprocessor.py              |   6 +-
 .../tasks/cp4d-delete-cr-instances.yml        |   6 +
 .../cp4d-cluster-configure/tasks/main.yml     |   2 +
 .../tasks/delete_wa_instance.yml              |  62 ++++
 .../tasks/delete_wa_instances.yml             |  19 ++
 .../tasks/main.yml                            |  32 ++
 .../tasks/provision_wa_instance.yml           |  46 +++
 .../tasks/provision_wa_instances.yml          |   6 +
 .../tasks/wait_wa_instance.yml                |  39 +++
 .../tasks/wait_wa_instances.yml               |   6 +
 .../templates/wa-instance.j2                  |   6 +
 .../vars/main.yml                             |   2 +
 .../config-samples/cp4d-480.yaml              |   3 +
 .../config-samples/cp4d-482.yaml              | 310 ++++++++++++++++++
 14 files changed, 544 insertions(+), 1 deletion(-)
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/main.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/vars/main.yml
 create mode 100644 sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml

diff --git a/automation-generators/generic/cp4d/preprocessor.py b/automation-generators/generic/cp4d/preprocessor.py
index 7c232629f..5e350417d 100644
--- a/automation-generators/generic/cp4d/preprocessor.py
+++ b/automation-generators/generic/cp4d/preprocessor.py
@@ -340,13 +340,17 @@ def preprocessor(attributes=None, fullConfig=None, moduleVariables=None):
                                     dep_found=True
                             if not dep_found:
                                 g.appendError(msg='Cartridge {} is selected to be installed but dependent cartridge {} is not'. format(c['name'],dep['name']))
+            # If instances for cartridge are specified, iterate over instances
+            if 'instances' in c and 'name' in c:
+                for i in c['instances']:
+                    if "name" not in i:
+                        g.appendError(msg='Instance name must be specifed for every instance in cartridge {}'.format(c['name']))        
         # Iteration over cartridges is done, now check if the required fields were found in the for-loop
         if cpfsFound==False:
             g.appendError(msg='You need to specify a cartridge for the Cloud Pak Foundational Services (cpfs or cp-foundation)')
         if cpdPlatformFound==False:
             g.appendError(msg='You need to specify a cartridge for the Cloud Pak for Data platform (cpd_platform or lite)')
         
-        
     result = {
         'attributes_updated': g.getExpandedAttributes(),
         'errors': g.getErrors()
diff --git a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-remove/tasks/cp4d-delete-cr-instances.yml b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-remove/tasks/cp4d-delete-cr-instances.yml
index 351b73238..3c5578dd0 100644
--- a/automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-remove/tasks/cp4d-delete-cr-instances.yml
+++ b/automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-remove/tasks/cp4d-delete-cr-instances.yml
@@ -48,3 +48,9 @@
     _p_delete_all_instances: True
   when: (_current_cartridge_cr.olm_utils_name | default("")) == "watsonx_ai"
 
+- name: Delete all Watson Assistant instances
+  include_role:
+    name: cp4d-instance-wa
+  vars:
+    _p_delete_all_instances: True
+  when: (_current_cartridge_cr.olm_utils_name | default("")) == "wa"
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml
index ddf4fee15..fb772d39b 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml
@@ -125,6 +125,7 @@
         - dv
         - openpages
         - watsonx_ai
+        - watson_assistant
 
     # Wait for provisioning to complete
     - name: Wait for provisioning of {{ item }} instances to complete
@@ -140,6 +141,7 @@
         - dv
         - openpages
         - watsonx_ai
+        - watson_assistant
 
 # Handle instance configurations
 - when: "all_config.cp4d_instance_configuration is defined"
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instance.yml
new file mode 100644
index 000000000..e8149574a
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instance.yml
@@ -0,0 +1,62 @@
+---
+- set_fact:
+    _wa_instance_name: "{{ _wa_instance.display_name }}"
+    _wa_instance_id: "{{ _wa_instance.id }}"
+    _wa_cartridge_listed: {}
+    _delete_wa_instance: False
+
+# If not all instances must be deleted, check if instance still in the configuration
+- block:
+  - set_fact:
+      _wa_cartridge_listed: "{{ _wa_cartridge.instances | json_query(_query_instance) | first | default({}) }}"
+    vars:
+      _query_instance: "[?name=='{{ _wa_instance_name }}']" 
+    when: _wa_cartridge.instances is defined
+  - set_fact:
+      _delete_wa_instance: True
+    when: _wa_cartridge_listed == {}
+  when: (_p_delete_all_instances | default(False)) == False
+
+- set_fact:
+    _delete_wa_instance: True
+  when: (_p_delete_all_instances | default(False)) == True
+
+- name: Fail if Watson Assistant instance will be deleted and destroy is not confirmed
+  fail:
+    msg: "Destroy was not confirmed via --confirm-destroy or global_config confirm_destroy property, will not delete Watson Assistant instance {{ _wa_instance_name }}"
+  when:
+  - _delete_wa_instance | bool
+  - _confirm_destroy | default (False) | bool == False
+
+- name: Deleting Watson Assistant instance {{ _wa_instance_name }}
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _wa_instance_id }}'
+    method: DELETE
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+  register: _wa_instance_delete_result
+  when: _delete_wa_instance | bool
+
+- name: Wait until Watson Assistant instance {{ _wa_instance_name }} has been deleted
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _wa_instance_id }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 404
+  register: _wa_instance_exists
+  until: _wa_instance_exists.status == 404
+  retries: 30
+  delay: 30
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
+  when: _delete_wa_instance | bool
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instances.yml
new file mode 100644
index 000000000..9463e872d
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/delete_wa_instances.yml
@@ -0,0 +1,19 @@
+---
+- name: Get all current Watson Assistant instances
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_wa_instance_addon_type }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _all_wa_instances
+
+# Loop through all deployed Watson Assistant instance definitions to check if instance no longer exists in the config
+- name: Loop through each Watson Assistant instance to check if it must be deleted
+  include_tasks: delete_wa_instance.yml
+  loop: "{{ _all_wa_instances.json.service_instances | default([]) }}"
+  loop_control:
+    loop_var: _wa_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/main.yml
new file mode 100644
index 000000000..dfbe96934
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+- set_fact:
+    _wa_cartridge: "{{ _cartridges_to_install | json_query(query_cartridge) | first | default({}) }}"
+  vars:
+    cartridge_name: "watson_assistant"
+    query_cartridge: "[?olm_utils_name=='{{ cartridge_name }}']"
+
+- name: Show Watson Assistant cartridge
+  debug:
+    var: _wa_cartridge
+
+- set_fact:
+    _configured_wa_instances: []
+- set_fact:
+    _configured_wa_instances: "{{ _wa_cartridge.instances | default([] )}}"
+  when:
+  - _wa_cartridge != {}
+  - (_wa_cartridge.state | default('installed')) == 'installed'
+
+- include_tasks: provision_wa_instances.yml
+  when:
+  - (_p_delete_all_instances | default(False)) == False
+  - (_p_wait_instances | default(False)) == False
+
+- include_tasks: delete_wa_instances.yml
+  when:
+  - (_p_wait_instances | default(False)) == False
+
+- include_tasks: wait_wa_instances.yml
+  when:
+  - (_p_delete_all_instances | default(False)) == False
+  - _p_wait_instances | default(False)
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
new file mode 100644
index 000000000..08aeedf15
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
@@ -0,0 +1,46 @@
+---
+- name: Show Watson Assistant instance
+  debug:
+    var: _wa_instance
+
+- name: Check if the Watson Assistant instance is already running
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_wa_instance_addon_type }}&display_name={{ _wa_instance.name }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _wa_instance_lookup_result
+
+- name: Determine if Watson Assistant instance already exists
+  debug:
+    msg: "Watson Assistant instance {{ _wa_instance.name }} already exists in OpenShift project {{ current_cp4d_cluster.project }}, skipping provisioning"
+  when: _wa_instance_lookup_result.json.total_count != 0
+
+- when: _wa_instance_lookup_result.json.total_count == 0
+  block:
+
+  - name: Prepare Watson Assistant instance {{ _wa_instance.name }} json file
+    template:
+      src: wa-instance.j2
+      dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _wa_instance.name }}-wa-instance.json"
+
+  - name: Create Watson Assistant instance {{ _wa_instance.name }}
+    shell: |
+      curl -v -k \
+        -H 'Authorization: Bearer {{ _cp4d_bearer_token }}' \
+        -H 'Content-Type: application/json' \
+        -X POST \
+        'https://{{ cp4d_url.stdout }}/watson/common/assistant/api/ibmcloud/resource-controller/resource_instances' \
+        -T {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _wa_instance.name }}-wa-instance.json
+    register: _create_wa_instance_result
+
+  - set_fact:
+      _create_wa_instance_result_json: "{{ _create_wa_instance_result.stdout | from_json }}"
+
+  - name: Show Watson Assistant instance result
+    debug:
+      var: _create_wa_instance_result_json
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instances.yml
new file mode 100644
index 000000000..00586c5c9
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instances.yml
@@ -0,0 +1,6 @@
+---
+- name: Provision Watson Assistant instance from the cartridge specification
+  include_tasks: provision_wa_instance.yml
+  loop: "{{ _configured_wa_instances }}"
+  loop_control:
+    loop_var: _wa_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instance.yml
new file mode 100644
index 000000000..b0fdd8b39
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instance.yml
@@ -0,0 +1,39 @@
+---
+- name: Show Watson Assistant instance
+  debug:
+    var: _wa_instance
+
+- name: Retrieve running Watson Assistant instance
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_wa_instance_addon_type }}&display_name={{ _wa_instance.name }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _wa_instance_lookup_result
+
+- set_fact:
+    _wa_instance_id: "{{ (_wa_instance_lookup_result.json.service_instances | first).id }}"
+
+- name: Waiting for Watson Assistant instance {{ _wa_instance.name }} to complete its provisioning
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _wa_instance_id }}?include_service_status=true'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _wa_instance_lookup_result
+  until: _wa_instance_lookup_result.json.services_status == "RUNNING"
+  retries: 60
+  delay: 30
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instances.yml
new file mode 100644
index 000000000..8265e9659
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/wait_wa_instances.yml
@@ -0,0 +1,6 @@
+---
+- name: Wait for Watson Assistant instance to provision
+  include_tasks: wait_wa_instance.yml
+  loop: "{{ _configured_wa_instances }}"
+  loop_control:
+    loop_var: _wa_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2 b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
new file mode 100644
index 000000000..7901efae6
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
@@ -0,0 +1,6 @@
+{
+   "name":"{{ _wa_instance.name }}",
+   "deployment_id":"cpd-wa",
+   "description":"{{ _ae_instance.description | default('Watson Assistant') }}",
+   "url":"https://wa-store.cpd.svc.cluster.local:443/csb/v2/service_instances"
+}
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/vars/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/vars/main.yml
new file mode 100644
index 000000000..2d24be5ee
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/vars/main.yml
@@ -0,0 +1,2 @@
+---
+cp4d_wa_instance_addon_type: assistant
\ No newline at end of file
diff --git a/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml b/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml
index a1abcb72b..b65a1a11f 100644
--- a/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml
+++ b/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml
@@ -199,6 +199,9 @@ cp4d:
     # noobaa_account_secret: noobaa-admin
     # noobaa_cert_secret: noobaa-s3-serving-cert
     state: removed
+    instances:
+    - name: wa-instance
+      description: "Watson Assistant instance"
 
   - name: watson-discovery
     description: Watson Discovery
diff --git a/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml b/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml
new file mode 100644
index 000000000..72b11a7eb
--- /dev/null
+++ b/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml
@@ -0,0 +1,310 @@
+---
+cp4d:
+- project: cpd
+  openshift_cluster_name: "{{ env_id }}"
+  cp4d_version: 4.8.2
+  cp4d_entitlement: cpd-enterprise
+  cp4d_production_license: True
+  accept_licenses: False
+  sequential_install: False
+  db2u_limited_privileges: False
+  use_fs_iam: True
+  operators_project: cpd-operators
+  cartridges:
+  - name: cp-foundation
+    license_service:
+      state: disabled
+      threads_per_core: 2
+  
+  - name: lite
+
+  - name: scheduler 
+    state: removed
+    
+#
+# All tested cartridges. To install, change the "state" property to "installed". To uninstall, change the state
+# to "removed" or comment out the entire cartridge. Make sure that the "-" and properties are aligned with the lite
+# cartridge; the "-" is at position 3 and the property starts at position 5.
+#
+# If a cartridge has dependencies and you want to install it, you must ensure that the dependent cartridge is also
+# installed.
+#
+
+  - name: analyticsengine 
+    description: Analytics Engine Powered by Apache Spark 
+    size: small 
+    state: removed
+
+  - name: bigsql
+    description: Db2 Big SQL
+    state: removed
+
+  - name: ca
+    description: Cognos Analytics
+    size: small
+    instances:
+    - name: ca-instance
+      metastore_ref: ca-metastore
+    state: removed
+
+  - name: dashboard
+    description: Cognos Dashboards
+    state: removed
+
+  - name: datagate
+    description: Db2 Data Gate
+    state: removed
+
+  - name: datastage-ent
+    description: DataStage Enterprise
+    state: removed
+
+  - name: datastage-ent-plus
+    description: DataStage Enterprise Plus
+    state: removed
+    # instances:
+    #   - name: ds-instance
+    #     # Optional settings
+    #     description: "datastage ds-instance"
+    #     size: medium
+    #     storage_class: efs-nfs-client
+    #     storage_size_gb: 60
+    #     # Custom Scale options
+    #     scale_px_runtime:
+    #       replicas: 2
+    #       cpu_request: 500m
+    #       cpu_limit: 2
+    #       memory_request: 2Gi
+    #       memory_limit: 4Gi
+    #     scale_px_compute:
+    #       replicas: 2
+    #       cpu_request: 1
+    #       cpu_limit: 3
+    #       memory_request: 4Gi
+    #       memory_limit: 12Gi    
+
+  - name: db2
+    description: Db2 OLTP
+    size: small
+    instances:
+    - name: ca-metastore
+      metadata_size_gb: 20
+      data_size_gb: 20
+      backup_size_gb: 20  
+      transactionlog_size_gb: 20
+    state: removed
+
+  - name: db2wh
+    description: Db2 Warehouse
+    state: removed
+
+  - name: dmc
+    description: Db2 Data Management Console
+    state: removed
+
+  - name: dods
+    description: Decision Optimization
+    size: small
+    state: removed
+
+  - name: dp
+    description: Data Privacy
+    size: small
+    state: removed
+
+  - name: dpra
+    description: Data Privacy Risk Assessment
+    state: removed
+
+  - name: dv
+    description: Data Virtualization
+    size: small 
+    instances:
+    - name: data-virtualization
+    state: removed
+
+  # Please note that for EDB Postgress, a secret edb-postgres-license-key must be created in the vault
+  # before deploying
+  - name: edb_cp4d
+    description: EDB Postgres
+    state: removed
+    instances:
+    - name: instance1
+      version: "15.4"
+      #type: Standard
+      #members: 1
+      #size_gb: 50
+      #resource_request_cpu: 1
+      #resource_request_memory: 4Gi
+      #resource_limit_cpu: 1
+      #resource_limit_memory: 4Gi
+
+  - name: factsheet
+    description: AI Factsheets
+    size: small
+    state: removed
+
+  - name: hadoop
+    description: Execution Engine for Apache Hadoop
+    size: small
+    state: removed
+
+  - name: mantaflow
+    description: MANTA Automated Lineage
+    size: small
+    state: removed
+
+  - name: match360
+    description: IBM Match 360
+    size: small
+    wkc_enabled: true
+    state: removed
+
+  - name: openpages
+    description: OpenPages
+    state: removed
+
+  # For Planning Analytics, the case version is needed due to defect in olm utils
+  - name: planning-analytics
+    description: Planning Analytics
+    state: removed
+
+  - name: replication
+    description: Data Replication
+    license: IDRC
+    size: small
+    state: removed
+
+  - name: rstudio
+    description: RStudio Server with R 3.6
+    size: small
+    state: removed
+
+  - name: spss
+    description: SPSS Modeler
+    state: removed
+
+  - name: syntheticdata
+    description: Synthetic Data Generator
+    state: removed
+
+  - name: voice-gateway
+    description: Voice Gateway
+    replicas: 1
+    state: removed
+
+  - name: watson-assistant
+    description: Watson Assistant
+    size: small
+    # noobaa_account_secret: noobaa-admin
+    # noobaa_cert_secret: noobaa-s3-serving-cert
+    state: removed
+    instances:
+    - name: wa-instance
+      description: "Watson Assistant instance"
+
+  - name: watson-discovery
+    description: Watson Discovery
+    # noobaa_account_secret: noobaa-admin
+    # noobaa_cert_secret: noobaa-s3-serving-cert
+    state: removed
+
+  - name: watson-ks
+    description: Watson Knowledge Studio
+    size: small
+    # noobaa_account_secret: noobaa-admin
+    # noobaa_cert_secret: noobaa-s3-serving-cert
+    state: removed
+
+  - name: watson-openscale
+    description: Watson OpenScale
+    size: small
+    state: removed
+
+  - name: watson-speech
+    description: Watson Speech (STT and TTS)
+    stt_size: xsmall
+    tts_size: xsmall
+    # noobaa_account_secret: noobaa-admin
+    # noobaa_cert_secret: noobaa-s3-serving-cert
+    state: removed
+
+  # Please note that for watsonx.ai foundation models, you neeed to install the
+  # Node Feature Discovery and NVIDIA GPU operators. You can do so by setting the openshift.gpu.install property to True
+  - name: watsonx_ai
+    description: watsonx.ai
+    state: removed
+    models:
+    - model_id: google-flan-t5-xxl
+      state: removed
+    - model_id: google-flan-ul2
+      state: removed
+    - model_id: eleutherai-gpt-neox-20b
+      state: removed
+    - model_id: ibm-granite-13b-chat-v1
+      state: removed
+    - model_id: ibm-granite-13b-instruct-v1
+      state: removed
+    - model_id: meta-llama-llama-2-70b-chat
+      state: removed
+    - model_id: ibm-mpt-7b-instruct2
+      state: removed
+    - model_id: bigscience-mt0-xxl
+      state: removed
+    - model_id: bigcode-starcoder
+      state: removed
+
+  - name: watsonx_data
+    description: watsonx.data
+    state: removed
+
+  - name: wkc
+    description: Watson Knowledge Catalog
+    size: small
+    state: removed
+    installation_options:
+      install_wkc_core_only: False
+      enableKnowledgeGraph: False
+      enableDataQuality: False
+      enableFactSheet: False
+
+  - name: wml
+    description: Watson Machine Learning
+    size: small
+    state: installed
+
+  - name: wml-accelerator
+    description: Watson Machine Learning Accelerator
+    replicas: 1
+    size: small
+    state: removed
+
+  - name: ws
+    description: Watson Studio
+    state: installed
+
+  - name: ws-pipelines
+    description: Watson Studio Pipelines
+    state: removed
+
+  - name: ws-runtimes
+    description: Watson Studio Runtimes
+    runtimes:
+    - ibm-cpd-ws-runtime-py39
+    - ibm-cpd-ws-runtime-222-py
+    - ibm-cpd-ws-runtime-py39gpu
+    - ibm-cpd-ws-runtime-222-pygpu
+    - ibm-cpd-ws-runtime-231-pygpu
+    - ibm-cpd-ws-runtime-r36
+    - ibm-cpd-ws-runtime-222-r
+    - ibm-cpd-ws-runtime-231-r
+    state: removed 
+
+#
+# Cartridges where extra work is needed (will not install automatically)
+# 
+  # Product Master requires set up of the Db2 instance secret before install
+  - name: productmaster
+    description: Product Master
+    size: small  
+    state: removed
\ No newline at end of file

From fd006c6a07c4b90d562e1e00b025fba3118958e2 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Sun, 28 Jan 2024 16:58:24 +0000
Subject: [PATCH 19/24] #631 Add DMC instance

---
 .../tasks/delete_dmc_instance.yml             | 63 +++++++++++++++++++
 .../tasks/delete_dmc_instances.yml            | 19 ++++++
 .../cp4d/cp4d-instance-dmc/tasks/main.yml     | 32 ++++++++++
 .../tasks/provision_dmc_instance.yml          | 49 +++++++++++++++
 .../tasks/provision_dmc_instances.yml         |  6 ++
 .../tasks/wait_dmc_instance.yml               | 47 ++++++++++++++
 .../tasks/wait_dmc_instances.yml              |  6 ++
 .../templates/dmc-instance.j2                 | 19 ++++++
 .../cp4d/cp4d-instance-dmc/vars/main.yml      |  2 +
 9 files changed, 243 insertions(+)
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/vars/main.yml

diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instance.yml
new file mode 100644
index 000000000..9e669cf59
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instance.yml
@@ -0,0 +1,63 @@
+---
+- set_fact:
+    _dmc_instance_name: "{{ _dmc_instance.display_name }}"
+    _dmc_instance_id: "{{ _dmc_instance.id }}"
+    _dmc_cartridge_listed: {}
+    _delete_dmc_instance: False
+
+# If not all instances must be deleted, check if instance still in the configuration
+- block:
+  - set_fact:
+      _dmc_cartridge_listed: "{{ _dmc_cartridge.instances | json_query(_query_instance) | first | default({}) }}"
+    vars:
+      _query_instance: "[?name=='{{ _dmc_instance_name }}']" 
+    when: _dmc_cartridge.instances is defined
+  - set_fact:
+      _delete_dmc_instance: True
+    when: _dmc_cartridge_listed == {} or (_dmc_cartridge.state | default('installed')) == 'removed'
+  when: (_p_delete_all_instances | default(False)) == False
+
+- set_fact:
+    _delete_dmc_instance: True
+  when: (_p_delete_all_instances | default(False)) == True
+
+- name: Fail if Data Management Console instance will be deleted and destroy is not confirmed
+  fail:
+    msg: "Destroy was not confirmed via --confirm-destroy or global_config confirm_destroy property, will not delete Data Management Console instance {{ _dmc_instance_name }}"
+  when:
+  - _delete_dmc_instance | bool
+  - _confirm_destroy | default (False) | bool == False
+
+- name: Deleting Data Management Console instance {{ _dmc_instance_name }}
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _dmc_instance_id }}'
+    method: DELETE
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 202
+  register: _dmc_instance_delete_result
+  when: _delete_dmc_instance | bool
+
+- name: Wait until Data Management Console instance {{ _dmc_instance_name }} has been deleted
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _dmc_instance_id }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 404
+  register: _dmc_instance_exists
+  until: _dmc_instance_exists.status == 404
+  retries: 30
+  delay: 30
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
+  when: _delete_dmc_instance | bool
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instances.yml
new file mode 100644
index 000000000..02845aafb
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/delete_dmc_instances.yml
@@ -0,0 +1,19 @@
+---
+- name: Get all current Data Management Console instances
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_dmc_addon_type }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _all_dmc_instances
+
+# Loop through all deployed Data Management Console instance definitions to check if instance no longer exists in the config
+- name: Loop through each Data Management Console instance to check if it must be deleted
+  include_tasks: delete_dmc_instance.yml
+  loop: "{{ _all_dmc_instances.json.service_instances | default([]) }}"
+  loop_control:
+    loop_var: _dmc_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml
new file mode 100644
index 000000000..e860d09ae
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+- set_fact:
+    _dmc_cartridge: "{{ _cartridges_to_install | json_query(query_cartridge) | first | default({}) }}"
+  vars:
+    cartridge_name: "dmc"
+    query_cartridge: "[?olm_utils_name=='{{ cartridge_name }}']"
+
+- name: Show Data Management Console cartridge
+  debug:
+    var: _dmc_cartridge
+
+- set_fact:
+    _configured_dmc_instances: []
+- set_fact:
+    _configured_dmc_instances: "{{ _dmc_cartridge.instances | default([] )}}"
+  when:
+  - _dmc_cartridge != {}
+  - (_dmc_cartridge.state | default('installed')) == 'installed'
+
+- include_tasks: provision_dmc_instances.yml
+  when:
+  - (_p_delete_all_instances | default(False)) == False
+  - (_p_wait_instances | default(False)) == False
+
+- include_tasks: delete_dmc_instances.yml
+  when:
+  - (_p_wait_instances | default(False)) == False
+
+- include_tasks: wait_dmc_instances.yml
+  when:
+  - (_p_delete_all_instances | default(False)) == False
+  - _p_wait_instances | default(False)
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instance.yml
new file mode 100644
index 000000000..3cd01a12d
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instance.yml
@@ -0,0 +1,49 @@
+---
+- name: Show current Data Management Console instance
+  debug:
+    msg: "{{ _dmc_instance }}"
+
+- name: Check if the Data Management Console instance is already running
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_dmc_addon_type }}&display_name={{ _dmc_instance.name }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _dmc_instance_lookup_result
+
+- name: Determine if Data Management Console instance already exists
+  debug:
+    msg: "Data Management Console instance {{ _dmc_instance.name }} already exists in OpenShift project {{ current_cp4d_cluster.project }}, will be skipped"
+  when: _dmc_instance_lookup_result.json.total_count != 0
+
+- when: _dmc_instance_lookup_result.json.total_count == 0
+  block:
+  - name: Obtain Data Management Console version
+    shell: |
+      oc get dmcaddon -n {{ current_cp4d_cluster.project }} dmc-addon -o jsonpath='{.spec.version}'
+    register: _dmc_version
+
+  - name: Create Data Management Console configuration file
+    template:
+      src: dmc-instance.j2
+      dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _dmc_instance.name }}-dmc-instance.json"
+
+  - name: Create Data Management Console instance
+    shell: |
+      curl -v -k \
+        -H 'Authorization: Bearer {{ _cp4d_bearer_token }}' \
+        -H 'Content-Type: application/json' \
+        -X POST 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances' \
+        -T {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _dmc_instance.name }}-dmc-instance.json
+    register: _provision_dmc_result
+
+  - name: Show Data Management Console instance provisioning result
+    debug:
+      msg: "{{ _provision_dmc_result }}"
+
+  - set_fact:
+      _provision_dmc_result_json: "{{ _provision_dmc_result.stdout | from_json }}"
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instances.yml
new file mode 100644
index 000000000..3c222308b
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/provision_dmc_instances.yml
@@ -0,0 +1,6 @@
+---
+- name: Provision Data Management Console instance from the dmc cartridge specification
+  include_tasks: provision_dmc_instance.yml
+  loop: "{{ _configured_dmc_instances }}"
+  loop_control:
+    loop_var: _dmc_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instance.yml
new file mode 100644
index 000000000..1f6d9c599
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instance.yml
@@ -0,0 +1,47 @@
+---
+- name: Show current Data Management Console instance
+  debug:
+    msg: "{{ _dmc_instance }}"
+
+- name: Retrieve running Data Management Console instance
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_dmc_addon_type }}&display_name={{ _dmc_instance.name }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _dmc_instance_lookup_result
+  until: _dmc_instance_lookup_result.json.total_count == 1
+  retries: 60
+  delay: 10
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
+
+- set_fact:
+    _dmc_instance_id: "{{ (_dmc_instance_lookup_result.json.service_instances | first).id }}"
+
+- name: Waiting for Data Management Console instance {{ _dmc_instance.name }} to complete its provisioning
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _dmc_instance_id }}?include_service_status=true'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _dmc_instance_lookup_result
+  until: _dmc_instance_lookup_result.json.services_status == "RUNNING"
+  retries: 120
+  delay: 60
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instances.yml
new file mode 100644
index 000000000..f322471d7
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/wait_dmc_instances.yml
@@ -0,0 +1,6 @@
+---
+- name: Wait for Data Management Console instance to provision
+  include_tasks: wait_dmc_instance.yml
+  loop: "{{ _configured_dmc_instances }}"
+  loop_control:
+    loop_var: _dmc_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2 b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2
new file mode 100644
index 000000000..e76146370
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2
@@ -0,0 +1,19 @@
+{
+   "addon_type": "{{ cp4d_dmc_addon_type }}",
+   "addon_version": "{{ _dmc_version }}",
+   "display_name": "{{ _dmc_instance.name }}",
+   "description": "{{ _dmc_instance.description | default('IBM Db2 Data Management Console') }}",
+   "namespace": "{{ current_cp4d_cluster.project }}",
+   "parameters": {
+      "storageClass": "{{ ocp_storage_class_file }}",
+      "storageSize": "{{ _dmc_instance.storage_size_gb | default('10') }}Gi"
+   },
+   "create_arguments": {
+      "metadata": {
+         "storageClass": "{{ ocp_storage_class_file }}",
+         "storageSize": "{{ _dmc_instance.storage_size_gb | default('10') }}Gi",
+         "scaleConfig": "{{ _dmc_instance.size | default('medium') }}"
+      }
+   },
+   "pre_existing_owner": false
+}
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/vars/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/vars/main.yml
new file mode 100644
index 000000000..4cdc57ea1
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/vars/main.yml
@@ -0,0 +1,2 @@
+---
+cp4d_dmc_addon_type: dmc
\ No newline at end of file

From 63e959b6fbcc4fe109b781a1af8edb65f4c61195 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Sun, 28 Jan 2024 16:59:03 +0000
Subject: [PATCH 20/24] #631 Fix WA instance

---
 .../tasks/provision_wa_instance.yml              |  6 +++++-
 .../templates/wa-instance.j2                     | 16 ++++++++++++----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
index 08aeedf15..58d11685c 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
@@ -22,6 +22,10 @@
 
 - when: _wa_instance_lookup_result.json.total_count == 0
   block:
+  - name: Obtain Watson Assistant Console version
+    shell: |
+      oc get wa -n {{ current_cp4d_cluster.project }} wa -o jsonpath='{.spec.version}'
+    register: _wa_version
 
   - name: Prepare Watson Assistant instance {{ _wa_instance.name }} json file
     template:
@@ -34,7 +38,7 @@
         -H 'Authorization: Bearer {{ _cp4d_bearer_token }}' \
         -H 'Content-Type: application/json' \
         -X POST \
-        'https://{{ cp4d_url.stdout }}/watson/common/assistant/api/ibmcloud/resource-controller/resource_instances' \
+        'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances' \
         -T {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _wa_instance.name }}-wa-instance.json
     register: _create_wa_instance_result
 
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2 b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
index 7901efae6..59d4fddcf 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
@@ -1,6 +1,14 @@
 {
-   "name":"{{ _wa_instance.name }}",
-   "deployment_id":"cpd-wa",
-   "description":"{{ _ae_instance.description | default('Watson Assistant') }}",
-   "url":"https://wa-store.cpd.svc.cluster.local:443/csb/v2/service_instances"
+    "addon_type": "{{ cp4d_wa_instance_addon_type }}",
+    "display_name": "{{ _wa_instance.name }}",
+    "namespace": "{{ current_cp4d_cluster.project }}",
+    "addon_version": "{{ _wa_version }}",
+    "create_arguments": {
+        "deployment_id": "{{ current_cp4d_cluster.project }}-wa", 
+	     "parameters": { 
+	         "serviceId": "assistant", 
+	         "url": "https://wa-store.{{ current_cp4d_cluster.project }}.svc.cluster.local:443/csb/v2/service_instances", 
+	         "watson": true 
+        }
+    }
 }
\ No newline at end of file

From 271ff661b4c4fb16fe47f0a627cb00557dab64bf Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Tue, 30 Jan 2024 06:46:52 +0000
Subject: [PATCH 21/24] #631 Add Watson Discovery instances through deployer

---
 .../tasks/provision_wa_instance.yml           |  2 +-
 .../templates/wa-instance.j2                  |  2 +-
 .../tasks/delete_wd_instance.yml              | 62 +++++++++++++++++++
 .../tasks/delete_wd_instances.yml             | 19 ++++++
 .../tasks/main.yml                            | 32 ++++++++++
 .../tasks/provision_wd_instance.yml           | 51 +++++++++++++++
 .../tasks/provision_wd_instances.yml          |  6 ++
 .../tasks/wait_wd_instance.yml                | 39 ++++++++++++
 .../tasks/wait_wd_instances.yml               |  6 ++
 .../templates/wd-instance.j2                  | 14 +++++
 .../vars/main.yml                             |  2 +
 11 files changed, 233 insertions(+), 2 deletions(-)
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/main.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instance.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instances.yml
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/templates/wd-instance.j2
 create mode 100644 automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/vars/main.yml

diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
index 58d11685c..b69d4ec2c 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/tasks/provision_wa_instance.yml
@@ -22,7 +22,7 @@
 
 - when: _wa_instance_lookup_result.json.total_count == 0
   block:
-  - name: Obtain Watson Assistant Console version
+  - name: Obtain Watson Assistant version
     shell: |
       oc get wa -n {{ current_cp4d_cluster.project }} wa -o jsonpath='{.spec.version}'
     register: _wa_version
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2 b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
index 59d4fddcf..0178bcf5e 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_assistant/templates/wa-instance.j2
@@ -2,7 +2,7 @@
     "addon_type": "{{ cp4d_wa_instance_addon_type }}",
     "display_name": "{{ _wa_instance.name }}",
     "namespace": "{{ current_cp4d_cluster.project }}",
-    "addon_version": "{{ _wa_version }}",
+    "addon_version": "{{ _wa_version.stdout }}",
     "create_arguments": {
         "deployment_id": "{{ current_cp4d_cluster.project }}-wa", 
 	     "parameters": { 
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instance.yml
new file mode 100644
index 000000000..32bd6556f
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instance.yml
@@ -0,0 +1,62 @@
+---
+- set_fact:
+    _wd_instance_name: "{{ _wd_instance.display_name }}"
+    _wd_instance_id: "{{ _wd_instance.id }}"
+    _wd_cartridge_listed: {}
+    _delete_wd_instance: False
+
+# If not all instances must be deleted, check if instance still in the configuration
+- block:
+  - set_fact:
+      _wd_cartridge_listed: "{{ _wd_cartridge.instances | json_query(_query_instance) | first | default({}) }}"
+    vars:
+      _query_instance: "[?name=='{{ _wd_instance_name }}']" 
+    when: _wd_cartridge.instances is defined
+  - set_fact:
+      _delete_wd_instance: True
+    when: _wd_cartridge_listed == {}
+  when: (_p_delete_all_instances | default(False)) == False
+
+- set_fact:
+    _delete_wd_instance: True
+  when: (_p_delete_all_instances | default(False)) == True
+
+- name: Fail if Watson Discovery instance will be deleted and destroy is not confirmed
+  fail:
+    msg: "Destroy was not confirmed via --confirm-destroy or global_config confirm_destroy property, will not delete Watson Discovery instance {{ _wd_instance_name }}"
+  when:
+  - _delete_wd_instance | bool
+  - _confirm_destroy | default (False) | bool == False
+
+- name: Deleting Watson Discovery instance {{ _wd_instance_name }}
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _wd_instance_id }}'
+    method: DELETE
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+  register: _wd_instance_delete_result
+  when: _delete_wd_instance | bool
+
+- name: Wait until Watson Discovery instance {{ _wd_instance_name }} has been deleted
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _wd_instance_id }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 404
+  register: _wd_instance_exists
+  until: _wd_instance_exists.status == 404
+  retries: 30
+  delay: 30
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
+  when: _delete_wd_instance | bool
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instances.yml
new file mode 100644
index 000000000..8b28fd831
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/delete_wd_instances.yml
@@ -0,0 +1,19 @@
+---
+- name: Get all current Watson Discovery instances
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_wd_instance_addon_type }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _all_wd_instances
+
+# Loop through all deployed Watson Discovery instance definitions to check if instance no longer exists in the config
+- name: Loop through each Watson Discovery instance to check if it must be deleted
+  include_tasks: delete_wd_instance.yml
+  loop: "{{ _all_wd_instances.json.service_instances | default([]) }}"
+  loop_control:
+    loop_var: _wd_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/main.yml
new file mode 100644
index 000000000..ef247076e
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+- set_fact:
+    _wd_cartridge: "{{ _cartridges_to_install | json_query(query_cartridge) | first | default({}) }}"
+  vars:
+    cartridge_name: "watson_discovery"
+    query_cartridge: "[?olm_utils_name=='{{ cartridge_name }}']"
+
+- name: Show Watson Discovery cartridge
+  debug:
+    var: _wd_cartridge
+
+- set_fact:
+    _configured_wd_instances: []
+- set_fact:
+    _configured_wd_instances: "{{ _wd_cartridge.instances | default([] )}}"
+  when:
+  - _wd_cartridge != {}
+  - (_wd_cartridge.state | default('installed')) == 'installed'
+
+- include_tasks: provision_wd_instances.yml
+  when:
+  - (_p_delete_all_instances | default(False)) == False
+  - (_p_wait_instances | default(False)) == False
+
+- include_tasks: delete_wd_instances.yml
+  when:
+  - (_p_wait_instances | default(False)) == False
+
+- include_tasks: wait_wd_instances.yml
+  when:
+  - (_p_delete_all_instances | default(False)) == False
+  - _p_wait_instances | default(False)
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instance.yml
new file mode 100644
index 000000000..31ca8b720
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instance.yml
@@ -0,0 +1,51 @@
+---
+- name: Show Watson Discovery instance
+  debug:
+    var: _wd_instance
+
+- name: Check if the Watson Discovery instance is already running
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_wd_instance_addon_type }}&display_name={{ _wd_instance.name }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _wd_instance_lookup_result
+
+- name: Determine if Watson Discovery instance already exists
+  debug:
+    msg: "Watson Discovery instance {{ _wd_instance.name }} already exists in OpenShift project {{ current_cp4d_cluster.project }}, skipping provisioning"
+  when: _wd_instance_lookup_result.json.total_count != 0
+
+- when: _wd_instance_lookup_result.json.total_count == 0
+  block:
+
+  - name: Obtain Watson Discovery version
+    shell: |
+      oc get wd -n {{ current_cp4d_cluster.project }} wd -o jsonpath='{.spec.version}'
+    register: _wd_version
+
+  - name: Prepare Watson Discovery instance {{ _wd_instance.name }} json file
+    template:
+      src: wd-instance.j2
+      dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _wd_instance.name }}-wd-instance.json"
+
+  - name: Create Watson Discovery instance {{ _wd_instance.name }}
+    shell: |
+      curl -v -k \
+        -H 'Authorization: Bearer {{ _cp4d_bearer_token }}' \
+        -H 'Content-Type: application/json' \
+        -X POST \
+        'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances' \
+        -T {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-{{ _wd_instance.name }}-wd-instance.json
+    register: _create_wd_instance_result
+
+  - set_fact:
+      _create_wd_instance_result_json: "{{ _create_wd_instance_result.stdout | from_json }}"
+
+  - name: Show Watson Discovery instance result
+    debug:
+      var: _create_wd_instance_result_json
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instances.yml
new file mode 100644
index 000000000..91dd67010
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/provision_wd_instances.yml
@@ -0,0 +1,6 @@
+---
+- name: Provision Watson Discovery instance from the cartridge specification
+  include_tasks: provision_wd_instance.yml
+  loop: "{{ _configured_wd_instances }}"
+  loop_control:
+    loop_var: _wd_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instance.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instance.yml
new file mode 100644
index 000000000..162e7bac1
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instance.yml
@@ -0,0 +1,39 @@
+---
+- name: Show Watson Discovery instance
+  debug:
+    var: _wd_instance
+
+- name: Retrieve running Watson Discovery instance
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances?addon_type={{ cp4d_wd_instance_addon_type }}&display_name={{ _wd_instance.name }}'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _wd_instance_lookup_result
+
+- set_fact:
+    _wd_instance_id: "{{ (_wd_instance_lookup_result.json.service_instances | first).id }}"
+
+- name: Waiting for Watson Discovery instance {{ _wd_instance.name }} to complete its provisioning
+  uri:
+    url: 'https://{{ cp4d_url.stdout }}/zen-data/v3/service_instances/{{ _wd_instance_id }}?include_service_status=true'
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ _cp4d_bearer_token }}"
+    return_content: yes
+    validate_certs: no
+    status_code: 200
+  register: _wd_instance_lookup_result
+  until: _wd_instance_lookup_result.json.services_status == "RUNNING"
+  retries: 60
+  delay: 30
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instances.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instances.yml
new file mode 100644
index 000000000..1560fe515
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/tasks/wait_wd_instances.yml
@@ -0,0 +1,6 @@
+---
+- name: Wait for Watson Discovery instance to provision
+  include_tasks: wait_wd_instance.yml
+  loop: "{{ _configured_wd_instances }}"
+  loop_control:
+    loop_var: _wd_instance
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/templates/wd-instance.j2 b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/templates/wd-instance.j2
new file mode 100644
index 000000000..1076e9e75
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/templates/wd-instance.j2
@@ -0,0 +1,14 @@
+{
+    "addon_type": "{{ cp4d_wd_instance_addon_type }}",
+    "display_name": "{{ _wd_instance.name }}",
+    "addon_version": "{{ _wd_version.stdout }}",
+    "namespace": "{{ current_cp4d_cluster.project }}",
+    "create_arguments": {
+        "deployment_id": "{{ current_cp4d_cluster.project }}-wd",
+        "parameters": {
+            "serviceId": "discovery",
+            "url": "https://wd-discovery-gateway.{{ current_cp4d_cluster.project }}.svc.cluster.local:60443/v2/service_instances",
+            "watson": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/vars/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/vars/main.yml
new file mode 100644
index 000000000..0da821e22
--- /dev/null
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-watson_discovery/vars/main.yml
@@ -0,0 +1,2 @@
+---
+cp4d_wd_instance_addon_type: discovery
\ No newline at end of file

From 6cf2f8b0c34f99cf062e3e2d19e5ef0dbe4e898a Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Wed, 31 Jan 2024 06:57:46 +0000
Subject: [PATCH 22/24] #631 Add DMC instance

---
 .../cp4d/cp4d-cluster-configure/tasks/main.yml               | 5 +++++
 .../cp4d/cp4d-instance-dmc/tasks/main.yml                    | 2 ++
 .../cp4d/cp4d-instance-dmc/templates/dmc-instance.j2         | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml
index fb772d39b..7c7b17c26 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-cluster-configure/tasks/main.yml
@@ -123,9 +123,11 @@
         - cpd-edb
         - datastage
         - dv
+        - dmc
         - openpages
         - watsonx_ai
         - watson_assistant
+        - watson_discovery
 
     # Wait for provisioning to complete
     - name: Wait for provisioning of {{ item }} instances to complete
@@ -139,9 +141,12 @@
         - cpd-edb
         - datastage
         - dv
+        - dmc
         - openpages
         - watsonx_ai
         - watson_assistant
+        - watson_discovery
+
 
 # Handle instance configurations
 - when: "all_config.cp4d_instance_configuration is defined"
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml
index e860d09ae..05f30961f 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/tasks/main.yml
@@ -22,9 +22,11 @@
   - (_p_delete_all_instances | default(False)) == False
   - (_p_wait_instances | default(False)) == False
 
+# Delete DMC instances if there is no longer a dependency by other cartridges
 - include_tasks: delete_dmc_instances.yml
   when:
   - (_p_wait_instances | default(False)) == False
+  - _surviving_cartridges | selectattr('CR_Kind','equalto',(_dmc_cartridge.CR_Kind | default(''))) | list | count == 0
 
 - include_tasks: wait_dmc_instances.yml
   when:
diff --git a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2 b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2
index e76146370..5d02a3a0d 100644
--- a/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2
+++ b/automation-roles/60-configure-cloud-pak/cp4d/cp4d-instance-dmc/templates/dmc-instance.j2
@@ -1,6 +1,6 @@
 {
    "addon_type": "{{ cp4d_dmc_addon_type }}",
-   "addon_version": "{{ _dmc_version }}",
+   "addon_version": "{{ _dmc_version.stdout }}",
    "display_name": "{{ _dmc_instance.name }}",
    "description": "{{ _dmc_instance.description | default('IBM Db2 Data Management Console') }}",
    "namespace": "{{ current_cp4d_cluster.project }}",

From 9488f3216ba8d0487ec692d567a4dfdb8eafdd30 Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Wed, 31 Jan 2024 07:01:16 +0000
Subject: [PATCH 23/24] #631 Update sample configuration

---
 .../sample-dynamic/config-samples/cp4d-480.yaml        | 10 ++++++++++
 .../sample-dynamic/config-samples/cp4d-482.yaml        |  9 +++++++++
 2 files changed, 19 insertions(+)

diff --git a/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml b/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml
index b65a1a11f..846f985ac 100644
--- a/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml
+++ b/sample-configurations/sample-dynamic/config-samples/cp4d-480.yaml
@@ -101,6 +101,12 @@ cp4d:
   - name: dmc
     description: Db2 Data Management Console
     state: removed
+    instances:
+    - name: data-management-console
+      description: Data Management Console
+      size: medium
+      storage_size_gb: 50
+
 
   - name: dods
     description: Decision Optimization
@@ -208,6 +214,10 @@ cp4d:
     # noobaa_account_secret: noobaa-admin
     # noobaa_cert_secret: noobaa-s3-serving-cert
     state: removed
+    instances:
+    - name: wd-instance
+      description: "Watson Discovery instance"
+
 
   - name: watson-ks
     description: Watson Knowledge Studio
diff --git a/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml b/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml
index 72b11a7eb..b4905f54d 100644
--- a/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml
+++ b/sample-configurations/sample-dynamic/config-samples/cp4d-482.yaml
@@ -101,6 +101,11 @@ cp4d:
   - name: dmc
     description: Db2 Data Management Console
     state: removed
+    instances:
+    - name: data-management-console
+      description: Data Management Console
+      size: medium
+      storage_size_gb: 50
 
   - name: dods
     description: Decision Optimization
@@ -208,6 +213,10 @@ cp4d:
     # noobaa_account_secret: noobaa-admin
     # noobaa_cert_secret: noobaa-s3-serving-cert
     state: removed
+    instances:
+    - name: wd-instance
+      description: "Watson Discovery instance"
+
 
   - name: watson-ks
     description: Watson Knowledge Studio

From f7b849189f397058f5d89348f3ed881b8a365a6a Mon Sep 17 00:00:00 2001
From: Frank Ketelaars <fketelaars@nl.ibm.com>
Date: Thu, 1 Feb 2024 06:14:05 +0000
Subject: [PATCH 24/24] #537 Add watsonx.a@ibm-open-source-bot

---
 ...run-deployer-on-openshift-using-console.md | 35 +++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/src/50-advanced/run-on-openshift/run-deployer-on-openshift-using-console.md b/docs/src/50-advanced/run-on-openshift/run-deployer-on-openshift-using-console.md
index 5fb6b27e5..ec4dd5fe9 100644
--- a/docs/src/50-advanced/run-on-openshift/run-deployer-on-openshift-using-console.md
+++ b/docs/src/50-advanced/run-on-openshift/run-deployer-on-openshift-using-console.md
@@ -93,6 +93,12 @@ data:
       ocp_version: "4.12"
       cluster_name: cpd-demo
       domain_name: example.com
+      mcg:
+        install: False
+        storage_type: storage-class
+        storage_class: managed-nfs-storage
+      gpu:
+        install: False
       openshift_storage:
       - storage_name: auto-storage
         storage_type: auto
@@ -100,8 +106,8 @@ data:
     cp4d:
     - project: cpd
       openshift_cluster_name: cpd-demo
-      cp4d_version: 4.7.4
-      sequential_install: True
+      cp4d_version: 4.8.1
+      sequential_install: False
       accept_licenses: True
       cartridges:
       cartridges:
@@ -307,6 +313,31 @@ data:
         # noobaa_cert_secret: noobaa-s3-serving-cert
         state: removed
 
+      # Please note that for watsonx.ai foundation models, you neeed to install the
+      # Node Feature Discovery and NVIDIA GPU operators. You can do so by setting the openshift.gpu.install property to True
+      - name: watsonx_ai
+        description: watsonx.ai
+        state: removed
+        models:
+        - model_id: google-flan-t5-xxl
+          state: removed
+        - model_id: google-flan-ul2
+          state: removed
+        - model_id: eleutherai-gpt-neox-20b
+          state: removed
+        - model_id: ibm-granite-13b-chat-v1
+          state: removed
+        - model_id: ibm-granite-13b-instruct-v1
+          state: removed
+        - model_id: meta-llama-llama-2-70b-chat
+          state: removed
+        - model_id: ibm-mpt-7b-instruct2
+          state: removed
+        - model_id: bigscience-mt0-xxl
+          state: removed
+        - model_id: bigcode-starcoder
+          state: removed
+
       - name: watsonx_data
         description: watsonx.data
         state: removed