build and deployment file changes

etl/ReadMe.md

@@ -37,18 +37,60 @@ jdbc:oracle:thin:@(description=(address=(protocol=tcps)(host=172.18.215.225.nip.
 
 # Create sample data in postgresql using init.sql
 
+# OpenShift Build and Deploy process
+
+## 1. Build custom postgis
+
+```
+     oc process -f debezium-postgis.build.yaml |oc apply -f - 
+```
+
+## 2. Build kafka jdbc connect component
+
+```
+     oc process -f debezium-jdbc.build.yaml --param-file=.env |oc apply -f - 
+```
+
+## 3. Build kafka broker.
+```
+     oc process -f debezium-kafka.build.yaml |oc apply -f - 
+```
+
+## 4. Deploy postgis (switch to correct project for env before this step)
+```
+     oc process -f debezium-postgis.deploy.yaml |oc apply -f - 
+```
+## 5. Deploy zookeeper
+```
+     oc process -f debezium-zookeeper.deploy.yaml |oc apply -f - 
+```
+## 6. Deploy kafka
+```
+     oc process -f debezium-kafka.deploy.yaml |oc apply -f - 
+```
+## 7. Deploy kafka-jdbc connect.
+```
+     oc process -f debezium-jdbc.deploy.yaml |oc apply -f - 
+```
 
 # Register postgresql connector
 ```
 curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json http://localhost:8083/connectors/ 
 
+curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
+
+
+
 ```
 
 # Register Oracle jdbc sink connector
 
 ```
 curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json http://localhost:8083/connectors/
 
+curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
+
+
 ```
 
 # Test with a kafka console consumer.
@@ -67,6 +109,16 @@ curl -X DELETE localhost:8083/connectors/<connector-name>
 
 curl -X DELETE localhost:8083/connectors/oracle-jdbc-sink-connector
 curl -X DELETE localhost:8083/connectors/postgres-source-connector
-```      
+
+curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/oracle-jdbc-sink-connector
+curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/postgres-source-connector
+
+```    
+
+# Get all connectors registered.
+```
+curl localhost:8083/connectors/
+curl https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors
+```
 
 

etl/debezium-kafka/Dockerfile

@@ -1,5 +1,3 @@
-ARG DEBEZIUM_VERSION
-
 
 FROM debezium/kafka:1.9
 

etl/register-oracle-jdbc-sink-connector.json

@@ -6,7 +6,7 @@
         "topics.regex": "dbserver1.public.(.*)",
         "connection.url": "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCPS)(PORT=1543)(HOST=nrcdb01.bcgov))(CONNECT_DATA=(SERVICE_NAME=SD57387.NRS.BCGOV))(SECURITY=(ssl_server_cert_dn=\"CN=nrcdb01.bcgov\")))",
         "connection.user": "JAITHOMA",
-        "connection.password": "jaithoma#1020",
+        "connection.password": "xxxxx",
         "dialect.name": "OracleDatabaseDialect",
         "security.protocol":"SSL",
         "ssl.enabled.protocols": "TLSv1.2,TLSv1.1",
@@ -16,7 +16,9 @@
         "table.name.format" : "SIS.${topic}", 
         "delete.enabled": "true",
         "pk.mode": "record_key",
-        "insert.mode": "upsert", 
+        "insert.mode": "upsert",
+        "max.retries": "30",
+        "retry.backoff.ms": "10000",
         "quote.sql.identifiers": "never",
         "transforms":"route,ConvertCreatedTimestamp,ConvertUpdatedTimestamp,topicCase",
         "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter",

openshift/templates/etl/debezium-kafka.build.yaml

@@ -0,0 +1,59 @@
+---
+kind: Template
+apiVersion: template.openshift.io/v1
+metadata:
+  name: "${API_NAME}-build-template"
+  creationTimestamp: 
+objects:
+- kind: ImageStream
+  apiVersion: v1
+  metadata:
+    name: "${API_NAME}"
+- kind: BuildConfig
+  apiVersion: v1
+  metadata:
+    name: "${API_NAME}-build"
+    labels:
+      app: "${API_NAME}-build"
+  spec:
+    runPolicy: Serial
+    source:
+      type: Git
+      git:
+        uri: "${GIT_REPO_URL}"
+        ref: "${GIT_REF}"
+      contextDir: "${SOURCE_CONTEXT_DIR}"
+    strategy:
+      type: Docker
+      dockerStrategy:
+        dockerfilePath: "${DOCKER_FILE_PATH}"
+    output:
+      to:
+        kind: ImageStreamTag
+        name: "${API_NAME}:latest"
+parameters:
+- name: API_NAME
+  displayName: Name
+  description: The name assigned to all of the resources defined in this template.
+  required: true
+  value: debezium-kafka
+- name: GIT_REPO_URL
+  displayName: Git Repo URL
+  description: The URL to your GIT repo.
+  required: true
+  value: https://github.com/bcgov/nr-epd-digital-services
+- name: GIT_REF
+  displayName: Git Reference
+  description: The git reference or branch.
+  required: true
+  value: fix/task-164
+- name: SOURCE_CONTEXT_DIR
+  displayName: Source Context Directory
+  description: The source context directory.
+  required: false
+  value: etl/debezium-kafka
+- name: DOCKER_FILE_PATH
+  displayName: Docker File Path
+  description: The path to the docker file defining the build.
+  required: false
+  value: "Dockerfile"

openshift/templates/etl/debezium-kafka.deploy.yaml

@@ -26,6 +26,15 @@ objects:
         maxSurge: 25%
     triggers:
     - type: ConfigChange
+    - type: ImageChange
+      imageChangeParams:
+        automatic: true
+        containerNames:
+        - "${API_NAME}"
+        from:
+          kind: ImageStreamTag
+          namespace: "${IMAGE_NAMESPACE}"
+          name: "${API_NAME}:${TAG_NAME}"
     replicas: 1
     test: false
     selector:
@@ -42,7 +51,7 @@ objects:
       spec:
         containers:
         - name: "${API_NAME}" 
-          image: debezium/kafka:1.9
+          image: "${API_NAME}"
           imagePullPolicy: Always
           ports:
           - containerPort: 9092
@@ -108,4 +117,4 @@ parameters:
   displayName: Environment TAG name
   description: The TAG name for this environment, e.g., dev, test, prod
   required: true
-  value: dev
+  value: latest