CNPG DBConfig

overlays/dev/Makefile

@@ -1,36 +1,28 @@
-SERVER_CHART_VERSION := 32.0.0
-DAEMON_CHART_VERSION := 32.0.2
-UI_CHART_VERSION := 32.0.1
-ELASTIC_VERSION := 0.8.0
+SERVER_CHART_VERSION := 35.0.0
+DAEMON_CHART_VERSION := 35.0.2
+UI_CHART_VERSION := 35.1.0
 
 helm:
 	helm repo add rucio https://rucio.github.io/helm-charts
-	helm repo add elastic https://helm.elastic.co
-	helm repo update rucio elastic
+	helm repo update rucio
 
 rucio-server: helm
-	helm template usdf rucio/rucio-server --version=${SERVER_CHART_VERSION} --values=rucio/values-rucio-server.yaml > rucio/helm-rucio-server.yaml
+	helm template usdf rucio/rucio-server --namespace rucio --version=${SERVER_CHART_VERSION} --values=rucio/values-rucio-server.yaml > rucio/helm-rucio-server.yaml
 
 rucio-daemons: helm
-	helm template usdf rucio/rucio-daemons --version=${DAEMON_CHART_VERSION} --values=rucio/values-rucio-daemons.yaml > rucio/helm-rucio-daemons.yaml
+	helm template usdf rucio/rucio-daemons --namespace rucio --version=${DAEMON_CHART_VERSION} --values=rucio/values-rucio-daemons.yaml > rucio/helm-rucio-daemons.yaml
 
 rucio-ui: helm
-	helm template usdf rucio/rucio-ui --version=${UI_CHART_VERSION} --values=rucio/values-rucio-ui.yaml > rucio/helm-rucio-ui.yaml
+	helm template usdf rucio/rucio-webui --namespace rucio --version=${UI_CHART_VERSION} --values=rucio/values-rucio-ui.yaml > rucio/helm-rucio-ui.yaml
 
 rucio: rucio-server rucio-daemons rucio-ui
 
-elasticsearch:
-	helm template elasticsearch elastic/eck-elasticsearch --version=${ELASTIC_VERSION} --values=elastic/elasticsearch/values-elasticsearch.yaml > elastic/elasticsearch/helm-elasticsearch.yaml
-	vi elastic/elasticsearch/helm-elasticsearch.yaml -c ":%s/enterprise/basic/g" -c ":wq"
-
-elastic: elasticsearch
-
 get-secrets:
 	mkdir -p rucio/etc/.secrets
 	vault kv get --field=usdf-server-hostkey secret/rubin/usdf-rucio-dev/rucio  > rucio/etc/.secrets/hostkey.pem
 	vault kv get --field=usdf-server-hostcert secret/rubin/usdf-rucio-dev/rucio  > rucio/etc/.secrets/hostcert.pem
 	vault kv get --field=usdf-server-cafile secret/rubin/usdf-rucio-dev/rucio  > rucio/etc/.secrets/ca.pem
-	vault kv get --field=usdf-db-conn-str secret/rubin/usdf-rucio-dev/rucio  > rucio/etc/.secrets/db-conn-str.txt
+	vault kv get --field=usdf-db-16b-conn-str secret/rubin/usdf-rucio-dev/rucio  > rucio/etc/.secrets/db-conn-str.txt
 	vault kv get --field=usdf-elasticsearch-password secret/rubin/usdf-rucio-dev/rucio  > rucio/etc/.secrets/usdf-elasticsearch-password
 	cp rucio/etc/automatix.json rucio/etc/.secrets/automatix.json
 	mkdir -p postgres/etc/.secrets
@@ -47,16 +39,14 @@ clean-secrets:
 run-dump: 
 	kubectl kustomize .
 
-dump: get-secrets rucio elastic run-dump clean-secrets 
+dump: get-secrets rucio run-dump clean-secrets 
 
 run-apply:  
 	kubectl apply -k .
 
-apply: get-secrets rucio elastic run-apply clean-secrets 
+apply: get-secrets rucio run-apply clean-secrets 
 
 run-destroy:
 	kubectl delete -k .
 
-destroy: get-secrets rucio elastic run-destroy clean-secrets 
-
-
+destroy: get-secrets rucio run-destroy clean-secrets 

overlays/dev/data-backbone/kafka/kafka-persistent.yaml

@@ -20,6 +20,8 @@ spec:
         port: 9094
         type: loadbalancer
         tls: false
+        #authentication:
+        #  type: scram-sha-512
         configuration:
           bootstrap:
             annotations:
@@ -47,7 +49,7 @@ spec:
       - id: 0
         type: persistent-claim
         class: wekafs--sdf-k8s01
-        size: 1Gi
+        size: 5Gi
         deleteClaim: false
   zookeeper:
     replicas: 3

overlays/dev/data-backbone/kafka/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - ns.yaml
   - kafka-persistent.yaml 
   - topics.yaml
+  - users.yaml

overlays/dev/data-backbone/kafka/topics.yaml

@@ -9,8 +9,8 @@ spec:
   partitions: 3
   replicas: 2
   config:
-    retention.ms: 604800000
-    segment.bytes: 1073741824
+    retention.ms: 604800
+    segment.bytes: 268435456
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaTopic
@@ -23,8 +23,8 @@ spec:
   partitions: 3
   replicas: 2
   config:
-    retention.ms: 604800000
-    segment.bytes: 1073741824
+    retention.ms: 604800
+    segment.bytes: 268435456
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaTopic
@@ -37,8 +37,8 @@ spec:
   partitions: 3
   replicas: 2
   config:
-    retention.ms: 604800000
-    segment.bytes: 1073741824
+    retention.ms: 604800
+    segment.bytes: 268435456
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaTopic
@@ -51,8 +51,8 @@ spec:
   partitions: 3
   replicas: 2
   config:
-    retention.ms: 604800000
-    segment.bytes: 1073741824
+    retention.ms: 604800
+    segment.bytes: 268435456
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaTopic
@@ -65,5 +65,19 @@ spec:
   partitions: 3
   replicas: 2
   config:
-    retention.ms: 604800000
-    segment.bytes: 1073741824
+    retention.ms: 604800
+    segment.bytes: 268435456
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: slac-butler-dev-disk
+  labels:
+    strimzi.io/cluster: rucio-kafka
+spec:
+  topicName: SLAC_BUTLER_DEV_DISK
+  partitions: 3
+  replicas: 2
+  config:
+    retention.ms: 604800
+    segment.bytes: 268435456

overlays/dev/data-backbone/kafka/users.yaml

@@ -0,0 +1,9 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: usdfkafka
+  labels:
+    strimzi.io/cluster: rucio-kafka
+spec:
+  authentication:
+    type: scram-sha-512