Merge pull request validatedpatterns#251 from wolfganghuse/custom-s3-…

…endpoint Custom s3 endpoint
claudiol · Feb 29, 2024 · 01fdc1b · 01fdc1b
2 parents 8d71b5d + 94dead3
commit 01fdc1b
Show file tree

Hide file tree

Showing 19 changed files with 162 additions and 51 deletions.
diff --git a/charts/datacenter/manuela-data-lake/templates/central-s3-store/kafka-to-s3-cm.yaml b/charts/datacenter/manuela-data-lake/templates/central-s3-store/kafka-to-s3-cm.yaml
@@ -15,6 +15,5 @@ data:
     s3.message.aggregation.count={{ .Values.global.s3.bucket.message.aggregation.count }}
     s3.custom.endpoint.enabled={{ .Values.global.s3.bucket.custom.endpoint.enabled }}
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.{{ .Values.global.localClusterDomain }}
-
+    s3.custom.endpoint.url={{ .Values.global.s3.bucket.custom.endpoint.url }}
 
diff --git a/charts/datacenter/manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml b/charts/datacenter/manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml
@@ -60,9 +60,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
-                + secret
-                + region;
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:{{ .Values.kafka.broker.topic.vibration }}?brokers={{ .Values.kafka.broker.uri }}")
                     .convertBodyTo(String.class)
@@ -88,9 +92,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:{{ .Values.kafka.broker.topic.temperature }}?brokers={{ .Values.kafka.broker.uri }}")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/charts/datacenter/manuela-tst/templates/development-s3-store/kafka-to-s3-cm.yaml b/charts/datacenter/manuela-tst/templates/development-s3-store/kafka-to-s3-cm.yaml
@@ -16,6 +16,5 @@ data:
     s3.message.aggregation.count={{ .Values.global.s3.bucket.message.aggregation.count }}
     s3.custom.endpoint.enabled={{ .Values.global.s3.bucket.custom.endpoint.enabled }}
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.{{ .Values.global.localClusterDomain }}
-
+    s3.custom.endpoint.url={{ .Values.global.s3.bucket.custom.endpoint.url }}
 
diff --git a/charts/datacenter/manuela-tst/templates/development-s3-store/kafka-to-s3-integration.yaml b/charts/datacenter/manuela-tst/templates/development-s3-store/kafka-to-s3-integration.yaml
@@ -59,9 +59,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
-            	String s3params = key
-                + secret
-                + region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
+                String s3params = key
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:{{ .Values.kafka.broker.topic.vibration }}?brokers={{ .Values.kafka.broker.uri }}")
                     .convertBodyTo(String.class)
@@ -87,9 +91,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
-            	String s3params = key
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
+                String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:{{ .Values.kafka.broker.topic.temperature }}?brokers={{ .Values.kafka.broker.uri }}")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/common-clustergroup-industrial-edge-factory.expected.yaml b/tests/common-clustergroup-industrial-edge-factory.expected.yaml
@@ -195,6 +195,7 @@ data:
           custom:
             endpoint:
               enabled: false
+              url: https://my.custom.endpoint
           message:
             aggregation:
               count: 50

diff --git a/tests/common-clustergroup-industrial-edge-hub.expected.yaml b/tests/common-clustergroup-industrial-edge-hub.expected.yaml
@@ -356,6 +356,7 @@ data:
           custom:
             endpoint:
               enabled: false
+              url: https://my.custom.endpoint
           message:
             aggregation:
               count: 50

diff --git a/tests/common-clustergroup-medical-diagnosis-hub.expected.yaml b/tests/common-clustergroup-medical-diagnosis-hub.expected.yaml
@@ -315,6 +315,7 @@ data:
           custom:
             endpoint:
               enabled: false
+              url: https://my.custom.endpoint
           message:
             aggregation:
               count: 50

diff --git a/tests/common-clustergroup-normal.expected.yaml b/tests/common-clustergroup-normal.expected.yaml
@@ -277,6 +277,7 @@ data:
           custom:
             endpoint:
               enabled: false
+              url: https://my.custom.endpoint
           message:
             aggregation:
               count: 50

diff --git a/tests/datacenter-manuela-data-lake-industrial-edge-factory.expected.yaml b/tests/datacenter-manuela-data-lake-industrial-edge-factory.expected.yaml
@@ -17,7 +17,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.apps.region.example.com
+    s3.custom.endpoint.url=https://my.custom.endpoint
 ---
 # Source: manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml
 apiVersion: camel.apache.org/v1
@@ -82,9 +82,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
-                + secret
-                + region;
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:manuela-factory.iot-sensor-sw-vibration?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
@@ -110,9 +114,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:manuela-factory.iot-sensor-sw-temperature?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/datacenter-manuela-data-lake-industrial-edge-hub.expected.yaml b/tests/datacenter-manuela-data-lake-industrial-edge-hub.expected.yaml
@@ -17,7 +17,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.apps.region.example.com
+    s3.custom.endpoint.url=https://my.custom.endpoint
 ---
 # Source: manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml
 apiVersion: camel.apache.org/v1
@@ -82,9 +82,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
-                + secret
-                + region;
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:manuela-factory.iot-sensor-sw-vibration?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
@@ -110,9 +114,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:manuela-factory.iot-sensor-sw-temperature?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/datacenter-manuela-data-lake-medical-diagnosis-hub.expected.yaml b/tests/datacenter-manuela-data-lake-medical-diagnosis-hub.expected.yaml
@@ -17,7 +17,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.apps.region.example.com
+    s3.custom.endpoint.url=https://my.custom.endpoint
 ---
 # Source: manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml
 apiVersion: camel.apache.org/v1
@@ -82,9 +82,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
-                + secret
-                + region;
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:manuela-factory.iot-sensor-sw-vibration?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
@@ -110,9 +114,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:manuela-factory.iot-sensor-sw-temperature?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/datacenter-manuela-data-lake-naked.expected.yaml b/tests/datacenter-manuela-data-lake-naked.expected.yaml
@@ -17,7 +17,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.
+    s3.custom.endpoint.url=
 ---
 # Source: manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml
 apiVersion: camel.apache.org/v1
@@ -82,9 +82,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
-                + secret
-                + region;
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:manuela-factory.iot-sensor-sw-vibration?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
@@ -110,9 +114,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:manuela-factory.iot-sensor-sw-temperature?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/datacenter-manuela-data-lake-normal.expected.yaml b/tests/datacenter-manuela-data-lake-normal.expected.yaml
@@ -17,7 +17,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.apps.region.example.com
+    s3.custom.endpoint.url=https://my.custom.endpoint
 ---
 # Source: manuela-data-lake/templates/central-s3-store/kafka-to-s3-integration.yaml
 apiVersion: camel.apache.org/v1
@@ -82,9 +82,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
-                + secret
-                + region;
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:manuela-factory.iot-sensor-sw-vibration?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
@@ -110,9 +114,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
                 String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:manuela-factory.iot-sensor-sw-temperature?brokers=prod-kafka-cluster-kafka-bootstrap.manuela-data-lake.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/datacenter-manuela-tst-industrial-edge-factory.expected.yaml b/tests/datacenter-manuela-tst-industrial-edge-factory.expected.yaml
@@ -18,7 +18,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.apps.region.example.com
+    s3.custom.endpoint.url=https://my.custom.endpoint
 ---
 # Source: manuela-tst-all/templates/line-dashboard/line-dashboard-configmap.config.yaml
 apiVersion: v1
@@ -672,9 +672,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
-            	String s3params = key
-                + secret
-                + region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
+                String s3params = key
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:iot-sensor-sw-vibration?brokers=dev-kafka-cluster-kafka-bootstrap.manuela-tst-all.svc:9092")
                     .convertBodyTo(String.class)
@@ -700,9 +704,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
-            	String s3params = key
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
+                String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:iot-sensor-sw-temperature?brokers=dev-kafka-cluster-kafka-bootstrap.manuela-tst-all.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)

diff --git a/tests/datacenter-manuela-tst-industrial-edge-hub.expected.yaml b/tests/datacenter-manuela-tst-industrial-edge-hub.expected.yaml
@@ -18,7 +18,7 @@ data:
     s3.message.aggregation.count=50
     s3.custom.endpoint.enabled=false
     # Convert this directory into a helm chart and use templating to set this
-    s3.custom.endpoint.url=s3-openshift-storage.apps.region.example.com
+    s3.custom.endpoint.url=https://my.custom.endpoint
 ---
 # Source: manuela-tst-all/templates/line-dashboard/line-dashboard-configmap.config.yaml
 apiVersion: v1
@@ -672,9 +672,13 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
-            	String s3params = key
-                + secret
-                + region;
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
+                String s3params = key
+                   + customendpoint
+                   + endpoint
+                   + secret
+                   + region;
 
                 from("kafka:iot-sensor-sw-vibration?brokers=dev-kafka-cluster-kafka-bootstrap.manuela-tst-all.svc:9092")
                     .convertBodyTo(String.class)
@@ -700,9 +704,14 @@ spec:
                 String key = "accessKey=RAW(" + s3_accessKey + ")";
                 String secret = "&secretKey=RAW(" + s3_secretKey + ")";
                 String region = "&region=" + s3_region;
-            	String s3params = key
+                String customendpoint = "&overrideEndpoint=" + s3_custom_endpoint_enabled;
+                String endpoint = "&uriEndpointOverride=" + s3_custom_endpoint_url;
+                String s3params = key
+                   + customendpoint
+                   + endpoint
                    + secret
                    + region;
+                   
                 from("kafka:iot-sensor-sw-temperature?brokers=dev-kafka-cluster-kafka-bootstrap.manuela-tst-all.svc:9092")
                     .convertBodyTo(String.class)
                     .aggregate(simple("true"), new GroupedBodyAggregationStrategy()).completionSize(s3_message_aggregation_count)