diff --git a/go.mod b/go.mod index f07021bbc7..ce931f6164 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 github.com/keboola/go-client v1.26.4 - github.com/keboola/go-utils v0.10.2 + github.com/keboola/go-utils v0.10.3-0.20240522072138-726d26d3bf0f github.com/klauspost/compress v1.17.8 github.com/klauspost/pgzip v1.2.6 github.com/kylelemons/godebug v1.1.0 diff --git a/go.sum b/go.sum index 34cd144ecb..3296db1172 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/keboola/go-mockoidc v0.0.0-20240405064136-5229d2b53db6 h1:HcvX1VQkiav github.com/keboola/go-mockoidc v0.0.0-20240405064136-5229d2b53db6/go.mod h1:eDjgYHYDJbPLBLsyZ6qRaugP0mX8vePOhZ5id1fdzJw= github.com/keboola/go-oauth2-proxy/v7 v7.6.1-0.20240418143152-9d00aaa29562 h1:EiwSnkbGt2i6XxvjDMrWx6/bGlQjVs+yq1mDJ5b3U1U= github.com/keboola/go-oauth2-proxy/v7 v7.6.1-0.20240418143152-9d00aaa29562/go.mod h1:uPrZkzwsuFyIPP04hIt6TG2KvWujglvkOnUUnQJyIdw= -github.com/keboola/go-utils v0.10.2 h1:omvBSTEL96zz2lHxfHCZ4UEBZ1J8M40M7GngmMwt070= -github.com/keboola/go-utils v0.10.2/go.mod h1:p+AIGpqlL7c0X+MWNOLdkAt2rMM5JXlycWwkOKmlrps= +github.com/keboola/go-utils v0.10.3-0.20240522072138-726d26d3bf0f h1:hHoHg3m0wqK01ze9lIic25IxDI2Iz/A8XqVPfitPiME= +github.com/keboola/go-utils v0.10.3-0.20240522072138-726d26d3bf0f/go.mod h1:p+AIGpqlL7c0X+MWNOLdkAt2rMM5JXlycWwkOKmlrps= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= diff --git a/internal/pkg/service/common/configmap/bind.go b/internal/pkg/service/common/configmap/bind.go index 3d7f041b2e..45f7312cf7 100644 --- a/internal/pkg/service/common/configmap/bind.go +++ b/internal/pkg/service/common/configmap/bind.go @@ -351,8 +351,10 @@ func collectValues(cfg BindConfig, flagToField FlagToFieldFn) (*orderedmap.Order // Store config.VisitAllRecursive(func(path orderedmap.Path, v any, parent any) { - // Copy leaf values - if _, ok := v.(*orderedmap.OrderedMap); !ok { + // Copy leaf values = value is not object AND key is map step, not slice step + _, isObject := v.(*orderedmap.OrderedMap) + _, isMapKey := path.Last().(orderedmap.MapStep) + if !isObject && isMapKey { if err := values.SetNestedPath(path, fieldValue{Value: v, SetBy: SetByConfig}); err != nil { errs.Append(err) } diff --git a/out.txt b/out.txt new file mode 100644 index 0000000000..c96664e428 --- /dev/null +++ b/out.txt @@ -0,0 +1,161 @@ +# Enable logging at DEBUG level. +debugLog: false +# Log HTTP client requests and responses as debug messages. +debugHTTPClient: false +# Path where CPU profile is saved. +cpuProfilePath: "" +# Unique ID of the node in the cluster. Validation rules: required +nodeID: "" +# Storage API host. Validation rules: required,hostname +storageApiHost: connection.keboola.com +datadog: + # Enable DataDog integration. + enabled: false + # Enable DataDog debug messages. + debug: false +etcd: + # Etcd endpoint. Validation rules: required + endpoint: etcd:2379 + # Etcd namespace. Validation rules: required + namespace: stream/ + # Etcd username. + username: root + # Etcd password. + password: '*****' + # Etcd connect timeout. Validation rules: required + connectTimeout: 30s + # Etcd keep alive timeout. Validation rules: required + keepAliveTimeout: 5s + # Etcd keep alive interval. Validation rules: required + keepAliveInterval: 10s + # Etcd operations logging as debug messages. + debugLog: false +metrics: + # Prometheus scraping metrics listen address. Validation rules: required,hostname_port + listen: 0.0.0.0:9000 +api: + # Listen address of the configuration HTTP API. Validation rules: required,hostname_port + listen: 0.0.0.0:8000 + # Public URL of the configuration HTTP API for link generation. Validation rules: required + publicUrl: http://localhost:8000 +distribution: + # The maximum time to wait for creating a new session. Validation rules: required,minDuration=1s,maxDuration=1m + grantTimeout: 5s + # Timeout for the node registration to the cluster. Validation rules: required,minDuration=1s,maxDuration=5m + startupTimeout: 1m0s + # Timeout for the node un-registration from the cluster. Validation rules: required,minDuration=1s,maxDuration=5m + shutdownTimeout: 10s + # Interval of processing changes in the topology. Use 0 to disable the grouping. Validation rules: maxDuration=30s + eventsGroupInterval: 5s + # Seconds after which the node is automatically un-registered if an outage occurs. Validation rules: required,min=1,max=30 + ttlSeconds: 15 +source: + http: + # Listen address of the HTTP source. Validation rules: required,hostname_port + listen: 0.0.0.0:7000 + # Public URL of the HTTP source for link generation. + publicUrl: null +storage: + # Mounted volumes path, each volume is in "{type}/{label}" subdir. Validation rules: required + volumesPath: "" + statistics: + sync: + # Statistics synchronization interval, from memory to the etcd. Validation rules: required,minDuration=100ms,maxDuration=5s + interval: 1s + # Statistics synchronization timeout. Validation rules: required,minDuration=1s,maxDuration=1m + timeout: 30s + cache: + L2: + # Enable statistics L2 in-memory cache, otherwise only L1 cache is used. + enabled: true + # Statistics L2 in-memory cache invalidation interval. Validation rules: required,minDuration=100ms,maxDuration=5s + interval: 1s + cleanup: + # Enable storage cleanup. + enabled: true + # Cleanup interval. Validation rules: required,minDuration=5m,maxDuration=24h + interval: 30m0s + # How many files are deleted in parallel. Validation rules: required,min=1,max=500 + concurrency: 100 + # Expiration interval of a file that has not yet been imported. Validation rules: required,minDuration=1h,maxDuration=720h,gtefield=ArchivedFileExpiration + activeFileExpiration: 168h0m0s + # Expiration interval of a file that has already been imported. Validation rules: required,minDuration=15m,maxDuration=720h + archivedFileExpiration: 24h0m0s + level: + local: + volume: + assignment: + # Volumes count simultaneously utilized per sink. Validation rules: required,min=1,max=100 + count: 1 + # List of preferred volume types, start with the most preferred. Validation rules: required,min=1 + preferredTypes: + - default + registration: + # Number of seconds after the volume registration expires if the node is not available. Validation rules: required,min=1,max=60 + ttlSeconds: 10 + sync: + # Sync mode: "disabled", "cache" or "disk". Validation rules: required,oneof=disabled disk cache + mode: disk + # Wait for sync to disk OS cache or to disk hardware, depending on the mode. + wait: true + # Minimal interval between syncs to disk. Validation rules: min=0,maxDuration=2s,required_if=Mode disk,required_if=Mode cache + checkInterval: 5ms + # Written records count to trigger sync. Validation rules: min=0,max=1000000,required_if=Mode disk,required_if=Mode cache + countTrigger: 500 + # Written size to trigger sync. Validation rules: maxBytes=100MB,required_if=Mode disk,required_if=Mode cache + bytesTrigger: 1MB + # Interval from the last sync to trigger sync. Validation rules: min=0,maxDuration=2s,required_if=Mode disk,required_if=Mode cache + intervalTrigger: 50ms + allocation: + # Allocate disk space for each slice. + enabled: true + # Size of allocated disk space for a slice. Validation rules: required + static: 100MB + # Allocate disk space as % from the previous slice size. Validation rules: min=100,max=500 + relative: 110 + compression: + # Compression type. Validation rules: required,oneof=none gzip zstd + type: gzip + gzip: + # GZIP compression level: 1-9. Validation rules: min=1,max=9 + level: 1 + # GZIP implementation: standard, fast, parallel. Validation rules: required,oneof=standard fast parallel + implementation: parallel + # GZIP parallel block size. Validation rules: required,minBytes=16kB,maxBytes=100MB + blockSize: 256KB + # GZIP parallel concurrency, 0 = auto. + concurrency: 0 + zstd: + # ZSTD compression level: fastest, default, better, best. Validation rules: min=1,max=4 + level: 1 + # ZSTD window size. Validation rules: required,minBytes=1kB,maxBytes=512MB + windowSize: 1MB + # ZSTD concurrency, 0 = auto + concurrency: 0 + staging: + # Maximum number of slices in a file, a new file is created after reaching it. Validation rules: required,min=1,max=50000 + maxSlicesPerFile: 100 + # Maximum number of the Storage API file resources created in parallel within one operation. Validation rules: required,min=1,max=500 + parallelFileCreateLimit: 50 + upload: + # Minimal interval between uploads. Validation rules: required,minDuration=1s,maxDuration=5m + minInterval: 5s + trigger: + # Records count to trigger slice upload. Validation rules: required,min=1,max=10000000 + count: 10000 + # Records size to trigger slice upload. Validation rules: required,minBytes=100B,maxBytes=50MB + size: 1MB + # Duration from the last slice upload to trigger the next upload. Validation rules: required,minDuration=1s,maxDuration=30m + interval: 1m0s + target: + import: + # Minimal interval between imports. Validation rules: required,minDuration=30s,maxDuration=30m + minInterval: 1m0s + trigger: + # Records count to trigger file import. Validation rules: required,min=1,max=10000000 + count: 50000 + # Records size to trigger file import. Validation rules: required,minBytes=100B,maxBytes=500MB + size: 5MB + # Duration from the last import to trigger the next import. Validation rules: required,minDuration=60s,maxDuration=24h + interval: 5m0s + diff --git a/provisioning/apps-proxy/deploy_local.sh b/provisioning/apps-proxy/deploy_local.sh index 7f62ab0735..92ff20032e 100755 --- a/provisioning/apps-proxy/deploy_local.sh +++ b/provisioning/apps-proxy/deploy_local.sh @@ -14,7 +14,7 @@ export RELEASE_RELEASENAME="${RELEASE_RELEASENAME:=my-release}" export KEBOOLA_STACK="${KEBOOLA_STACK:=local-machine}" export HOSTNAME_SUFFIX="${HOSTNAME_SUFFIX:=keboola.com}" export APPS_PROXY_REPOSITORY="${APPS_PROXY_REPOSITORY:=docker.io/keboola/apps-proxy}" # docker.io prefix is required -export APPS_PROXY_IMAGE_TAG="${APPS_PROXY_IMAGE_TAG:=$(git rev-parse --short HEAD)}" +export APPS_PROXY_IMAGE_TAG="${APPS_PROXY_IMAGE_TAG:=$(git rev-parse --short HEAD)-$(date +%s)}" export APPS_PROXY_REPLICAS="${APPS_PROXY_REPLICAS:=3}" export APPS_PROXY_SANDBOXES_API_TOKEN="${APPS_PROXY_SANDBOXES_API_TOKEN:=token}" diff --git a/provisioning/common/etcd/values.yaml b/provisioning/common/etcd/values.yaml index f02b40b8b5..6d954bbe83 100644 --- a/provisioning/common/etcd/values.yaml +++ b/provisioning/common/etcd/values.yaml @@ -12,7 +12,7 @@ nodeSelector: # Locked/updated etcd image image: - tag: 3.5.11-debian-11-r3 + tag: 3.5.13-debian-12-r10 # Additional pod annotations podAnnotations: @@ -56,6 +56,9 @@ extraEnvVars: # Default value in etcd v3.2+ is "100 000": https://etcd.io/docs/v3.5/op-guide/maintenance/#raft-log-retention - name: ETCD_SNAPSHOT_COUNT value: "10000" + # GOMEMLIMIT to prevent OOM kill, update the value together with "resources" spec. + - name: GOMEMLIMIT + value: "256MiB" # The root password is used for authentication, the client gets a JWT token with short expiration. auth: @@ -105,7 +108,7 @@ startupProbe: periodSeconds: 5 timeoutSeconds: 5 successThreshold: 1 - failureThreshold: 60 + failureThreshold: 300 livenessProbe: initialDelaySeconds: 0 # replaced by startupProbe readinessProbe: diff --git a/provisioning/stream/README.md b/provisioning/stream/README.md index 886e3d4272..423dbdce30 100644 --- a/provisioning/stream/README.md +++ b/provisioning/stream/README.md @@ -1,4 +1,4 @@ -# Provisioning of the Buffer Service +# Provisioning of the Stream Service ## Directory Structure @@ -43,10 +43,10 @@ If you need to debug or test something in a Kubernetes cluster, you can use loca At the end of the script, the URL of the service is printed. ```sh To interact with the MiniKube profile run: -export MINIKUBE_PROFILE=buffer +export MINIKUBE_PROFILE=stream To clear the MiniKube: -MINIKUBE_PROFILE=buffer minikube delete --purge +MINIKUBE_PROFILE=stream minikube delete --purge Load balancer of the service is accessible at: http://172.17.0.2:32183 @@ -55,7 +55,7 @@ http://172.17.0.2:32183 ### etcd Etcd deployment includes a network policy, -only pods with `buffer-etcd-client=true` can connect to the etcd. +only pods with `stream-etcd-client=true` can connect to the etcd. #### Client @@ -63,14 +63,14 @@ If you need to start the etcd client, you can use this following commands. Run interactive container: ``` -export ETCD_ROOT_PASSWORD=$(kubectl get secret --namespace "buffer" buffer-etcd -o jsonpath="{.data.etcd-root-password}" 2>/dev/null | base64 -d) +export ETCD_ROOT_PASSWORD=$(kubectl get secret --namespace "stream" stream-etcd -o jsonpath="{.data.etcd-root-password}" 2>/dev/null | base64 -d) -kubectl run --tty --stdin --rm --restart=Never buffer-etcd-client \ - --namespace buffer \ +kubectl run --tty --stdin --rm --restart=Never stream-etcd-client \ + --namespace stream \ --image docker.io/bitnami/etcd:3.5.5-debian-11-r16 \ - --labels="buffer-etcd-client=true" \ + --labels="stream-etcd-client=true" \ --env="ETCD_ROOT_PASSWORD=$ETCD_ROOT_PASSWORD" \ - --env="ETCDCTL_ENDPOINTS=buffer-etcd:2379" \ + --env="ETCDCTL_ENDPOINTS=stream-etcd:2379" \ --command -- bash ``` diff --git a/provisioning/stream/azure.sh b/provisioning/stream/azure.sh index f286ac42a4..f68ed8a5eb 100644 --- a/provisioning/stream/azure.sh +++ b/provisioning/stream/azure.sh @@ -37,11 +37,11 @@ TIME_WAITED=0 # every 10 seconds but in total max 15 minutes try to fetch IP_ADDRESS #shellcheck disable=2203 while [[ -z "$IP_ADDRESS" && $TIME_WAITED -lt 15*60 ]]; do - echo "Waiting for Buffer API ingress IP..." + echo "Waiting for Stream API ingress IP..." sleep 10; TIME_WAITED=$((TIME_WAITED + 10)) IP_ADDRESS=$(kubectl get services \ - --selector "app=buffer-api" \ + --selector "app=stream-api" \ --namespace "$NAMESPACE" \ --no-headers \ --output jsonpath="{.items[0].status.loadBalancer.ingress[0].ip}") @@ -56,12 +56,12 @@ APPLICATION_GATEWAY_NAME=$(az deployment group show \ az network application-gateway address-pool update \ --gateway-name="$APPLICATION_GATEWAY_NAME" \ --resource-group "$RESOURCE_GROUP" \ - --name=buffer \ + --name=stream \ --servers "$IP_ADDRESS" az network application-gateway probe update \ --gateway-name="$APPLICATION_GATEWAY_NAME" \ --resource-group "$RESOURCE_GROUP" \ - --name=buffer-health-probe \ + --name=stream-health-probe \ --host "$IP_ADDRESS" diff --git a/provisioning/stream/common.sh b/provisioning/stream/common.sh index 642035d304..5dffb2e9af 100644 --- a/provisioning/stream/common.sh +++ b/provisioning/stream/common.sh @@ -1,33 +1,30 @@ #!/usr/bin/env bash -# Prevent direct run of the script -if [ "${BASH_SOURCE[0]}" -ef "$0" ] -then - echo 'This script should not be executed directly, please run "deploy.sh" instead.' - exit 1 -fi +## Prevent direct run of the script +#if [ "${BASH_SOURCE[0]}" -ef "$0" ] +#then +# echo 'This script should not be executed directly, please run "deploy.sh" instead.' +# exit 1 +#fi # Required ENVs : ${RELEASE_ID?"Missing RELEASE_ID"} : ${KEBOOLA_STACK?"Missing KEBOOLA_STACK"} : ${HOSTNAME_SUFFIX?"Missing HOSTNAME_SUFFIX"} -: ${BUFFER_API_REPOSITORY?"Missing BUFFER_API_REPOSITORY"} -: ${BUFFER_API_IMAGE_TAG?"Missing BUFFER_API_IMAGE_TAG"} -: ${BUFFER_WORKER_REPOSITORY?"Missing BUFFER_WORKER_REPOSITORY"} -: ${BUFFER_WORKER_IMAGE_TAG?"Missing BUFFER_WORKER_IMAGE_TAG"} -: ${BUFFER_API_REPLICAS?"Missing BUFFER_API_REPLICAS"} -: ${BUFFER_WORKER_REPLICAS?"Missing BUFFER_WORKER_REPLICAS"} -: ${BUFFER_ETCD_REPLICAS?"Missing BUFFER_ETCD_REPLICAS"} +: ${STREAM_IMAGE_REPOSITORY?"Missing STREAM_IMAGE_REPOSITORY"} +: ${STREAM_IMAGE_TAG?"Missing STREAM_IMAGE_TAG"} +: ${STREAM_API_REPLICAS?"Missing STREAM_API_REPLICAS"} +: ${STREAM_STORAGE_REPLICAS?"Missing STREAM_SOURCE_REPLICAS"} +: ${STREAM_COORDINATOR_REPLICAS?"Missing STREAM_COORDINATOR_REPLICAS"} +: ${STREAM_ETCD_REPLICAS?"Missing STREAM_ETCD_REPLICAS"} : ${ETCD_STORAGE_CLASS_NAME?"Missing ETCD_STORAGE_CLASS_NAME"} # Default values -export BUFFER_WORKER_MEMORY="${BUFFER_WORKER_MEMORY:="512Mi"}" -export BUFFER_WORKER_MEMORY_GOMEMLIMIT="${BUFFER_WORKER_MEMORY_GOMEMLIMIT:="400MiB"}" -export BUFFER_ETCD_MEMORY="${BUFFER_ETCD_MEMORY:="2000Mi"}" +export STREAM_ETCD_MEMORY="${STREAM_ETCD_MEMORY:="2000Mi"}" # Constants -export NAMESPACE="buffer" -ETCD_HELM_CHART_VERSION="8.5.8" +export NAMESPACE="stream" +ETCD_HELM_CHART_VERSION="10.1.0" # Common part of the deployment. Same for AWS/Azure/Local ./kubernetes/build.sh @@ -36,30 +33,32 @@ ETCD_HELM_CHART_VERSION="8.5.8" kubectl apply -f ./kubernetes/deploy/namespace.yaml # Get etcd root password, if it is already present -export ETCD_ROOT_PASSWORD=$(kubectl get secret --namespace "$NAMESPACE" buffer-etcd -o jsonpath="{.data.etcd-root-password}" 2>/dev/null | base64 -d) +export ETCD_ROOT_PASSWORD=$(kubectl get secret --namespace "$NAMESPACE" stream-etcd -o jsonpath="{.data.etcd-root-password}" 2>/dev/null | base64 -d) # Deploy etcd cluster helm repo add --force-update bitnami https://charts.bitnami.com/bitnami helm upgrade \ - --install buffer-etcd bitnami/etcd \ + --install stream-etcd bitnami/etcd \ --version "$ETCD_HELM_CHART_VERSION" \ --values ./kubernetes/deploy/etcd/values_common.yaml \ - --values ./kubernetes/deploy/etcd/values_buffer.yaml \ + --values ./kubernetes/deploy/etcd/values_stream.yaml \ --namespace "$NAMESPACE" \ - --set "replicaCount=$BUFFER_ETCD_REPLICAS" \ + --set "replicaCount=$STREAM_ETCD_REPLICAS" \ --set "auth.rbac.rootPassword=$ETCD_ROOT_PASSWORD" \ --set "persistence.storageClass=$ETCD_STORAGE_CLASS_NAME" \ - --set "resources.requests.memory=$BUFFER_ETCD_MEMORY" \ - --set "resources.limits.memory=$BUFFER_ETCD_MEMORY" + --set "resources.requests.memory=$STREAM_ETCD_MEMORY" \ + --set "resources.limits.memory=$STREAM_ETCD_MEMORY" \ + --set "extraEnvVars[3].name=GOMEMLIMIT" \ + --set "extraEnvVars[3].value=${STREAM_ETCD_MEMORY}B" + +# Config +kubectl apply -f ./kubernetes/deploy/config/config-map.yaml # API -kubectl apply -f ./kubernetes/deploy/api/config-map.yaml kubectl apply -f ./kubernetes/deploy/api/pdb.yaml kubectl apply -f ./kubernetes/deploy/api/network-policy.yaml kubectl apply -f ./kubernetes/deploy/api/deployment.yaml -# Worker -kubectl apply -f ./kubernetes/deploy/worker/config-map.yaml -kubectl apply -f ./kubernetes/deploy/worker/pdb.yaml -kubectl apply -f ./kubernetes/deploy/worker/network-policy.yaml -kubectl apply -f ./kubernetes/deploy/worker/deployment.yaml +# Storage +kubectl apply -f ./kubernetes/deploy/storage/network-policy.yaml +kubectl apply -f ./kubernetes/deploy/storage/statefulset.yaml diff --git a/provisioning/stream/deploy_local.sh b/provisioning/stream/deploy_local.sh index e9c7bb8f17..caef5ae821 100755 --- a/provisioning/stream/deploy_local.sh +++ b/provisioning/stream/deploy_local.sh @@ -8,18 +8,17 @@ cd "$(dirname "$0")" export CLOUD_PROVIDER=local # Default values for the local deployment -export MINIKUBE_PROFILE="${MINIKUBE_PROFILE:=buffer}" +export MINIKUBE_PROFILE="${MINIKUBE_PROFILE:=stream}" export BUILD_BUILDID="${BUILD_BUILDID:=dev}" export RELEASE_RELEASENAME="${RELEASE_RELEASENAME:=my-release}" export KEBOOLA_STACK="${KEBOOLA_STACK:=local-machine}" export HOSTNAME_SUFFIX="${HOSTNAME_SUFFIX:=keboola.com}" -export BUFFER_API_REPOSITORY="${BUFFER_API_REPOSITORY:=docker.io/keboola/buffer-api}" -export BUFFER_API_IMAGE_TAG="${BUFFER_API_IMAGE_TAG:=$(git rev-parse --short HEAD)}" -export BUFFER_WORKER_REPOSITORY="${BUFFER_WORKER_REPOSITORY:=docker.io/keboola/buffer-worker}" -export BUFFER_WORKER_IMAGE_TAG="${BUFFER_WORKER_IMAGE_TAG:=$(git rev-parse --short HEAD)}" -export BUFFER_API_REPLICAS="${BUFFER_API_REPLICAS:=2}" -export BUFFER_WORKER_REPLICAS="${BUFFER_WORKER_REPLICAS:=2}" -export BUFFER_ETCD_REPLICAS="${BUFFER_ETCD_REPLICAS:=3}" -export BUFFER_ETCD_MEMORY="${BUFFER_ETCD_MEMORY:="512Mi"}" +export STREAM_IMAGE_REPOSITORY="${STREAM_IMAGE_REPOSITORY:=docker.io/keboola/stream-api}" +export STREAM_IMAGE_TAG="${STREAM_IMAGE_TAG:=$(git rev-parse --short HEAD)-$(date +%s)}" +export STREAM_API_REPLICAS="${STREAM_API_REPLICAS:=2}" +export STREAM_STORAGE_REPLICAS="${STREAM_STORAGE_REPLICAS:=2}" +export STREAM_COORDINATOR_REPLICAS="${STREAM_COORDINATOR_REPLICAS:=2}" +export STREAM_ETCD_REPLICAS="${STREAM_ETCD_REPLICAS:=3}" +export STREAM_ETCD_MEMORY="${STREAM_ETCD_MEMORY:="512Mi"}" ./deploy.sh diff --git a/provisioning/stream/gcp.sh b/provisioning/stream/gcp.sh index 156be7d1d0..9aa37ec1ad 100755 --- a/provisioning/stream/gcp.sh +++ b/provisioning/stream/gcp.sh @@ -21,7 +21,7 @@ echo "" terraform -chdir=./gcp init -input=false -no-color \ -backend-config="bucket=${TERRAFORM_REMOTE_STATE_BUCKET}" \ - -backend-config="prefix=keboola-as-code/buffer" + -backend-config="prefix=keboola-as-code/stream" echo "=> Validating configuration" terraform validate -no-color diff --git a/provisioning/stream/kubernetes/build.sh b/provisioning/stream/kubernetes/build.sh index 18c0f11da6..c3f012ef32 100755 --- a/provisioning/stream/kubernetes/build.sh +++ b/provisioning/stream/kubernetes/build.sh @@ -9,19 +9,19 @@ envsubst < templates/namespace.yaml > deploy/namespace.yaml # Etcd cp ../../common/etcd/values.yaml deploy/etcd/values_common.yaml -cp templates/etcd.yaml deploy/etcd/values_buffer.yaml +cp templates/etcd.yaml deploy/etcd/values_stream.yaml + +# Config +envsubst < templates/config/config-map.yaml > deploy/config/config-map.yaml # API -envsubst < templates/api/config-map.yaml > deploy/api/config-map.yaml -envsubst < templates/api/pdb.yaml > deploy/api/pdb.yaml -envsubst < templates/api/network-policy.yaml > deploy/api/network-policy.yaml -envsubst < templates/api/deployment.yaml > deploy/api/deployment.yaml - -# Worker -envsubst < templates/worker/config-map.yaml > deploy/worker/config-map.yaml -envsubst < templates/worker/pdb.yaml > deploy/worker/pdb.yaml -envsubst < templates/worker/network-policy.yaml > deploy/worker/network-policy.yaml -envsubst < templates/worker/deployment.yaml > deploy/worker/deployment.yaml +envsubst < templates/api/pdb.yaml > deploy/api/pdb.yaml +envsubst < templates/api/network-policy.yaml > deploy/api/network-policy.yaml +envsubst < templates/api/deployment.yaml > deploy/api/deployment.yaml + +# Storage +envsubst < templates/storage/network-policy.yaml > deploy/storage/network-policy.yaml +envsubst < templates/storage/statefulset.yaml > deploy/storage/statefulset.yaml # Remove resources requests/limits to fit all pods to the CI environment REMOVE_RESOURCES_LIMITS="${REMOVE_RESOURCES_LIMITS:=false}" diff --git a/provisioning/stream/kubernetes/deploy/worker/.gitignore b/provisioning/stream/kubernetes/deploy/config/.gitignore similarity index 100% rename from provisioning/stream/kubernetes/deploy/worker/.gitignore rename to provisioning/stream/kubernetes/deploy/config/.gitignore diff --git a/provisioning/stream/kubernetes/deploy/namespace.yaml b/provisioning/stream/kubernetes/deploy/namespace.yaml index a6bf286ed6..72172c3676 100644 --- a/provisioning/stream/kubernetes/deploy/namespace.yaml +++ b/provisioning/stream/kubernetes/deploy/namespace.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: buffer + name: stream diff --git a/provisioning/stream/kubernetes/deploy/namespace.yaml.original b/provisioning/stream/kubernetes/deploy/namespace.yaml.original deleted file mode 100644 index a6bf286ed6..0000000000 --- a/provisioning/stream/kubernetes/deploy/namespace.yaml.original +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: buffer diff --git a/provisioning/stream/kubernetes/deploy/storage/.gitignore b/provisioning/stream/kubernetes/deploy/storage/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/provisioning/stream/kubernetes/deploy/storage/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/provisioning/stream/kubernetes/templates/api/config-map.yaml b/provisioning/stream/kubernetes/templates/api/config-map.yaml deleted file mode 100644 index 509f68e767..0000000000 --- a/provisioning/stream/kubernetes/templates/api/config-map.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: buffer-api - namespace: $NAMESPACE - labels: - app: buffer-api -data: - storageApiHost: "https://connection.$HOSTNAME_SUFFIX" - bufferApiHost: "https://buffer.$HOSTNAME_SUFFIX" - keboolaStack: "$KEBOOLA_STACK" - keboolaRevision: "$RELEASE_ID" diff --git a/provisioning/stream/kubernetes/templates/api/deployment.yaml b/provisioning/stream/kubernetes/templates/api/deployment.yaml index 4c7261f8ec..62863645a7 100644 --- a/provisioning/stream/kubernetes/templates/api/deployment.yaml +++ b/provisioning/stream/kubernetes/templates/api/deployment.yaml @@ -2,29 +2,29 @@ kind: Deployment apiVersion: apps/v1 metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api releaseId: $RELEASE_ID spec: - replicas: $BUFFER_API_REPLICAS + replicas: $STREAM_API_REPLICAS selector: matchLabels: - app: buffer-api + app: stream-api template: metadata: labels: - app: buffer-api + app: stream-api releaseId: $RELEASE_ID - buffer-etcd-client: "true" + stream-etcd-client: "true" tags.datadoghq.com/env: "$KEBOOLA_STACK" - tags.datadoghq.com/service: "buffer-api" + tags.datadoghq.com/service: "stream" tags.datadoghq.com/version: "$RELEASE_ID" annotations: log: "true" ad.datadoghq.com/tags: '{"git.commit.sha": "$COMMIT_HASH", "git.repository_url": "github.com/keboola/keboola-as-code"}' - ad.datadoghq.com/buffer-api.checks: | + ad.datadoghq.com/stream-api.checks: | { "openmetrics": { "instances": [ @@ -39,11 +39,14 @@ spec: } } spec: - terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 60 containers: - - name: buffer-api - image: $BUFFER_API_REPOSITORY:$BUFFER_API_IMAGE_TAG + - name: stream-api + image: $STREAM_IMAGE_REPOSITORY:$STREAM_IMAGE_TAG + command: [ "/app/service" ] + args: [ "--config-file", "/stream/config/config.yaml", "api" ] # run API component resources: + # Note: Update also soft GOMEMLIMIT bellow requests: cpu: "200m" memory: "512Mi" @@ -52,31 +55,23 @@ spec: ports: - containerPort: 8000 # API - containerPort: 9000 # metrics + volumeMounts: + - name: config + mountPath: /stream/config + readOnly: true env: # 80% of the resources.limits.memory - name: GOMEMLIMIT value: "400MiB" - - name: BUFFER_API_STORAGE_API_HOST + - name: STREAM_NODE_ID valueFrom: - configMapKeyRef: - name: buffer-api - key: storageApiHost - - name: BUFFER_API_PUBLIC_ADDRESS - valueFrom: - configMapKeyRef: - name: buffer-api - key: bufferApiHost - - name: BUFFER_API_ETCD_ENDPOINT - value: buffer-etcd.buffer.svc.cluster.local:2379 - - name: BUFFER_API_ETCD_USERNAME - value: root - - name: BUFFER_API_ETCD_PASSWORD + fieldRef: + fieldPath: metadata.name + - name: STREAM_ETCD_PASSWORD valueFrom: secretKeyRef: - name: buffer-etcd + name: stream-etcd key: etcd-root-password - - name: BUFFER_API_ETCD_NAMESPACE - value: buffer - name: DD_AGENT_HOST valueFrom: fieldRef: @@ -121,5 +116,12 @@ spec: value: KubernetesLivenessProbe periodSeconds: 10 failureThreshold: 3 + volumes: + - name: config + configMap: + name: stream + items: + - key: config.yaml + path: config.yaml nodeSelector: nodepool: main diff --git a/provisioning/stream/kubernetes/templates/api/network-policy.yaml b/provisioning/stream/kubernetes/templates/api/network-policy.yaml index 1c486d63b9..87c81a218d 100644 --- a/provisioning/stream/kubernetes/templates/api/network-policy.yaml +++ b/provisioning/stream/kubernetes/templates/api/network-policy.yaml @@ -2,15 +2,15 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api releaseId: $RELEASE_ID spec: podSelector: matchLabels: - app: buffer-api + app: stream-api policyTypes: - Ingress ingress: diff --git a/provisioning/stream/kubernetes/templates/api/pdb.yaml b/provisioning/stream/kubernetes/templates/api/pdb.yaml index efd52df1e2..b8fe5a3f70 100644 --- a/provisioning/stream/kubernetes/templates/api/pdb.yaml +++ b/provisioning/stream/kubernetes/templates/api/pdb.yaml @@ -2,12 +2,12 @@ apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: buffer-api-pdb + name: stream-api-pdb namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api spec: maxUnavailable: 1 selector: matchLabels: - app: buffer-api + app: stream-api diff --git a/provisioning/stream/kubernetes/templates/cloud/aws/ingress.yaml b/provisioning/stream/kubernetes/templates/cloud/aws/ingress.yaml index d91063578d..c2e29f7247 100644 --- a/provisioning/stream/kubernetes/templates/cloud/aws/ingress.yaml +++ b/provisioning/stream/kubernetes/templates/cloud/aws/ingress.yaml @@ -2,20 +2,20 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api annotations: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: "buffer.${HOSTNAME_SUFFIX}" + - host: "stream.${HOSTNAME_SUFFIX}" http: paths: - backend: service: - name: buffer-api + name: stream-api port: number: 80 path: / diff --git a/provisioning/stream/kubernetes/templates/cloud/aws/service.yaml b/provisioning/stream/kubernetes/templates/cloud/aws/service.yaml index 46f50a5d06..d698c2ffd6 100644 --- a/provisioning/stream/kubernetes/templates/cloud/aws/service.yaml +++ b/provisioning/stream/kubernetes/templates/cloud/aws/service.yaml @@ -2,14 +2,14 @@ kind: Service apiVersion: v1 metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api spec: type: ClusterIP selector: - app: buffer-api + app: stream-api ports: - port: 80 targetPort: 8000 diff --git a/provisioning/stream/kubernetes/templates/cloud/azure/service.yaml b/provisioning/stream/kubernetes/templates/cloud/azure/service.yaml index e7662214d5..86742741d3 100644 --- a/provisioning/stream/kubernetes/templates/cloud/azure/service.yaml +++ b/provisioning/stream/kubernetes/templates/cloud/azure/service.yaml @@ -2,16 +2,16 @@ kind: Service apiVersion: v1 metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" spec: type: LoadBalancer selector: - app: buffer-api + app: stream-api ports: - port: 80 targetPort: 8000 diff --git a/provisioning/stream/kubernetes/templates/cloud/gcp/ingress.yaml b/provisioning/stream/kubernetes/templates/cloud/gcp/ingress.yaml index d91063578d..c2e29f7247 100644 --- a/provisioning/stream/kubernetes/templates/cloud/gcp/ingress.yaml +++ b/provisioning/stream/kubernetes/templates/cloud/gcp/ingress.yaml @@ -2,20 +2,20 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api annotations: kubernetes.io/ingress.class: "nginx" spec: rules: - - host: "buffer.${HOSTNAME_SUFFIX}" + - host: "stream.${HOSTNAME_SUFFIX}" http: paths: - backend: service: - name: buffer-api + name: stream-api port: number: 80 path: / diff --git a/provisioning/stream/kubernetes/templates/cloud/gcp/service.yaml b/provisioning/stream/kubernetes/templates/cloud/gcp/service.yaml index 46f50a5d06..d698c2ffd6 100644 --- a/provisioning/stream/kubernetes/templates/cloud/gcp/service.yaml +++ b/provisioning/stream/kubernetes/templates/cloud/gcp/service.yaml @@ -2,14 +2,14 @@ kind: Service apiVersion: v1 metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api spec: type: ClusterIP selector: - app: buffer-api + app: stream-api ports: - port: 80 targetPort: 8000 diff --git a/provisioning/stream/kubernetes/templates/cloud/local/service.yaml b/provisioning/stream/kubernetes/templates/cloud/local/service.yaml index fdf4364160..080c638ac7 100644 --- a/provisioning/stream/kubernetes/templates/cloud/local/service.yaml +++ b/provisioning/stream/kubernetes/templates/cloud/local/service.yaml @@ -2,14 +2,14 @@ kind: Service apiVersion: v1 metadata: - name: buffer-api + name: stream-api namespace: $NAMESPACE labels: - app: buffer-api + app: stream-api spec: type: LoadBalancer selector: - app: buffer-api + app: stream-api ports: - port: 80 targetPort: 8000 diff --git a/provisioning/stream/kubernetes/templates/config/config-map.yaml b/provisioning/stream/kubernetes/templates/config/config-map.yaml new file mode 100644 index 0000000000..b87e1d853a --- /dev/null +++ b/provisioning/stream/kubernetes/templates/config/config-map.yaml @@ -0,0 +1,172 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: stream + namespace: $NAMESPACE + labels: + app: stream +data: + keboolaStack: "$KEBOOLA_STACK" + keboolaRevision: "$RELEASE_ID" + config.yaml: | + # Enable logging at DEBUG level. + debugLog: false + # Log HTTP client requests and responses as debug messages. + debugHTTPClient: false + # Path where CPU profile is saved. + cpuProfilePath: "" + # Unique ID of the node in the cluster. Validation rules: required + #nodeID: FROM ENV STREAM_NODE_ID + # Storage API host. Validation rules: required,hostname + storageApiHost: "connection.$HOSTNAME_SUFFIX" + datadog: + # Enable DataDog integration. + enabled: true + # Enable DataDog debug messages. + debug: false + etcd: + # Etcd endpoint. Validation rules: required + endpoint: stream-etcd.stream.svc.cluster.local:2379 + # Etcd namespace. Validation rules: required + namespace: stream/ + # Etcd username. + username: root + # Etcd password. + #password: FROM ENV STREAM_ETCD_PASSWORD + # Etcd connect timeout. Validation rules: required + connectTimeout: 30s + # Etcd keep alive timeout. Validation rules: required + keepAliveTimeout: 5s + # Etcd keep alive interval. Validation rules: required + keepAliveInterval: 10s + # Etcd operations logging as debug messages. + debugLog: false + metrics: + # Prometheus scraping metrics listen address. Validation rules: required,hostname_port + listen: 0.0.0.0:9000 + api: + # Listen address of the configuration HTTP API. Validation rules: required,hostname_port + listen: 0.0.0.0:8000 + # Public URL of the configuration HTTP API for link generation. Validation rules: required + publicUrl: "https://stream.$HOSTNAME_SUFFIX" + distribution: + # The maximum time to wait for creating a new session. Validation rules: required,minDuration=1s,maxDuration=1m + grantTimeout: 5s + # Timeout for the node registration to the cluster. Validation rules: required,minDuration=1s,maxDuration=5m + startupTimeout: 1m0s + # Timeout for the node un-registration from the cluster. Validation rules: required,minDuration=1s,maxDuration=5m + shutdownTimeout: 10s + # Interval of processing changes in the topology. Use 0 to disable the grouping. Validation rules: maxDuration=30s + eventsGroupInterval: 5s + # Seconds after which the node is automatically un-registered if an outage occurs. Validation rules: required,min=1,max=30 + ttlSeconds: 15 + source: + http: + # Listen address of the HTTP source. Validation rules: required,hostname_port + listen: 0.0.0.0:7000 + # Public URL of the HTTP source for link generation. + publicUrl: "https://stream-in.$HOSTNAME_SUFFIX" + storage: + # Mounted volumes path, each volume is in "{type}/{label}" subdir. Validation rules: required + volumesPath: "/stream/volumes" + statistics: + sync: + # Statistics synchronization interval, from memory to the etcd. Validation rules: required,minDuration=100ms,maxDuration=5s + interval: 1s + # Statistics synchronization timeout. Validation rules: required,minDuration=1s,maxDuration=1m + timeout: 30s + cache: + L2: + # Enable statistics L2 in-memory cache, otherwise only L1 cache is used. + enabled: true + # Statistics L2 in-memory cache invalidation interval. Validation rules: required,minDuration=100ms,maxDuration=5s + interval: 1s + cleanup: + # Enable storage cleanup. + enabled: true + # Cleanup interval. Validation rules: required,minDuration=5m,maxDuration=24h + interval: 30m0s + # How many files are deleted in parallel. Validation rules: required,min=1,max=500 + concurrency: 100 + # Expiration interval of a file that has not yet been imported. Validation rules: required,minDuration=1h,maxDuration=720h,gtefield=ArchivedFileExpiration + activeFileExpiration: 168h0m0s + # Expiration interval of a file that has already been imported. Validation rules: required,minDuration=15m,maxDuration=720h + archivedFileExpiration: 24h0m0s + level: + local: + volume: + assignment: + # Volumes count simultaneously utilized per sink. Validation rules: required,min=1,max=100 + count: 1 + # List of preferred volume types, start with the most preferred. Validation rules: required,min=1 + preferredTypes: + - default + registration: + # Number of seconds after the volume registration expires if the node is not available. Validation rules: required,min=1,max=60 + ttlSeconds: 10 + sync: + # Sync mode: "disabled", "cache" or "disk". Validation rules: required,oneof=disabled disk cache + mode: disk + # Wait for sync to disk OS cache or to disk hardware, depending on the mode. + wait: true + # Minimal interval between syncs to disk. Validation rules: min=0,maxDuration=2s,required_if=Mode disk,required_if=Mode cache + checkInterval: 5ms + # Written records count to trigger sync. Validation rules: min=0,max=1000000,required_if=Mode disk,required_if=Mode cache + countTrigger: 500 + # Written size to trigger sync. Validation rules: maxBytes=100MB,required_if=Mode disk,required_if=Mode cache + bytesTrigger: 1MB + # Interval from the last sync to trigger sync. Validation rules: min=0,maxDuration=2s,required_if=Mode disk,required_if=Mode cache + intervalTrigger: 50ms + allocation: + # Allocate disk space for each slice. + enabled: true + # Size of allocated disk space for a slice. Validation rules: required + static: 100MB + # Allocate disk space as % from the previous slice size. Validation rules: min=100,max=500 + relative: 110 + compression: + # Compression type. Validation rules: required,oneof=none gzip zstd + type: gzip + gzip: + # GZIP compression level: 1-9. Validation rules: min=1,max=9 + level: 1 + # GZIP implementation: standard, fast, parallel. Validation rules: required,oneof=standard fast parallel + implementation: parallel + # GZIP parallel block size. Validation rules: required,minBytes=16kB,maxBytes=100MB + blockSize: 256KB + # GZIP parallel concurrency, 0 = auto. + concurrency: 0 + zstd: + # ZSTD compression level: fastest, default, better, best. Validation rules: min=1,max=4 + level: 1 + # ZSTD window size. Validation rules: required,minBytes=1kB,maxBytes=512MB + windowSize: 1MB + # ZSTD concurrency, 0 = auto + concurrency: 0 + staging: + # Maximum number of slices in a file, a new file is created after reaching it. Validation rules: required,min=1,max=50000 + maxSlicesPerFile: 100 + # Maximum number of the Storage API file resources created in parallel within one operation. Validation rules: required,min=1,max=500 + parallelFileCreateLimit: 50 + upload: + # Minimal interval between uploads. Validation rules: required,minDuration=1s,maxDuration=5m + minInterval: 5s + trigger: + # Records count to trigger slice upload. Validation rules: required,min=1,max=10000000 + count: 10000 + # Records size to trigger slice upload. Validation rules: required,minBytes=100B,maxBytes=50MB + size: 1MB + # Duration from the last slice upload to trigger the next upload. Validation rules: required,minDuration=1s,maxDuration=30m + interval: 1m0s + target: + import: + # Minimal interval between imports. Validation rules: required,minDuration=30s,maxDuration=30m + minInterval: 1m0s + trigger: + # Records count to trigger file import. Validation rules: required,min=1,max=10000000 + count: 50000 + # Records size to trigger file import. Validation rules: required,minBytes=100B,maxBytes=500MB + size: 5MB + # Duration from the last import to trigger the next import. Validation rules: required,minDuration=60s,maxDuration=24h + interval: 5m0s diff --git a/provisioning/stream/kubernetes/templates/worker/network-policy.yaml b/provisioning/stream/kubernetes/templates/storage/network-policy.yaml similarity index 86% rename from provisioning/stream/kubernetes/templates/worker/network-policy.yaml rename to provisioning/stream/kubernetes/templates/storage/network-policy.yaml index 514c8523f1..f02b9de112 100644 --- a/provisioning/stream/kubernetes/templates/worker/network-policy.yaml +++ b/provisioning/stream/kubernetes/templates/storage/network-policy.yaml @@ -2,15 +2,15 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: buffer-worker + name: stream-storage namespace: $NAMESPACE labels: - app: buffer-worker + app: stream-storage releaseId: $RELEASE_ID spec: podSelector: matchLabels: - app: buffer-worker + app: stream-storage policyTypes: - Ingress ingress: diff --git a/provisioning/stream/kubernetes/templates/worker/deployment.yaml b/provisioning/stream/kubernetes/templates/storage/statefulset.yaml similarity index 55% rename from provisioning/stream/kubernetes/templates/worker/deployment.yaml rename to provisioning/stream/kubernetes/templates/storage/statefulset.yaml index 37f326c4f4..7b7a5f44bb 100644 --- a/provisioning/stream/kubernetes/templates/worker/deployment.yaml +++ b/provisioning/stream/kubernetes/templates/storage/statefulset.yaml @@ -1,30 +1,31 @@ --- -kind: Deployment +kind: StatefulSet apiVersion: apps/v1 metadata: - name: buffer-worker + name: stream-storage namespace: $NAMESPACE labels: - app: buffer-worker + app: stream-storage releaseId: $RELEASE_ID spec: - replicas: $BUFFER_WORKER_REPLICAS + serviceName: "stream-storage" + replicas: $STREAM_STORAGE_REPLICAS selector: matchLabels: - app: buffer-worker + app: stream-storage template: metadata: labels: - app: buffer-worker + app: stream-storage releaseId: $RELEASE_ID - buffer-etcd-client: "true" + stream-etcd-client: "true" tags.datadoghq.com/env: "$KEBOOLA_STACK" - tags.datadoghq.com/service: "buffer-worker" + tags.datadoghq.com/service: "stream" tags.datadoghq.com/version: "$RELEASE_ID" annotations: log: "true" ad.datadoghq.com/tags: '{"git.commit.sha": "$COMMIT_HASH", "git.repository_url": "github.com/keboola/keboola-as-code"}' - ad.datadoghq.com/buffer-worker.checks: | + ad.datadoghq.com/stream-storage.checks: | { "openmetrics": { "instances": [ @@ -39,38 +40,40 @@ spec: } } spec: -# terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 600 containers: - - name: buffer-worker - image: $BUFFER_WORKER_REPOSITORY:$BUFFER_WORKER_IMAGE_TAG + - name: stream-disk-writer + image: $STREAM_IMAGE_REPOSITORY:$STREAM_IMAGE_TAG + command: [ "/app/service" ] + args: [ "--config-file", "/stream/config/config.yaml", "disk-writer" ] # run disk writer component resources: + # Note: Update also soft GOMEMLIMIT bellow requests: cpu: "200m" - memory: $BUFFER_WORKER_MEMORY + memory: "512Mi" limits: - memory: $BUFFER_WORKER_MEMORY + memory: "512Mi" ports: - containerPort: 9000 # metrics + volumeMounts: + - name: config + mountPath: /stream/config + readOnly: true + - name: ssd-001 + mountPath: /stream/volumes/ssd/ssd-001 env: # 80% of the resources.limits.memory - name: GOMEMLIMIT - value: $BUFFER_WORKER_MEMORY_GOMEMLIMIT - - name: BUFFER_WORKER_STORAGE_API_HOST + value: "400MiB" + - name: STREAM_NODE_ID valueFrom: - configMapKeyRef: - name: buffer-worker - key: storageApiHost - - name: BUFFER_WORKER_ETCD_ENDPOINT - value: buffer-etcd.buffer.svc.cluster.local:2379 - - name: BUFFER_WORKER_ETCD_USERNAME - value: root - - name: BUFFER_WORKER_ETCD_PASSWORD + fieldRef: + fieldPath: metadata.name + - name: STREAM_ETCD_PASSWORD valueFrom: secretKeyRef: - name: buffer-etcd + name: stream-etcd key: etcd-root-password - - name: BUFFER_WORKER_ETCD_NAMESPACE - value: buffer - name: DD_AGENT_HOST valueFrom: fieldRef: @@ -87,5 +90,21 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['tags.datadoghq.com/version'] + volumes: + - name: config + configMap: + name: stream + items: + - key: config.yaml + path: config.yaml nodeSelector: nodepool: main + volumeClaimTemplates: + - metadata: + name: ssd-001 + spec: + accessModes: [ "ReadWriteOnce" ] + #storageClassName: "my-storage-class" + resources: + requests: + storage: 10Gi diff --git a/provisioning/stream/kubernetes/templates/worker/config-map.yaml b/provisioning/stream/kubernetes/templates/worker/config-map.yaml deleted file mode 100644 index 0a541b3832..0000000000 --- a/provisioning/stream/kubernetes/templates/worker/config-map.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: buffer-worker - namespace: $NAMESPACE - labels: - app: buffer-worker -data: - storageApiHost: "https://connection.$HOSTNAME_SUFFIX" - keboolaStack: "$KEBOOLA_STACK" - keboolaRevision: "$RELEASE_ID" diff --git a/provisioning/stream/kubernetes/templates/worker/pdb.yaml b/provisioning/stream/kubernetes/templates/worker/pdb.yaml deleted file mode 100644 index e55787a4ee..0000000000 --- a/provisioning/stream/kubernetes/templates/worker/pdb.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: buffer-worker-pdb - namespace: $NAMESPACE - labels: - app: buffer-worker -spec: - maxUnavailable: 1 - selector: - matchLabels: - app: buffer-worker diff --git a/provisioning/stream/local.sh b/provisioning/stream/local.sh index 97b594fcdf..71b98e8f4c 100644 --- a/provisioning/stream/local.sh +++ b/provisioning/stream/local.sh @@ -14,20 +14,15 @@ if ! minikube status > /dev/null; then fi # Build Docker image in the local Docker, so it is cached, if Minikube is destroyed -API_IMAGE="$BUFFER_API_REPOSITORY:$BUFFER_API_IMAGE_TAG" -WORKER_IMAGE="$BUFFER_WORKER_REPOSITORY:$BUFFER_WORKER_IMAGE_TAG" +SERVICE_IMAGE="$STREAM_IMAGE_REPOSITORY:$STREAM_IMAGE_TAG" echo -echo "Building API image ..." +echo "Building Service image ..." echo "--------------------------" -docker build -t "$API_IMAGE" -f "./docker/api/Dockerfile" "../../" +docker build -t "$SERVICE_IMAGE" -f "./docker/service/Dockerfile" "../../" echo -echo "Building Worker image ..." -echo "--------------------------" -docker build -t "$WORKER_IMAGE" -f "./docker/worker/Dockerfile" "../../" # Load the images to the Minikube -minikube image load --overwrite=true "$API_IMAGE" -minikube image load --overwrite=true "$WORKER_IMAGE" +minikube image load --overwrite=true "$SERVICE_IMAGE" echo echo "Images in the MiniKube:" @@ -59,5 +54,5 @@ echo "To clear the MiniKube:" echo "MINIKUBE_PROFILE=${MINIKUBE_PROFILE} minikube delete --purge" echo echo "Load balancer of the service is accessible at:" -minikube service --url --namespace "$NAMESPACE" buffer-api +minikube service --url --namespace "$NAMESPACE" stream-api diff --git a/provisioning/stream/wait.sh b/provisioning/stream/wait.sh index dde221a4e7..94e806933f 100644 --- a/provisioning/stream/wait.sh +++ b/provisioning/stream/wait.sh @@ -15,7 +15,7 @@ KUBERNETES_ROLLOUT_WAIT="${KUBERNETES_ROLLOUT_WAIT:=1200s}" echo echo "Waiting for the etcd rollout ..." echo "--------------------------" -if kubectl rollout status sts/buffer-etcd --namespace "$NAMESPACE" --timeout "$KUBERNETES_ROLLOUT_WAIT"; then +if kubectl rollout status sts/stream-etcd --namespace "$NAMESPACE" --timeout "$KUBERNETES_ROLLOUT_WAIT"; then echo echo "Etcd deployment has been successful." else @@ -24,7 +24,7 @@ else echo "--------------------------" echo "Logs:" echo "--------------------------" - kubectl logs --namespace "$NAMESPACE" --selector "app=buffer-etcd" --all-containers --prefix --timestamps --tail=-1 + kubectl logs --namespace "$NAMESPACE" --selector "app=stream-etcd" --all-containers --prefix --timestamps --tail=-1 echo "--------------------------" exit 1 fi @@ -33,7 +33,7 @@ fi echo echo "Waiting for the API rollout ..." echo "--------------------------" -if kubectl rollout status deployment/buffer-api --namespace "$NAMESPACE" --timeout "$KUBERNETES_ROLLOUT_WAIT"; then +if kubectl rollout status deployment/stream-api --namespace "$NAMESPACE" --timeout "$KUBERNETES_ROLLOUT_WAIT"; then echo echo "API deployment has been successful." echo "--------------------------" @@ -43,26 +43,26 @@ else echo "--------------------------" echo "Logs:" echo "--------------------------" - kubectl logs --namespace "$NAMESPACE" --selector "app=buffer-api" --all-containers --prefix --timestamps --tail=-1 + kubectl logs --namespace "$NAMESPACE" --selector "app=stream-api" --all-containers --prefix --timestamps --tail=-1 echo "--------------------------" exit 1 fi -# Wait for the Worker rollout +# Wait for the Storage rollout echo -echo "Waiting for the Worker rollout ..." +echo "Waiting for the Storage rollout ..." echo "--------------------------" -if kubectl rollout status deployment/buffer-worker --namespace "$NAMESPACE" --timeout "$KUBERNETES_ROLLOUT_WAIT"; then +if kubectl rollout status statefulset/stream-storage --namespace "$NAMESPACE" --timeout "$KUBERNETES_ROLLOUT_WAIT"; then echo - echo "Worker deployment has been successful." + echo "Storage rollout has been successful." echo "--------------------------" else echo - echo "Worker deployment failed." + echo "Storage rollout failed." echo "--------------------------" echo "Logs:" echo "--------------------------" - kubectl logs --namespace "$NAMESPACE" --selector "app=buffer-worker" --all-containers --prefix --timestamps --tail=-1 + kubectl logs --namespace "$NAMESPACE" --selector "app=stream-storage" --all-containers --prefix --timestamps --tail=-1 echo "--------------------------" exit 1 fi diff --git a/provisioning/templates-api/common.sh b/provisioning/templates-api/common.sh index e5f2ff6efa..525c036c3b 100644 --- a/provisioning/templates-api/common.sh +++ b/provisioning/templates-api/common.sh @@ -17,9 +17,12 @@ fi : ${TEMPLATES_API_ETCD_REPLICAS?"Missing TEMPLATES_API_ETCD_REPLICAS"} : ${ETCD_STORAGE_CLASS_NAME?"Missing ETCD_STORAGE_CLASS_NAME"} +# Default values +export STREAM_ETCD_MEMORY="${STREAM_ETCD_MEMORY:="256Mi"}" + # Constants export NAMESPACE="templates-api" -ETCD_HELM_CHART_VERSION="8.5.8" +ETCD_HELM_CHART_VERSION="10.1.0" # Disable pod disruption budget (51%) if replicaCount=1, so it doesn't block the rollout. TEMPLATES_API_ETCD_PDB_CREATE=$([[ $TEMPLATES_API_ETCD_REPLICAS -gt 1 ]] && echo 'true' || echo 'false') @@ -43,7 +46,11 @@ helm upgrade \ --set "replicaCount=$TEMPLATES_API_ETCD_REPLICAS" \ --set "pdb.create=$TEMPLATES_API_ETCD_PDB_CREATE" \ --set "auth.rbac.rootPassword=$ETCD_ROOT_PASSWORD" \ - --set "persistence.storageClass=$ETCD_STORAGE_CLASS_NAME" + --set "persistence.storageClass=$ETCD_STORAGE_CLASS_NAME" \ + --set "resources.requests.memory=$STREAM_ETCD_MEMORY" \ + --set "resources.limits.memory=$STREAM_ETCD_MEMORY" \ + --set "extraEnvVars[3].name=GOMEMLIMIT" \ + --set "extraEnvVars[3].value=${STREAM_ETCD_MEMORY}B" # API kubectl apply -f ./kubernetes/deploy/api/config-map.yaml diff --git a/provisioning/templates-api/deploy_local.sh b/provisioning/templates-api/deploy_local.sh index 6da6f9013e..f1aaeef8a0 100755 --- a/provisioning/templates-api/deploy_local.sh +++ b/provisioning/templates-api/deploy_local.sh @@ -14,7 +14,7 @@ export RELEASE_RELEASENAME="${RELEASE_RELEASENAME:=my-release}" export KEBOOLA_STACK="${KEBOOLA_STACK:=local-machine}" export HOSTNAME_SUFFIX="${HOSTNAME_SUFFIX:=keboola.com}" export TEMPLATES_API_REPOSITORY="${TEMPLATES_API_REPOSITORY:=docker.io/keboola/templates-api}" # docker.io prefix is required -export TEMPLATES_API_IMAGE_TAG="${TEMPLATES_API_IMAGE_TAG:=$(git rev-parse --short HEAD)}" +export TEMPLATES_API_IMAGE_TAG="${TEMPLATES_API_IMAGE_TAG:=$(git rev-parse --short HEAD)-$(date +%s)}" export TEMPLATES_API_REPLICAS="${TEMPLATES_API_REPLICAS:=3}" export TEMPLATES_API_ETCD_REPLICAS="${TEMPLATES_API_ETCD_REPLICAS:=1}"