From 3b196c91f7a3621de02777de30eb95699345f854 Mon Sep 17 00:00:00 2001 From: silversupreme Date: Tue, 21 Dec 2021 23:56:25 +0000 Subject: [PATCH 1/2] Add a demo for multiple configs provided to a Vector instance. --- .gitignore | 4 +++- multi-config/Chart.lock | 6 +++++ multi-config/Chart.yaml | 8 +++++++ multi-config/Makefile | 15 +++++++++++++ multi-config/README.md | 34 +++++++++++++++++++++++++++++ multi-config/templates/configs.yaml | 9 ++++++++ multi-config/values.yaml | 4 ++++ multi-config/vector/one.yaml | 4 ++++ multi-config/vector/two.yaml | 7 ++++++ 9 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 multi-config/Chart.lock create mode 100644 multi-config/Chart.yaml create mode 100644 multi-config/Makefile create mode 100644 multi-config/README.md create mode 100644 multi-config/templates/configs.yaml create mode 100644 multi-config/values.yaml create mode 100644 multi-config/vector/one.yaml create mode 100644 multi-config/vector/two.yaml diff --git a/.gitignore b/.gitignore index 8b0e640..4898111 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ auth_token -*.log \ No newline at end of file +*.log + +charts/ \ No newline at end of file diff --git a/multi-config/Chart.lock b/multi-config/Chart.lock new file mode 100644 index 0000000..236b400 --- /dev/null +++ b/multi-config/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: vector-aggregator + repository: https://helm.vector.dev + version: 0.19.1 +digest: sha256:279ac29f5b8c17a82d04de31a3a79ac5be39e7f9801da6802c09b5b9937a68b8 +generated: "2021-12-20T23:22:03.415603962Z" diff --git a/multi-config/Chart.yaml b/multi-config/Chart.yaml new file mode 100644 index 0000000..493d519 --- /dev/null +++ b/multi-config/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: multi-config-vector +version: 1.0.0 +description: A demonstration of multiple ConfigMaps being merged into a single Vector config. +dependencies: + - name: vector-aggregator + version: ~0.19.1 + repository: https://helm.vector.dev \ No newline at end of file diff --git a/multi-config/Makefile b/multi-config/Makefile new file mode 100644 index 0000000..fe8ef82 --- /dev/null +++ b/multi-config/Makefile @@ -0,0 +1,15 @@ +start-local: + @echo "ℹ️ \033[36mRunning Vector locally to demonstrate how configs are loaded from a directory.\033[0m\n" + @vector -C ./vector + +start-k8s: + @echo "ℹ️ \033[36mRunning Vector in target k8s cluster to demonstrate merged configmaps.\033[0m" + @echo "⚠️ \033[33mNote that this will run in whatever your current kubectl target cluster is!\033[0m\n" + @helm install -n vector test . + +stop-k8s: + @echo "⚠️ \033[33mRemoving the test k8s Vector instance.\033[0m\n" + @helm uninstall -n vector test + +logs-k8s: + @kubectl logs -n vector test-vector-aggregator-0 \ No newline at end of file diff --git a/multi-config/README.md b/multi-config/README.md new file mode 100644 index 0000000..59aa161 --- /dev/null +++ b/multi-config/README.md @@ -0,0 +1,34 @@ +# Composing Vector Config Files +This repo is a simple demonstration of providing Vector with multiple `ConfigMap` files from a Kubernetes cluster. By +leveraging this, users can break apart pieces of their Vector configs for management by multiple business units. + +## Constraints +* **This solution works inside of a vanilla Kubernetes cluster.** This is intended as a simple solution that any + customer can easily hit the ground running with. +* **Helm does not support templating in `values.yaml`.** [The method we use to provide extra config files to Vector does not support templated names](https://github.com/helm/helm/issues/2492), + which makes it difficult to make this a truly generic solution for your needs. In particular, this makes it + impossible to properly scope the ConfigMap names with `{{ .Release.Name }}` as you might wish. + +## The Basics +Multiple configs are provided to Vector by wrapping the official Helm chart with this one, which generates a ConfigMap +from a directory and adds that to the supported `extraConfigDirSources` for the Vector pod. + +To add configs to Vector in this setup, all you must do is add them to the `./vector` directory: both the locally-running +Vector as well as the one installed by the Helm chart are configured to utilize all files there. Only `.yaml` files are +supported for now. + +A simple `Makefile` is supplied which demonstrates the proper commands to test this setup both locally and through Helm. +Teardown commands are also provided there for when testing is complete. + +The Helm chart was tested in `minikube` running on an EC2 instance. + +## Next Steps +While useful, this setup does have limitations. In particular, the configs must be managed and provisioned inside of the +same Kubernetes cluster that Vector runs in. Managing multiple clusters still imposes a non-trivial operational overhead. +[Helm's template control structures](https://helm.sh/docs/chart_template_guide/control_structures/) could provide some +ways of providing for multiple kinds of clusters/deployments from a single chart, although doing so is beyond the scope +of this demo. + +Datadog's Observability Pipelines feature set (coming soon) can more easily provide config synchronization between a +fleet of Vector instances, and allow remote management of Vector in an easy-to-use UI. If you're interested, please +contact your TAM or Datadog Sales for more information on pricing and timelines. \ No newline at end of file diff --git a/multi-config/templates/configs.yaml b/multi-config/templates/configs.yaml new file mode 100644 index 0000000..dbfd07c --- /dev/null +++ b/multi-config/templates/configs.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: configs +data: +{{ range $path, $_ := .Files.Glob "vector/**.yaml" }} + {{ $path | base }}: | +{{ $.Files.Get $path | indent 4}} +{{ end }} diff --git a/multi-config/values.yaml b/multi-config/values.yaml new file mode 100644 index 0000000..bdbe23f --- /dev/null +++ b/multi-config/values.yaml @@ -0,0 +1,4 @@ +vector-aggregator: + extraConfigDirSources: + - configMap: + name: configs \ No newline at end of file diff --git a/multi-config/vector/one.yaml b/multi-config/vector/one.yaml new file mode 100644 index 0000000..255fb0f --- /dev/null +++ b/multi-config/vector/one.yaml @@ -0,0 +1,4 @@ +sources: + demo: + type: demo_logs + format: syslog \ No newline at end of file diff --git a/multi-config/vector/two.yaml b/multi-config/vector/two.yaml new file mode 100644 index 0000000..b1d9117 --- /dev/null +++ b/multi-config/vector/two.yaml @@ -0,0 +1,7 @@ +sinks: + debug: + type: console + target: stderr + encoding: json + inputs: + - demo From ddf08d37abab1bcc7cd8acb5a9c8a469f602cdb2 Mon Sep 17 00:00:00 2001 From: silversupreme Date: Thu, 23 Dec 2021 19:04:55 +0000 Subject: [PATCH 2/2] Update for clarity on versions and issues with config sizes. Update to latest Vector Helm chart. --- multi-config/Chart.lock | 8 ++++---- multi-config/Chart.yaml | 4 ++-- multi-config/Makefile | 2 +- multi-config/README.md | 12 +++++++----- multi-config/values.yaml | 7 +++---- multi-config/vector/global.yaml | 5 +++++ 6 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 multi-config/vector/global.yaml diff --git a/multi-config/Chart.lock b/multi-config/Chart.lock index 236b400..5d9b594 100644 --- a/multi-config/Chart.lock +++ b/multi-config/Chart.lock @@ -1,6 +1,6 @@ dependencies: -- name: vector-aggregator +- name: vector repository: https://helm.vector.dev - version: 0.19.1 -digest: sha256:279ac29f5b8c17a82d04de31a3a79ac5be39e7f9801da6802c09b5b9937a68b8 -generated: "2021-12-20T23:22:03.415603962Z" + version: 0.2.2 +digest: sha256:ae9c41d335a82592b150e8511e6ae5dbe26fbb364399dad5ba764ca88dcdc824 +generated: "2021-12-23T18:59:04.637364547Z" diff --git a/multi-config/Chart.yaml b/multi-config/Chart.yaml index 493d519..2fe79cc 100644 --- a/multi-config/Chart.yaml +++ b/multi-config/Chart.yaml @@ -3,6 +3,6 @@ name: multi-config-vector version: 1.0.0 description: A demonstration of multiple ConfigMaps being merged into a single Vector config. dependencies: - - name: vector-aggregator - version: ~0.19.1 + - name: vector + version: ~0.2 repository: https://helm.vector.dev \ No newline at end of file diff --git a/multi-config/Makefile b/multi-config/Makefile index fe8ef82..226be93 100644 --- a/multi-config/Makefile +++ b/multi-config/Makefile @@ -12,4 +12,4 @@ stop-k8s: @helm uninstall -n vector test logs-k8s: - @kubectl logs -n vector test-vector-aggregator-0 \ No newline at end of file + @kubectl logs -n vector test-vector-0 \ No newline at end of file diff --git a/multi-config/README.md b/multi-config/README.md index 59aa161..133e384 100644 --- a/multi-config/README.md +++ b/multi-config/README.md @@ -1,17 +1,19 @@ # Composing Vector Config Files -This repo is a simple demonstration of providing Vector with multiple `ConfigMap` files from a Kubernetes cluster. By -leveraging this, users can break apart pieces of their Vector configs for management by multiple business units. +This repo is a simple demonstration of providing Vector with multiple config files, which are merged into a ConfigMap +in the destination Kubernetes cluster. By leveraging this, users can break apart pieces of their Vector configs for +management by multiple business units. ## Constraints * **This solution works inside of a vanilla Kubernetes cluster.** This is intended as a simple solution that any customer can easily hit the ground running with. * **Helm does not support templating in `values.yaml`.** [The method we use to provide extra config files to Vector does not support templated names](https://github.com/helm/helm/issues/2492), which makes it difficult to make this a truly generic solution for your needs. In particular, this makes it - impossible to properly scope the ConfigMap names with `{{ .Release.Name }}` as you might wish. + impossible to properly scope the ConfigMap names with `{{ .Release.Name }}` as you might wish, or to mount each config + file as a separate ConfigMap to avoid [file size limits](https://kubernetes.io/docs/concepts/configuration/configmap/#:~:text=A%20ConfigMap%20is%20not%20designed,separate%20database%20or%20file%20service.). ## The Basics Multiple configs are provided to Vector by wrapping the official Helm chart with this one, which generates a ConfigMap -from a directory and adds that to the supported `extraConfigDirSources` for the Vector pod. +from a directory and adds that to the supported `existingConfigMaps` for the Vector pod. To add configs to Vector in this setup, all you must do is add them to the `./vector` directory: both the locally-running Vector as well as the one installed by the Helm chart are configured to utilize all files there. Only `.yaml` files are @@ -20,7 +22,7 @@ supported for now. A simple `Makefile` is supplied which demonstrates the proper commands to test this setup both locally and through Helm. Teardown commands are also provided there for when testing is complete. -The Helm chart was tested in `minikube` running on an EC2 instance. +The Helm chart was tested in `minikube v1.24.0 (76b94fb3c4e8ac5062daf70d60cf03ddcc0a741b)` running on Ubuntu 20.04.3 on a `c6g.2xlarge` EC2 instance. ## Next Steps While useful, this setup does have limitations. In particular, the configs must be managed and provisioned inside of the diff --git a/multi-config/values.yaml b/multi-config/values.yaml index bdbe23f..0d9c1e7 100644 --- a/multi-config/values.yaml +++ b/multi-config/values.yaml @@ -1,4 +1,3 @@ -vector-aggregator: - extraConfigDirSources: - - configMap: - name: configs \ No newline at end of file +vector: + existingConfigMaps: + - configs \ No newline at end of file diff --git a/multi-config/vector/global.yaml b/multi-config/vector/global.yaml new file mode 100644 index 0000000..7ce86dc --- /dev/null +++ b/multi-config/vector/global.yaml @@ -0,0 +1,5 @@ +data_dir: /vector-data-dir +api: + enabled: true + address: 127.0.0.1:8686 + playground: false \ No newline at end of file