From d0399e077f2d7767bb44a754e3f7152aab507a12 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 16 Apr 2024 17:45:35 +0800 Subject: [PATCH] add_cloud_metadata: env var override for providers (#38669) * add_cloud_metadata: env var override for providers Add support for configuring the add_cloud_metadata providers with an environment variable, $BEATS_ADD_CLOUD_METADATA_PROVIDERS. This may be useful when deploying Elastic Agent standalone in a cloud provider managed Kubernetes cluster, where the cloud provider is known at deployment time. * Fix changelog * appease golangci-lint * More appeasement --- CHANGELOG.next.asciidoc | 1 + .../docs/add_cloud_metadata.asciidoc | 3 ++ .../add_cloud_metadata/providers.go | 17 ++++++++++ .../add_cloud_metadata/providers_test.go | 33 +++++++++++++++---- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 96ed34ffbee4..97d2ae10b14d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -173,6 +173,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] - Update to Go 1.21.9. {pulk}38727[38727] - Enable early event encoding in the Elasticsearch output, improving cpu and memory use {pull}38572[38572] +- The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669] *Auditbeat* diff --git a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc index f2c8db137306..a80cd7a8be4e 100644 --- a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc +++ b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc @@ -50,6 +50,9 @@ cloud or on-premise). The second optional setting is `providers`. The `providers` settings accepts a list of cloud provider names to be used. If `providers` is not configured, then all providers that do not access a remote endpoint are enabled by default. +The list of providers may alternatively be configured with the environment +variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS`, by setting it to a comma-separated +list of provider names. List of names the `providers` setting supports: diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go index 77c4c7042add..a9978251cfd6 100644 --- a/libbeat/processors/add_cloud_metadata/providers.go +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -22,6 +22,8 @@ import ( "fmt" "net" "net/http" + "os" + "strings" "time" conf "github.com/elastic/elastic-agent-libs/config" @@ -73,6 +75,21 @@ func selectProviders(configList providerList, providers map[string]provider) map } func providersFilter(configList providerList, allProviders map[string]provider) func(string) bool { + if v, ok := os.LookupEnv("BEATS_ADD_CLOUD_METADATA_PROVIDERS"); ok { + // We allow users to override the config and defaults with + // this environment variable as a workaround in case the + // configured/default providers misbehave. + configList = nil + for _, name := range strings.Split(v, ",") { + configList = append(configList, strings.TrimSpace(name)) + } + if len(configList) == 0 { + // User explicitly disabled all providers. + return func(string) bool { + return false + } + } + } if len(configList) == 0 { return func(name string) bool { ff, ok := allProviders[name] diff --git a/libbeat/processors/add_cloud_metadata/providers_test.go b/libbeat/processors/add_cloud_metadata/providers_test.go index d974ef77695c..85336a4c2b3f 100644 --- a/libbeat/processors/add_cloud_metadata/providers_test.go +++ b/libbeat/processors/add_cloud_metadata/providers_test.go @@ -18,6 +18,7 @@ package add_cloud_metadata import ( + "os" "sort" "testing" @@ -26,11 +27,13 @@ import ( conf "github.com/elastic/elastic-agent-libs/config" ) +func init() { + os.Unsetenv("BEATS_ADD_CLOUD_METADATA_PROVIDERS") +} + func TestProvidersFilter(t *testing.T) { - var all []string var allLocal []string for name, ff := range cloudMetaProviders { - all = append(all, name) if ff.Local { allLocal = append(allLocal, name) } @@ -38,6 +41,7 @@ func TestProvidersFilter(t *testing.T) { cases := map[string]struct { config map[string]interface{} + env string fail bool expected []string }{ @@ -45,6 +49,16 @@ func TestProvidersFilter(t *testing.T) { config: map[string]interface{}{}, expected: allLocal, }, + "BEATS_ADD_CLOUD_METADATA_PROVIDERS overrides default": { + config: map[string]interface{}{}, + env: "alibaba, digitalocean", + expected: []string{"alibaba", "digitalocean"}, + }, + "none if BEATS_ADD_CLOUD_METADATA_PROVIDERS is explicitly set to an empty list": { + config: map[string]interface{}{}, + env: " ", + expected: nil, + }, "fail to load if unknown name is used": { config: map[string]interface{}{ "providers": []string{"unknown"}, @@ -56,18 +70,25 @@ func TestProvidersFilter(t *testing.T) { "providers": []string{"aws", "gcp", "digitalocean"}, }, }, + "BEATS_ADD_CLOUD_METADATA_PROVIDERS overrides selected": { + config: map[string]interface{}{ + "providers": []string{"aws", "gcp", "digitalocean"}, + }, + env: "alibaba, digitalocean", + expected: []string{"alibaba", "digitalocean"}, + }, } copyStrings := func(in []string) (out []string) { - for _, str := range in { - out = append(out, str) - } - return out + return append(out, in...) } for name, test := range cases { t.Run(name, func(t *testing.T) { rawConfig := conf.MustNewConfigFrom(test.config) + if test.env != "" { + t.Setenv("BEATS_ADD_CLOUD_METADATA_PROVIDERS", test.env) + } config := defaultConfig() err := rawConfig.Unpack(&config)