From 5b67a542997cfcbbdaf79ba5d41ee892106ad852 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Sat, 13 Apr 2024 11:22:03 +0200 Subject: [PATCH] [Metricbeat][Aerospike Module] Add support for TLS (#38126) * Add TLS Support for Metricbeat module aerospike * Module aerospike: add support for basic auth * Move the initialization of the clientpolicy from the metricset to the module * Aerospike: add test to check the initialization of ssl and user/password in the clientpolicy * Aerospike: Add support for cluster_name setting * Aerospike: Add test for ClusterName * Aerospike module: add commented username, password, cluster_name and TLS in the aerospike.yml.disabled config * Aerospike Module: Fix unit test * Fix test to avoid using hard-coded value * Adjust configuration of aerospike to show username/password and TLS settings * Executed goimports -w -local github.com/elastic metricbeat/module/aerospike * Improve Test error messages to provide more context * Drop support for Basic Auth that would require an upgrade of the dependencies * Fix Documentation by adding SSL reference - Add to config and config.reference ssl examples - Adjust the fields.yml to contain reference to ssl * Update metricbeat/module/aerospike/aerospike_test.go Co-authored-by: subham sarkar * Address reviewer's request: - [reference-config] add empty line after hosts list - [test] remove empty line after function header - [test] use generic pointer instead of custom one * Run mage config * mage check * Replace "Optional SSL/TLS. By default is off." with disabled by default * Convert ClusterName to a simple string instead of pointer to a string * Update x-pack/metricbeat.reference.yml --------- Co-authored-by: subham sarkar Co-authored-by: subham sarkar --- metricbeat/docs/modules/aerospike.asciidoc | 17 +++++ metricbeat/metricbeat.reference.yml | 15 +++++ .../aerospike/_meta/config.reference.yml | 15 +++++ metricbeat/module/aerospike/_meta/config.yml | 15 +++++ metricbeat/module/aerospike/_meta/fields.yml | 1 + metricbeat/module/aerospike/aerospike.go | 28 ++++++++ metricbeat/module/aerospike/aerospike_test.go | 67 +++++++++++++++++++ metricbeat/module/aerospike/fields.go | 2 +- .../module/aerospike/namespace/namespace.go | 15 +++-- metricbeat/modules.d/aerospike.yml.disabled | 15 +++++ x-pack/metricbeat/metricbeat.reference.yml | 15 +++++ 11 files changed, 200 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/modules/aerospike.asciidoc b/metricbeat/docs/modules/aerospike.asciidoc index b900aae5a130..a2c873665e11 100644 --- a/metricbeat/docs/modules/aerospike.asciidoc +++ b/metricbeat/docs/modules/aerospike.asciidoc @@ -41,8 +41,25 @@ metricbeat.modules: enabled: true period: 10s hosts: ["localhost:3000"] + + # Aerospike Cluster Name + #cluster_name: myclustername + + # Optional SSL/TLS (disabled by default) + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" ---- +This module supports TLS connections when using `ssl` config field, as described in <>. + [float] === Metricsets diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3f9ccb0a9dbc..2538bef77d35 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -149,6 +149,21 @@ metricbeat.modules: period: 10s hosts: ["localhost:3000"] + # Aerospike Cluster Name + #cluster_name: myclustername + + # Optional SSL/TLS (disabled by default) + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" + #-------------------------------- Apache Module -------------------------------- - module: apache metricsets: ["status"] diff --git a/metricbeat/module/aerospike/_meta/config.reference.yml b/metricbeat/module/aerospike/_meta/config.reference.yml index 38aca68f9514..cca162aa3db1 100644 --- a/metricbeat/module/aerospike/_meta/config.reference.yml +++ b/metricbeat/module/aerospike/_meta/config.reference.yml @@ -3,3 +3,18 @@ enabled: true period: 10s hosts: ["localhost:3000"] + + # Aerospike Cluster Name + #cluster_name: myclustername + + # Optional SSL/TLS (disabled by default) + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/module/aerospike/_meta/config.yml b/metricbeat/module/aerospike/_meta/config.yml index a4e7384ed558..42db4e483321 100644 --- a/metricbeat/module/aerospike/_meta/config.yml +++ b/metricbeat/module/aerospike/_meta/config.yml @@ -3,3 +3,18 @@ # - namespace period: 10s hosts: ["localhost:3000"] + + # Aerospike Cluster Name + #cluster_name: myclustername + + # Optional SSL/TLS (disabled by default) + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/module/aerospike/_meta/fields.yml b/metricbeat/module/aerospike/_meta/fields.yml index 2f6aa736c7a0..496aa734452a 100644 --- a/metricbeat/module/aerospike/_meta/fields.yml +++ b/metricbeat/module/aerospike/_meta/fields.yml @@ -3,6 +3,7 @@ description: > Aerospike module release: ga + settings: ["ssl"] fields: - name: aerospike type: group diff --git a/metricbeat/module/aerospike/aerospike.go b/metricbeat/module/aerospike/aerospike.go index 65cfabf62396..745914c055d0 100644 --- a/metricbeat/module/aerospike/aerospike.go +++ b/metricbeat/module/aerospike/aerospike.go @@ -22,9 +22,37 @@ import ( "strconv" "strings" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" + as "github.com/aerospike/aerospike-client-go" ) +type Config struct { + ClusterName string `config:"cluster_name"` + TLS *tlscommon.Config `config:"ssl"` +} + +// DefaultConfig return default config for the aerospike module. +func DefaultConfig() Config { + return Config{} +} + +func ParseClientPolicy(config Config) (*as.ClientPolicy, error) { + clientPolicy := as.NewClientPolicy() + if config.TLS.IsEnabled() { + tlsconfig, err := tlscommon.LoadTLSConfig(config.TLS) + if err != nil { + return nil, fmt.Errorf("could not initialize TLS configurations %w", err) + } + clientPolicy.TlsConfig = tlsconfig.ToConfig() + } + + if config.ClusterName != "" { + clientPolicy.ClusterName = config.ClusterName + } + return clientPolicy, nil +} + func ParseHost(host string) (*as.Host, error) { pieces := strings.Split(host, ":") if len(pieces) != 2 { diff --git a/metricbeat/module/aerospike/aerospike_test.go b/metricbeat/module/aerospike/aerospike_test.go index cb533e5a6bf7..9dbcbeec4a5d 100644 --- a/metricbeat/module/aerospike/aerospike_test.go +++ b/metricbeat/module/aerospike/aerospike_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" + as "github.com/aerospike/aerospike-client-go" ) @@ -96,3 +98,68 @@ func TestParseInfo(t *testing.T) { assert.Equal(t, test.expected, result, test.Name) } } + +func pointer[T any](d T) *T { + return &d +} + +func TestParseClientPolicy(t *testing.T) { + sampleClusterName := "TestCluster" + + TLSPolicy := as.NewClientPolicy() + tlsconfig, _ := tlscommon.LoadTLSConfig(&tlscommon.Config{Enabled: pointer(true)}) + TLSPolicy.TlsConfig = tlsconfig.ToConfig() + + ClusterNamePolicy := as.NewClientPolicy() + ClusterNamePolicy.ClusterName = sampleClusterName + + tests := []struct { + Name string + Config Config + expectedClientPolicy *as.ClientPolicy + expectedErr error + }{ + { + Name: "Empty configuration leads to default policy", + Config: Config{}, + expectedClientPolicy: as.NewClientPolicy(), + expectedErr: nil, + }, + { + Name: "TLS Declaration", + Config: Config{ + TLS: &tlscommon.Config{ + Enabled: pointer(true), + }, + }, + expectedClientPolicy: TLSPolicy, + expectedErr: nil, + }, + { + Name: "Cluster Name Setting", + Config: Config{ + ClusterName: sampleClusterName, + }, + expectedClientPolicy: ClusterNamePolicy, + expectedErr: nil, + }, + } + + for _, test := range tests { + result, err := ParseClientPolicy(test.Config) + if err != nil { + if test.expectedErr != nil { + assert.Equalf(t, test.expectedErr.Error(), err.Error(), + "Aerospike policy the error produced is not the one expected: got '%s', expected '%s'", err.Error(), test.expectedErr.Error()) + continue + } + t.Error(err) + continue + } + assert.Equalf(t, test.expectedClientPolicy.ClusterName, result.ClusterName, + "Aerospike policy cluster name is wrong. Got '%s' expected '%s'", result.ClusterName, test.expectedClientPolicy.ClusterName) + if test.Config.TLS.IsEnabled() { + assert.NotNil(t, result.TlsConfig, "Aerospike policy: TLS is not set even though TLS is specified in the configuration") + } + } +} diff --git a/metricbeat/module/aerospike/fields.go b/metricbeat/module/aerospike/fields.go index 7fe2567ee93a..d7ac01c32831 100644 --- a/metricbeat/module/aerospike/fields.go +++ b/metricbeat/module/aerospike/fields.go @@ -32,5 +32,5 @@ func init() { // AssetAerospike returns asset data. // This is the base64 encoded zlib format compressed contents of module/aerospike. func AssetAerospike() string { - return "eJzUmMGO4zYMhu95CmIve2nyADkUWLSXHnZRFL0VxYCR6FgdWTQkalIDffhCdjzj2HKSycxuPDrkYEvk/8mkRGYNj9RsAclzqM0jrQDEiKUtfPrSP/u0AtAUlDe1GHZb+HkFAPD8HirW0aalnixhoC3scQVQGLI6bNvJa3BY0amjNKSp03TPsT4+yXg6NTU0l35DjYqe3+RMzprtRs7IKUk/xjKGUpQ15OTk1ZyWC3rS+KW1BkFQwmb0NqdiqESTJaHJ63NqrlA0UNU5APHoAqq0IOSVnlM7VEzes8/O6EVbdvuZCVfoTuNbrHbkgYvjl8pSSIkCBRpLGg5GSkDXicuRDcKQ5aHg6PRiGDyFaIU0GAeY9EGr7zxGiEpRCD8M4uiviPYMz3nJYiriOE677yf50r4nPRo4ylR2L9kT5uLkvZIzmf/oqTlm6FRcCIW7JeFE7SQDP0z+TYJnkdmX3/ArUu/gzXe9GFv7Hz35MhA3Xov3j+cpyyIDem7LZ0L6pdB7MmoczzeXnL+a8AhB2OOeurB9Zd2JT2gs7ixtapXbwk5ZUGhJPxSWMTepYF+hbKEmr6b19BUYaXwlDNFT2kWCyjhTxQoUOzH7yDGAblFTyQ+oPIcAaG37NBwLpr4lmD9LCk/3B/29W5y+GBcdlsIalZGmFZi8gJQmXEMkLGg3u0Yol7Jng7+HmVt8BcqfyXtn4Jnl+ImsZYXpKhUewUB7+KcnrM+QxUB6gWBJFuya65l6nvJQPew8oSppXO10NDtmS+hedwD8VoD4SD8NOvsSA/SO4HNp9uX6gEJ+/VfC+K+iin3z97pW8vliqPXiu0XvdWp9ba2dnluvbZgXmMrdJl1M5tfEv0bBuyXBl4qjkwEZKxVr08V/UvZGOOM0/btMunQHtfLeiBgWzBhIsdPom46UwhtZF3UVHXlvPq3Tb/a4e6TmwH58gl8Q+O3Z88TuSyesaVNyyP8beZtX1gQTkycO3xkzOZwl5N0/pCYF6s2XyB+k2OuZZu3S7VFhEMp1aWfD9IowfGkT/FHfMNbgUBpVAvpUw4p5oqOQTIdzUuD9EKHGjdPk5TyYzZMgXD+0rVD+w76lqBnLMQFU9J6c2Kb9eyYVmQfj9l0vFjar/wMAAP//lvJazw==" + return "eJzUmMFu4zYQhu9+ikEue6n9AD4UWLSXHnZRFL0tFsGYHFlsKFLgDOMK6MMXlKxElijbcbIbhQcfRHHm/6gZcsZreKBmC0jBc20eaAUgRixt4e5z/+xuBaCJVTC1GO+28OsKAOBpHiqvo01LA1lCpi3scQXAJGLcnrfw7Y7Z3n1fARSGrOZta2ANDis6dZ6GNHUyEXysj08y3k9NDc2lX65R0dNMzuSs2W7kjJzS9WMsYyhFWUNOTqbmtFzQk8ZvrTVgQeHNaDanYqhEkyWhyfQ5NVcoGqjqHIAEdIwqLeC80nNqh4opBB+yb/SirXf7mReu0J3G11jtKIAvjl8qSyElChRoLGk4GCkBXScuRzYIQy/3hY9OL4YhEEcrpME4wKQPWn3nMTgqRcw/DeLor4j2DM95yWIq8nGcdj9O8qV9T3o0+ChT2b3kQJiLk7dKzmT+o6fmmKFTcSEU3i0JJ2onGfhh8m8SPIvMvvyGX5F6h2B+6MXY2v/oyZeBuPFafP94nrIsMqDntnwmpJ8LvUejxvF8c8n5u+EHYPEB99SF7QvrTnxEY3FnaVOr3BZ2ylihJX1fWI+5lwofKpQt1BTUtJ6+AiONL4QcA6VdJKiMM1WsQHknZh99ZNAtair5AVXwzIDWtk/5WDD1LcH8WVIEen/QP7vF6Yv5osNSWKMy0rQCkxeQ0vA1ROIF7WbXCOVS9mzw9zBzi69A+Tt57ww8sRw/kbVeYbpKxY9goD380xOvz5BFJr1AsCQLds31TD1Peajud4FQlTSudjqanfeW0L3sAPijAAmRfhl0+yUy9I7gU2n25fqAQmH9LWH8V1HlQ/N9XSv5dDHUevHdorc6tb601k7PrZc2zAtM5W6TLibzS+Jfo+C7JcHnykcnAzKvVKxNF/9J2SvhjNP07zLp0h3UynslIi+YkUl5pzE0HSnxK1kXdRUdeW8+rdNv9rh7oObgw/gEvyDw65Pnid3nTljTpvSc/zfyNq9eE0xMnjh8Y8zkcJbQ7/4hNSlQb75E/iLlg55p1i7dHhWyUK5LOxumV4Thc5sQjvqGsQaH0qgSMKQaVswjHYVkOpyTAu+nCDVunCbP58FsnrD4+r5thfIf9jVFzViOYVAxBHJim/bvmVRkHozbd70Yb1b/BwAA//89umEy" } diff --git a/metricbeat/module/aerospike/namespace/namespace.go b/metricbeat/module/aerospike/namespace/namespace.go index 97beb050ce13..265db9f933bc 100644 --- a/metricbeat/module/aerospike/namespace/namespace.go +++ b/metricbeat/module/aerospike/namespace/namespace.go @@ -42,15 +42,16 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - host *as.Host - client *as.Client + host *as.Host + clientPolicy *as.ClientPolicy + client *as.Client } // New create a new instance of the MetricSet // Part of new is also setting up the configuration by processing additional // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := aerospike.DefaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } @@ -60,9 +61,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, fmt.Errorf("Invalid host format, expected hostname:port: %w", err) } + clientPolicy, err := aerospike.ParseClientPolicy(config) + if err != nil { + return nil, fmt.Errorf("could not initialize aerospike client policy: %w", err) + } + return &MetricSet{ BaseMetricSet: base, host: host, + clientPolicy: clientPolicy, }, nil } @@ -105,7 +112,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { // create an aerospike client if it doesn't exist yet func (m *MetricSet) connect() error { if m.client == nil { - client, err := as.NewClientWithPolicyAndHost(as.NewClientPolicy(), m.host) + client, err := as.NewClientWithPolicyAndHost(m.clientPolicy, m.host) if err != nil { return err } diff --git a/metricbeat/modules.d/aerospike.yml.disabled b/metricbeat/modules.d/aerospike.yml.disabled index 5294b90301ed..35aad6b8e406 100644 --- a/metricbeat/modules.d/aerospike.yml.disabled +++ b/metricbeat/modules.d/aerospike.yml.disabled @@ -6,3 +6,18 @@ # - namespace period: 10s hosts: ["localhost:3000"] + + # Aerospike Cluster Name + #cluster_name: myclustername + + # Optional SSL/TLS (disabled by default) + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index f71e58904fd7..6877f2b45346 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -158,6 +158,21 @@ metricbeat.modules: period: 10s hosts: ["localhost:3000"] + # Aerospike Cluster Name + #cluster_name: myclustername + + # Optional SSL/TLS (disabled by default) + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" + #------------------------------- Airflow Module ------------------------------- - module: airflow host: "localhost"