From 71d6a46d5326bec05527764937ffa64b9383eb61 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Thu, 9 Jan 2025 14:27:12 +0100 Subject: [PATCH] Add logs support --- cmd/otelcontribcol/builder-config.yaml | 1 + exporter/sematextexporter/README.md | 10 ++- exporter/sematextexporter/config.go | 40 +++++++-- exporter/sematextexporter/config_test.go | 90 ++++++++++++++++++- exporter/sematextexporter/factory.go | 29 +++++- .../generated_component_test.go | 7 ++ .../internal/metadata/generated_status.go | 1 + exporter/sematextexporter/metadata.yaml | 4 +- .../sematextexporter/testdata/config.yaml | 4 +- 9 files changed, 167 insertions(+), 19 deletions(-) diff --git a/cmd/otelcontribcol/builder-config.yaml b/cmd/otelcontribcol/builder-config.yaml index b31409e27bae..6809c9a2c8e3 100644 --- a/cmd/otelcontribcol/builder-config.yaml +++ b/cmd/otelcontribcol/builder-config.yaml @@ -89,6 +89,7 @@ exporters: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/pulsarexporter v0.114.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter v0.114.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sapmexporter v0.114.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter v0.114.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sentryexporter v0.114.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter v0.114.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter v0.114.0 diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md index b7514a1698b6..eeb10660cc5f 100644 --- a/exporter/sematextexporter/README.md +++ b/exporter/sematextexporter/README.md @@ -2,16 +2,16 @@ | Status | | | ------------- |-----------| -| Stability | [development]: metrics | +| Stability | [development]: metrics, logs | | Distributions | [contrib] | | Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aexporter%2Fsematext%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aexporter%2Fsematext) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aexporter%2Fsematext%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aexporter%2Fsematext) | -| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@Eromosele-SM](https://www.github.com/Eromosele-SM) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@AkhigbeEromo](https://www.github.com/AkhigbeEromo) | -[development]: https://github.com/open-telemetry/opentelemetry-collector#development +[development]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#development [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -This exporter supports sending metrics to [Sematext Cloud](https://sematext.com/) in Influx line protocol format +This exporter supports sending metrics to [Sematext Cloud](https://sematext.com/) in Influx line protocol format and logs using the Bulk Index Api format. ## Configuration @@ -53,4 +53,6 @@ metrics: queue_size: 10 payload_max_lines: 100 payload_max_bytes: 1000 +logs: + app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ``` \ No newline at end of file diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index ed11f43ce3fd..fba6bbec8e78 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -12,7 +12,15 @@ import ( "go.opentelemetry.io/collector/exporter/exporterhelper" ) -const appTokenLength = 36 +const ( + euRegion = "eu" + usRegion = "us" + euMetricsEndpoint = "https://spm-receiver.eu.sematext.com" + euLogsEndpoint = "https://logsene-receiver.eu.sematext.com" + usMetricsEndpoint = "https://spm-receiver.sematext.com" + usLogsEndpoint = "https://logsene-receiver.sematext.com" + appTokenLength = 36 +) type Config struct { confighttp.ClientConfig `mapstructure:",squash"` @@ -21,8 +29,11 @@ type Config struct { // Options: // - EU // - US - Region string `mapstructure:"region"` + Region string `mapstructure:"region"` + // MetricsConfig defines the configuration specific to metrics MetricsConfig `mapstructure:"metrics"` + // LogsConfig defines the configuration specific to logs + LogsConfig `mapstructure:"logs"` } type MetricsConfig struct { @@ -39,20 +50,31 @@ type MetricsConfig struct { // PayloadMaxBytes is the maximum number of line protocol bytes to POST in a single request. PayloadMaxBytes int `mapstructure:"payload_max_bytes"` } +type LogsConfig struct { + // App token is the token of Sematext Monitoring App to which you want to send the logs. + AppToken string `mapstructure:"app_token"` + // LogsEndpoint specifies the endpoint for receiving logs in Sematext + LogsEndpoint string `mapstructure:"logs_endpoint"` +} // Validate checks for invalid or missing entries in the configuration. func (cfg *Config) Validate() error { - if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" && strings.ToLower(cfg.Region) != "custom" { + if strings.ToLower(cfg.Region) != euRegion && strings.ToLower(cfg.Region) != usRegion { return fmt.Errorf("invalid region: %s. please use either 'EU' or 'US'", cfg.Region) } - if len(cfg.AppToken) != appTokenLength { - return fmt.Errorf("invalid app_token: %s. app_token should be 36 characters", cfg.AppToken) + if len(cfg.MetricsConfig.AppToken) != appTokenLength { + return fmt.Errorf("invalid metrics app_token: %s. app_token should be 36 characters", cfg.MetricsConfig.AppToken) + } + if len(cfg.LogsConfig.AppToken) != appTokenLength { + return fmt.Errorf("invalid logs app_token: %s. app_token should be 36 characters", cfg.LogsConfig.AppToken) } - if strings.ToLower(cfg.Region) == "eu" { - cfg.MetricsEndpoint = "https://spm-receiver.eu.sematext.com" + if strings.ToLower(cfg.Region) == euRegion { + cfg.MetricsEndpoint = euMetricsEndpoint + cfg.LogsEndpoint = euLogsEndpoint } - if strings.ToLower(cfg.Region) == "us" { - cfg.MetricsEndpoint = "https://spm-receiver.sematext.com" + if strings.ToLower(cfg.Region) == usRegion { + cfg.MetricsEndpoint = usMetricsEndpoint + cfg.LogsEndpoint = usLogsEndpoint } return nil diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 7b3dc4b0d2b4..5428a5e642ba 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -54,6 +54,10 @@ func TestLoadConfig(t *testing.T) { PayloadMaxLines: 72, PayloadMaxBytes: 27, }, + LogsConfig: LogsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + LogsEndpoint: "https://logsene-receiver.sematext.com", + }, BackOffConfig: configretry.BackOffConfig{ Enabled: true, @@ -63,7 +67,7 @@ func TestLoadConfig(t *testing.T) { RandomizationFactor: backoff.DefaultRandomizationFactor, Multiplier: backoff.DefaultMultiplier, }, - Region: "US", + Region: "us", }, }, } @@ -82,3 +86,87 @@ func TestLoadConfig(t *testing.T) { }) } } +func TestConfigValidation(t *testing.T) { + tests := []struct { + name string + config *Config + expectError bool + }{ + { + name: "Valid configuration 1", + config: &Config{ + Region: "US", + MetricsConfig: MetricsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + LogsConfig: LogsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + }, + expectError: false, + }, + { + name: "Valid configuration 2", + config: &Config{ + Region: "EU", + MetricsConfig: MetricsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + LogsConfig: LogsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + }, + expectError: false, + }, + { + name: "Invalid region", + config: &Config{ + Region: "ASIA", + MetricsConfig: MetricsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + LogsConfig: LogsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + }, + expectError: true, + }, + { + name: "Invalid metrics AppToken length", + config: &Config{ + Region: "US", + MetricsConfig: MetricsConfig{ + AppToken: "short-token", + }, + LogsConfig: LogsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + }, + expectError: true, + }, + { + name: "Invalid logs AppToken length", + config: &Config{ + Region: "EU", + MetricsConfig: MetricsConfig{ + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, + LogsConfig: LogsConfig{ + AppToken: "short-token", + }, + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.expectError { + assert.Error(t, err, "Expected an error for invalid configuration") + } else { + assert.NoError(t, err, "Expected no error for valid configuration") + } + }) + } +} diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 7521a9da0e7d..672ca017d8cf 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -15,6 +15,7 @@ import ( "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "github.com/influxdata/influxdb-observability/common" @@ -30,11 +31,12 @@ func NewFactory() exporter.Factory { metadata.Type, createDefaultConfig, exporter.WithMetrics(createMetricsExporter, metadata.MetricsStability), + exporter.WithLogs(createLogsExporter, metadata.LogsStability), ) } func createDefaultConfig() component.Config { - return &Config{ + cfg := &Config{ ClientConfig: confighttp.ClientConfig{ Timeout: 5 * time.Second, Headers: map[string]configopaque.String{ @@ -42,15 +44,21 @@ func createDefaultConfig() component.Config { }, }, MetricsConfig: MetricsConfig{ + MetricsEndpoint: "https://spm-receiver.sematext.com", MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), AppToken: appToken, QueueSettings: exporterhelper.NewDefaultQueueConfig(), PayloadMaxLines: 1_000, PayloadMaxBytes: 300_000, }, + LogsConfig: LogsConfig{ + LogsEndpoint: "https://logsene-receiver.sematext.com", + AppToken: appToken, + }, BackOffConfig: configretry.NewDefaultBackOffConfig(), - Region: "custom", + Region: "us", } + return cfg } func createMetricsExporter( @@ -69,3 +77,20 @@ func createMetricsExporter( }, ) } + +func createLogsExporter( + ctx context.Context, + set exporter.Settings, + config component.Config, +) (exporter.Logs, error) { + cfg := config.(*Config) + + return exporterhelper.NewLogs( + ctx, + set, + cfg, + func(_ context.Context, _ plog.Logs) error { + return nil + }, + ) +} diff --git a/exporter/sematextexporter/generated_component_test.go b/exporter/sematextexporter/generated_component_test.go index f11f1cb2b58e..7445590fe974 100644 --- a/exporter/sematextexporter/generated_component_test.go +++ b/exporter/sematextexporter/generated_component_test.go @@ -35,6 +35,13 @@ func TestComponentLifecycle(t *testing.T) { createFn func(ctx context.Context, set exporter.Settings, cfg component.Config) (component.Component, error) }{ + { + name: "logs", + createFn: func(ctx context.Context, set exporter.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateLogs(ctx, set, cfg) + }, + }, + { name: "metrics", createFn: func(ctx context.Context, set exporter.Settings, cfg component.Config) (component.Component, error) { diff --git a/exporter/sematextexporter/internal/metadata/generated_status.go b/exporter/sematextexporter/internal/metadata/generated_status.go index e221a6ab9c5d..49e2d6d1a7ef 100644 --- a/exporter/sematextexporter/internal/metadata/generated_status.go +++ b/exporter/sematextexporter/internal/metadata/generated_status.go @@ -13,4 +13,5 @@ var ( const ( MetricsStability = component.StabilityLevelDevelopment + LogsStability = component.StabilityLevelDevelopment ) diff --git a/exporter/sematextexporter/metadata.yaml b/exporter/sematextexporter/metadata.yaml index 3dee628c6c03..6e764db0ac65 100644 --- a/exporter/sematextexporter/metadata.yaml +++ b/exporter/sematextexporter/metadata.yaml @@ -3,10 +3,10 @@ type: sematext status: class: exporter stability: - development: [metrics] + development: [metrics,logs] distributions: [contrib] codeowners: - active: [Eromosele-SM] + active: [AkhigbeEromo] tests: expect_consumer_error: true diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index e07da20f2e78..53185ba0523e 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -1,7 +1,7 @@ sematext/default-config: sematext/override-config: timeout: 500ms - region: US + region: us retry_on_failure: enabled: true initial_interval: 1s @@ -15,3 +15,5 @@ sematext/override-config: queue_size: 10 payload_max_lines: 72 payload_max_bytes: 27 + logs: + app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \ No newline at end of file