diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7fd42771c56ca8..9b48736e6dfe43 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -325,7 +325,8 @@ /comp/haagent @DataDog/ndm-core /comp/languagedetection/client @DataDog/container-platform /comp/rdnsquerier @DataDog/ndm-integrations -/comp/serializer/compression @DataDog/agent-metrics-logs +/comp/serializer/logscompression @DataDog/agent-processing-and-routing +/comp/serializer/metricscompression @DataDog/agent-processing-and-routing /comp/snmpscan @DataDog/ndm-core # END COMPONENTS diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 80f4b199117efa..47af68299c343c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -874,7 +874,7 @@ workflow: - comp/dogstatsd/**/* - comp/forwarder/**/* - comp/logs/**/* - - comp/serializer/compression/**/* + - comp/serializer/**/* - pkg/aggregator/**/* - pkg/collector/**/* - pkg/commonchecks/**/* diff --git a/cmd/agent/subcommands/diagnose/command.go b/cmd/agent/subcommands/diagnose/command.go index 3654ab5be676b4..f0ae880b9858b9 100644 --- a/cmd/agent/subcommands/diagnose/command.go +++ b/cmd/agent/subcommands/diagnose/command.go @@ -29,7 +29,8 @@ import ( workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompressorfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompressorfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/diagnose" @@ -104,9 +105,10 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { fx.Supply(option.None[collector.Component]()), dualTaggerfx.Module(common.DualTaggerParams()), autodiscoveryimpl.Module(), - compressionfx.Module(), diagnosesendermanagerimpl.Module(), haagentfx.Module(), + logscompressorfx.Module(), + metricscompressorfx.Module(), ) }, } diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index 2f137d5d28fafc..75fa34136feacb 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -51,7 +51,8 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost/inventoryhostimpl" "github.com/DataDog/datadog-agent/comp/metadata/inventoryotel/inventoryotelimpl" "github.com/DataDog/datadog-agent/comp/metadata/resources/resourcesimpl" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompressorfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompressorfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/settings" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -131,7 +132,6 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { localTaggerfx.Module(tagger.Params{}), autodiscoveryimpl.Module(), fx.Supply(option.None[collector.Component]()), - compressionfx.Module(), diagnosesendermanagerimpl.Module(), // We need inventoryagent to fill the status page generated by the flare. inventoryagentimpl.Module(), @@ -147,6 +147,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), core.Bundle(), haagentfx.Module(), + logscompressorfx.Module(), + metricscompressorfx.Module(), ) }, } diff --git a/cmd/agent/subcommands/jmx/command.go b/cmd/agent/subcommands/jmx/command.go index c969a329663c7a..be114f3345cbae 100644 --- a/cmd/agent/subcommands/jmx/command.go +++ b/cmd/agent/subcommands/jmx/command.go @@ -54,7 +54,8 @@ import ( integrations "github.com/DataDog/datadog-agent/comp/logs/integrations/def" "github.com/DataDog/datadog-agent/comp/remote-config/rcservice" "github.com/DataDog/datadog-agent/comp/remote-config/rcservicemrf" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/cli/standalone" pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" @@ -129,7 +130,6 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { fx.Supply(cliParams), fx.Supply(params), core.Bundle(), - compressionfx.Module(), diagnosesendermanagerimpl.Module(), fx.Supply(func(diagnoseSenderManager diagnosesendermanager.Component) (sender.SenderManager, error) { return diagnoseSenderManager.LazyGetSenderManager() @@ -168,6 +168,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), fx.Provide(func() remoteagentregistry.Component { return nil }), haagentfx.Module(), + logscompression.Module(), + metricscompression.Module(), ) } diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 0cab289f949f97..f27345b6668a00 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -32,7 +32,6 @@ import ( agenttelemetry "github.com/DataDog/datadog-agent/comp/core/agenttelemetry/def" agenttelemetryfx "github.com/DataDog/datadog-agent/comp/core/agenttelemetry/fx" haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" // checks implemented as components @@ -124,6 +123,7 @@ import ( "github.com/DataDog/datadog-agent/comp/remote-config/rcservice/rcserviceimpl" "github.com/DataDog/datadog-agent/comp/remote-config/rcservicemrf/rcservicemrfimpl" "github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl" + metricscompressorfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/comp/snmptraps" snmptrapsServer "github.com/DataDog/datadog-agent/comp/snmptraps/server" traceagentStatusImpl "github.com/DataDog/datadog-agent/comp/trace/status/statusimpl" @@ -380,7 +380,6 @@ func getSharedFxOption() fx.Option { authtokenimpl.Module(), apiimpl.Module(), commonendpoints.Module(), - compressionfx.Module(), demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams(demultiplexerimpl.WithDogstatsdNoAggregationPipelineConfig())), demultiplexerendpointfx.Module(), dogstatsd.Bundle(dogstatsdServer.Params{Serverless: false}), @@ -474,6 +473,7 @@ func getSharedFxOption() fx.Option { networkpath.Bundle(), remoteagentregistryfx.Module(), haagentfx.Module(), + metricscompressorfx.Module(), ) } diff --git a/cmd/agent/subcommands/snmp/command.go b/cmd/agent/subcommands/snmp/command.go index f10751fc3921f8..d1b431feb3b7d0 100644 --- a/cmd/agent/subcommands/snmp/command.go +++ b/cmd/agent/subcommands/snmp/command.go @@ -23,7 +23,8 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl" "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" snmpscan "github.com/DataDog/datadog-agent/comp/snmpscan/def" snmpscanfx "github.com/DataDog/datadog-agent/comp/snmpscan/fx" "github.com/DataDog/datadog-agent/pkg/networkdevice/metadata" @@ -98,10 +99,11 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithFeatures(defaultforwarder.CoreFeatures))), orchestratorimpl.Module(orchestratorimpl.NewDefaultParams()), eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()), - compressionfx.Module(), nooptagger.Module(), eventplatformreceiverimpl.Module(), haagentfx.Module(), + metricscompression.Module(), + logscompression.Module(), ) if err != nil { var ue configErr @@ -162,10 +164,11 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithFeatures(defaultforwarder.CoreFeatures))), eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()), eventplatformreceiverimpl.Module(), - compressionfx.Module(), nooptagger.Module(), snmpscanfx.Module(), haagentfx.Module(), + metricscompression.Module(), + logscompression.Module(), ) if err != nil { var ue configErr diff --git a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go index 1ada82ab7fa2c6..d565da2063fc19 100644 --- a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go +++ b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go @@ -51,7 +51,8 @@ import ( orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx" integrations "github.com/DataDog/datadog-agent/comp/logs/integrations/def" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/clusteragent" "github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks" pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" @@ -86,7 +87,6 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), core.Bundle(), forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithResolvers())), - compressionfx.Module(), demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()), orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDisabledParams()), eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()), @@ -128,6 +128,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { proccontainers.InitSharedContainerProvider(wmeta, tagger) }), haagentfx.Module(), + logscompressionfx.Module(), + metricscompressionfx.Module(), ) }, } diff --git a/cmd/cluster-agent/subcommands/diagnose/command.go b/cmd/cluster-agent/subcommands/diagnose/command.go index 5e2521b60f8c37..a871858e4edcb2 100644 --- a/cmd/cluster-agent/subcommands/diagnose/command.go +++ b/cmd/cluster-agent/subcommands/diagnose/command.go @@ -18,7 +18,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/secrets" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" "github.com/DataDog/datadog-agent/pkg/diagnose" "github.com/DataDog/datadog-agent/pkg/diagnose/diagnosis" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -38,7 +37,6 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { LogParams: log.ForOneShot(command.LoggerName, "off", true), // no need to show regular logs }), core.Bundle(), - compressionfx.Module(), ) }, } diff --git a/cmd/cluster-agent/subcommands/start/command.go b/cmd/cluster-agent/subcommands/start/command.go index 756ab5c646fcbf..938e11c3d4d4f5 100644 --- a/cmd/cluster-agent/subcommands/start/command.go +++ b/cmd/cluster-agent/subcommands/start/command.go @@ -59,7 +59,9 @@ import ( rccomp "github.com/DataDog/datadog-agent/comp/remote-config/rcservice" "github.com/DataDog/datadog-agent/comp/remote-config/rcservice/rcserviceimpl" "github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/clusteragent" admissionpkg "github.com/DataDog/datadog-agent/pkg/clusteragent/admission" admissionpatch "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/patch" @@ -137,7 +139,6 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), core.Bundle(), forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithResolvers(), defaultforwarder.WithDisableAPIKeyChecking())), - compressionfx.Module(), demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()), orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDefaultParams()), eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()), @@ -204,6 +205,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { proccontainers.InitSharedContainerProvider(wmeta, tagger) }), haagentfx.Module(), + logscompressionfx.Module(), + metricscompressionfx.Module(), ) }, } @@ -226,6 +229,7 @@ func start(log log.Component, logReceiver option.Option[integrations.Component], _ healthprobe.Component, settings settings.Component, + compression logscompression.Component, datadogConfig config.Component, ) error { stopCh := make(chan struct{}) @@ -441,7 +445,7 @@ func start(log log.Component, go func() { defer wg.Done() - if err := runCompliance(mainCtx, demultiplexer, wmeta, apiCl, le.IsLeader); err != nil { + if err := runCompliance(mainCtx, demultiplexer, wmeta, apiCl, compression, le.IsLeader); err != nil { pkglog.Errorf("Error while running compliance agent: %v", err) } }() diff --git a/cmd/cluster-agent/subcommands/start/compliance.go b/cmd/cluster-agent/subcommands/start/compliance.go index 780e390331cee6..74e4a1f78b97b5 100644 --- a/cmd/cluster-agent/subcommands/start/compliance.go +++ b/cmd/cluster-agent/subcommands/start/compliance.go @@ -16,6 +16,7 @@ import ( workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/compliance" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -31,9 +32,9 @@ const ( intakeTrackType = "compliance" ) -func runCompliance(ctx context.Context, senderManager sender.SenderManager, wmeta workloadmeta.Component, apiCl *apiserver.APIClient, isLeader func() bool) error { +func runCompliance(ctx context.Context, senderManager sender.SenderManager, wmeta workloadmeta.Component, apiCl *apiserver.APIClient, compression logscompression.Component, isLeader func() bool) error { stopper := startstop.NewSerialStopper() - if err := startCompliance(senderManager, wmeta, stopper, apiCl, isLeader); err != nil { + if err := startCompliance(senderManager, wmeta, stopper, apiCl, isLeader, compression); err != nil { return err } @@ -72,7 +73,7 @@ func newLogContextCompliance() (*config.Endpoints, *client.DestinationsContext, return newLogContext(logsConfigComplianceKeys, "cspm-intake.") } -func startCompliance(senderManager sender.SenderManager, wmeta workloadmeta.Component, stopper startstop.Stopper, apiCl *apiserver.APIClient, isLeader func() bool) error { +func startCompliance(senderManager sender.SenderManager, wmeta workloadmeta.Component, stopper startstop.Stopper, apiCl *apiserver.APIClient, isLeader func() bool, compression logscompression.Component) error { endpoints, ctx, err := newLogContextCompliance() if err != nil { log.Error(err) @@ -87,7 +88,7 @@ func startCompliance(senderManager sender.SenderManager, wmeta workloadmeta.Comp return err } - reporter := compliance.NewLogReporter(hname, "compliance-agent", "compliance", endpoints, ctx) + reporter := compliance.NewLogReporter(hname, "compliance-agent", "compliance", endpoints, ctx, compression) statsdClient, err := simpleTelemetrySenderFromSenderManager(senderManager) if err != nil { return err diff --git a/cmd/dogstatsd/subcommands/start/command.go b/cmd/dogstatsd/subcommands/start/command.go index 85f719e5e86c35..8071cbd3cfe0d1 100644 --- a/cmd/dogstatsd/subcommands/start/command.go +++ b/cmd/dogstatsd/subcommands/start/command.go @@ -54,7 +54,8 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/resources/resourcesimpl" "github.com/DataDog/datadog-agent/comp/metadata/runner" metadatarunnerimpl "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util/coredump" @@ -141,7 +142,8 @@ func RunDogstatsdFct(cliParams *CLIParams, defaultConfPath string, defaultLogFil AgentType: workloadmeta.NodeAgent, InitHelper: common.GetWorkloadmetaInit(), }), - compressionfx.Module(), + metricscompressionfx.Module(), + logscompressionfx.Module(), demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams( demultiplexerimpl.WithContinueOnMissingHostname(), demultiplexerimpl.WithDogstatsdNoAggregationPipelineConfig(), diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index 87e8b1b74349a7..347e38f312ab03 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -32,7 +32,6 @@ import ( remoteTaggerFx "github.com/DataDog/datadog-agent/comp/core/tagger/fx-remote" taggerTypes "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" - workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" @@ -48,7 +47,9 @@ import ( "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline/logsagentpipelineimpl" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient" - compressionfxzlib "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-zlib" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + metricscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-otel" traceagentfx "github.com/DataDog/datadog-agent/comp/trace/agent/fx" traceagentcomp "github.com/DataDog/datadog-agent/comp/trace/agent/impl" gzipfx "github.com/DataDog/datadog-agent/comp/trace/compression/fx-gzip" @@ -58,6 +59,7 @@ import ( pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/trace/telemetry" + "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/option" @@ -160,8 +162,13 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) }), logsagentpipelineimpl.Module(), - // We directly select fxzlib - compressionfxzlib.Module(), + logscompressionfx.Module(), + metricscompressionfx.Module(), + // For FX to provide the compression.Compressor interface (used by serializer.NewSerializer) + // implemented by the metricsCompression.Component + fx.Provide(func(c metricscompression.Component) compression.Compressor { + return c + }), fx.Provide(serializer.NewSerializer), // For FX to provide the serializer.MetricSerializer from the serializer.Serializer fx.Provide(func(s *serializer.Serializer) serializer.MetricSerializer { diff --git a/cmd/security-agent/main_windows.go b/cmd/security-agent/main_windows.go index 25d85cdf06b195..c926b1850e2a47 100644 --- a/cmd/security-agent/main_windows.go +++ b/cmd/security-agent/main_windows.go @@ -45,6 +45,8 @@ import ( "github.com/DataDog/datadog-agent/comp/dogstatsd" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" commonsettings "github.com/DataDog/datadog-agent/pkg/config/settings" "github.com/DataDog/datadog-agent/pkg/collector/python" @@ -124,7 +126,7 @@ func (s *service) Run(svcctx context.Context) error { workloadmetafx.Module(workloadmeta.Params{ AgentType: workloadmeta.Remote, }), - fx.Provide(func(log log.Component, config config.Component, statsd statsd.Component, wmeta workloadmeta.Component) (status.InformationProvider, *agent.RuntimeSecurityAgent, error) { + fx.Provide(func(log log.Component, config config.Component, statsd statsd.Component, wmeta workloadmeta.Component, compression logscompression.Component) (status.InformationProvider, *agent.RuntimeSecurityAgent, error) { stopper := startstop.NewSerialStopper() statsdClient, err := statsd.CreateForHostPort(setup.GetBindHost(config), config.GetInt("dogstatsd_port")) @@ -138,7 +140,7 @@ func (s *service) Run(svcctx context.Context) error { return status.NewInformationProvider(nil), nil, err } - runtimeAgent, err := runtime.StartRuntimeSecurity(log, config, hostnameDetected, stopper, statsdClient, wmeta) + runtimeAgent, err := runtime.StartRuntimeSecurity(log, config, hostnameDetected, stopper, statsdClient, wmeta, compression) if err != nil { return status.NewInformationProvider(nil), nil, err } @@ -175,6 +177,7 @@ func (s *service) Run(svcctx context.Context) error { } }), settingsimpl.Module(), + logscompressionfx.Module(), ) return err diff --git a/cmd/security-agent/subcommands/check/command.go b/cmd/security-agent/subcommands/check/command.go index 2acc79141850f5..78cd3b1cbefefc 100644 --- a/cmd/security-agent/subcommands/check/command.go +++ b/cmd/security-agent/subcommands/check/command.go @@ -31,6 +31,8 @@ import ( "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/sysprobeconfigimpl" "github.com/DataDog/datadog-agent/comp/dogstatsd" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" "github.com/DataDog/datadog-agent/pkg/compliance" "github.com/DataDog/datadog-agent/pkg/compliance/k8sconfig" "github.com/DataDog/datadog-agent/pkg/security/common" @@ -93,6 +95,7 @@ func commandsWrapped(bundleParamsFactory func() core.BundleParams) []*cobra.Comm fx.Supply(checkArgs), fx.Supply(bundleParams), core.Bundle(), + logscompressionfx.Module(), dogstatsd.ClientBundle, ) }, @@ -109,7 +112,7 @@ func commandsWrapped(bundleParamsFactory func() core.BundleParams) []*cobra.Comm } // RunCheck runs a check -func RunCheck(log log.Component, config config.Component, _ secrets.Component, statsdComp statsd.Component, checkArgs *CliParams) error { +func RunCheck(log log.Component, config config.Component, _ secrets.Component, statsdComp statsd.Component, checkArgs *CliParams, compression logscompression.Component) error { hname, err := hostname.Get(context.TODO()) if err != nil { return err @@ -218,7 +221,7 @@ func RunCheck(log log.Component, config config.Component, _ secrets.Component, s } } if checkArgs.report { - if err := reportComplianceEvents(log, events); err != nil { + if err := reportComplianceEvents(log, events, compression); err != nil { log.Error(err) return err } @@ -241,7 +244,7 @@ func dumpComplianceEvents(reportFile string, events []*compliance.CheckEvent) er return nil } -func reportComplianceEvents(log log.Component, events []*compliance.CheckEvent) error { +func reportComplianceEvents(log log.Component, events []*compliance.CheckEvent, compression logscompression.Component) error { hostnameDetected, err := hostnameutils.GetHostnameWithContextAndFallback(context.Background()) if err != nil { return log.Errorf("Error while getting hostname, exiting: %v", err) @@ -250,7 +253,7 @@ func reportComplianceEvents(log log.Component, events []*compliance.CheckEvent) if err != nil { return fmt.Errorf("reporter: could not reate log context for compliance: %w", err) } - reporter := compliance.NewLogReporter(hostnameDetected, "compliance-agent", "compliance", endpoints, context) + reporter := compliance.NewLogReporter(hostnameDetected, "compliance-agent", "compliance", endpoints, context, compression) defer reporter.Stop() for _, event := range events { reporter.ReportEvent(event) diff --git a/cmd/security-agent/subcommands/compliance/command.go b/cmd/security-agent/subcommands/compliance/command.go index af8cdf96e699b6..c80614d164ad2e 100644 --- a/cmd/security-agent/subcommands/compliance/command.go +++ b/cmd/security-agent/subcommands/compliance/command.go @@ -24,6 +24,8 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/secrets" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" "github.com/DataDog/datadog-agent/pkg/compliance" "github.com/DataDog/datadog-agent/pkg/compliance/aptconfig" "github.com/DataDog/datadog-agent/pkg/compliance/dbconfig" @@ -82,6 +84,7 @@ func complianceLoadCommand(globalParams *command.GlobalParams) *cobra.Command { LogParams: log.ForOneShot(command.LoggerName, "info", true), }), core.Bundle(), + logscompressionfx.Module(), ) }, } @@ -160,6 +163,7 @@ func complianceEventCommand(globalParams *command.GlobalParams) *cobra.Command { LogParams: log.ForOneShot(command.LoggerName, "info", true), }), core.Bundle(), + logscompressionfx.Module(), ) }, Hidden: true, @@ -176,7 +180,7 @@ func complianceEventCommand(globalParams *command.GlobalParams) *cobra.Command { return eventCmd } -func eventRun(log log.Component, eventArgs *eventCliParams) error { +func eventRun(log log.Component, eventArgs *eventCliParams, compression compression.Component) error { hostnameDetected, err := hostnameutils.GetHostnameWithContextAndFallback(context.Background()) if err != nil { return log.Errorf("Error while getting hostname, exiting: %v", err) @@ -187,7 +191,7 @@ func eventRun(log log.Component, eventArgs *eventCliParams) error { return err } - reporter := compliance.NewLogReporter(hostnameDetected, eventArgs.sourceName, eventArgs.sourceType, endpoints, dstContext) + reporter := compliance.NewLogReporter(hostnameDetected, eventArgs.sourceName, eventArgs.sourceType, endpoints, dstContext, compression) defer reporter.Stop() eventData := make(map[string]interface{}) diff --git a/cmd/security-agent/subcommands/compliance/compliance.go b/cmd/security-agent/subcommands/compliance/compliance.go index f37f90613d3c43..984e1808ed7f89 100644 --- a/cmd/security-agent/subcommands/compliance/compliance.go +++ b/cmd/security-agent/subcommands/compliance/compliance.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/comp/dogstatsd/constants" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/compliance" "github.com/DataDog/datadog-agent/pkg/security/common" "github.com/DataDog/datadog-agent/pkg/security/telemetry" @@ -28,7 +29,16 @@ import ( // StartCompliance runs the compliance sub-agent running compliance benchmarks // and checks. -func StartCompliance(log log.Component, config config.Component, sysprobeconfig sysprobeconfig.Component, hostname string, stopper startstop.Stopper, statsdClient ddgostatsd.ClientInterface, wmeta workloadmeta.Component) (*compliance.Agent, error) { +func StartCompliance(log log.Component, + config config.Component, + sysprobeconfig sysprobeconfig.Component, + hostname string, + stopper startstop.Stopper, + statsdClient ddgostatsd.ClientInterface, + wmeta workloadmeta.Component, + compression compression.Component, +) (*compliance.Agent, error) { + enabled := config.GetBool("compliance_config.enabled") configDir := config.GetString("compliance_config.dir") metricsEnabled := config.GetBool("compliance_config.metrics.enabled") @@ -67,7 +77,7 @@ func StartCompliance(log log.Component, config config.Component, sysprobeconfig enabledConfigurationsExporters = append(enabledConfigurationsExporters, compliance.DBExporter) } - reporter := compliance.NewLogReporter(hostname, "compliance-agent", "compliance", endpoints, context) + reporter := compliance.NewLogReporter(hostname, "compliance-agent", "compliance", endpoints, context, compression) telemetrySender := telemetry.NewSimpleTelemetrySenderFromStatsd(statsdClient) agent := compliance.NewAgent(telemetrySender, wmeta, compliance.AgentOptions{ diff --git a/cmd/security-agent/subcommands/runtime/command.go b/cmd/security-agent/subcommands/runtime/command.go index c0b5e2c0775ff4..e95c92c8a325d6 100644 --- a/cmd/security-agent/subcommands/runtime/command.go +++ b/cmd/security-agent/subcommands/runtime/command.go @@ -33,6 +33,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/secrets" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" secagent "github.com/DataDog/datadog-agent/pkg/security/agent" "github.com/DataDog/datadog-agent/pkg/security/common" @@ -709,7 +710,7 @@ func reloadRuntimePolicies(_ log.Component, _ config.Component, _ secrets.Compon } // StartRuntimeSecurity starts runtime security -func StartRuntimeSecurity(log log.Component, config config.Component, hostname string, stopper startstop.Stopper, statsdClient ddgostatsd.ClientInterface, wmeta workloadmeta.Component) (*secagent.RuntimeSecurityAgent, error) { +func StartRuntimeSecurity(log log.Component, config config.Component, hostname string, stopper startstop.Stopper, statsdClient ddgostatsd.ClientInterface, wmeta workloadmeta.Component, compression compression.Component) (*secagent.RuntimeSecurityAgent, error) { enabled := config.GetBool("runtime_security_config.enabled") if !enabled { log.Info("Datadog runtime security agent disabled by config") @@ -733,7 +734,7 @@ func StartRuntimeSecurity(log log.Component, config config.Component, hostname s } stopper.Add(ctx) - reporter, err := reporter.NewCWSReporter(hostname, stopper, endpoints, ctx) + reporter, err := reporter.NewCWSReporter(hostname, stopper, endpoints, ctx, compression) if err != nil { return nil, err } diff --git a/cmd/security-agent/subcommands/runtime/command_unsupported.go b/cmd/security-agent/subcommands/runtime/command_unsupported.go index 83f7a482737880..793136ca42ff17 100644 --- a/cmd/security-agent/subcommands/runtime/command_unsupported.go +++ b/cmd/security-agent/subcommands/runtime/command_unsupported.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" secagent "github.com/DataDog/datadog-agent/pkg/security/agent" "github.com/DataDog/datadog-agent/pkg/util/startstop" "github.com/DataDog/datadog-go/v5/statsd" @@ -28,7 +29,7 @@ func Commands(*command.GlobalParams) []*cobra.Command { } // StartRuntimeSecurity starts runtime security -func StartRuntimeSecurity(log log.Component, config config.Component, _ string, _ startstop.Stopper, _ statsd.ClientInterface, _ workloadmeta.Component) (*secagent.RuntimeSecurityAgent, error) { +func StartRuntimeSecurity(log log.Component, config config.Component, _ string, _ startstop.Stopper, _ statsd.ClientInterface, _ workloadmeta.Component, _ compression.Component) (*secagent.RuntimeSecurityAgent, error) { enabled := config.GetBool("runtime_security_config.enabled") if !enabled { log.Info("Datadog runtime security agent disabled by config") diff --git a/cmd/security-agent/subcommands/start/command.go b/cmd/security-agent/subcommands/start/command.go index 708bd2f0102227..182c637b6778e4 100644 --- a/cmd/security-agent/subcommands/start/command.go +++ b/cmd/security-agent/subcommands/start/command.go @@ -54,6 +54,8 @@ import ( "github.com/DataDog/datadog-agent/comp/dogstatsd" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/collector/python" pkgCompliance "github.com/DataDog/datadog-agent/pkg/compliance" @@ -123,13 +125,13 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { fx.Provide(func(config config.Component, statsd statsd.Component) (ddgostatsd.ClientInterface, error) { return statsd.CreateForHostPort(pkgconfigsetup.GetBindHost(config), config.GetInt("dogstatsd_port")) }), - fx.Provide(func(stopper startstop.Stopper, log log.Component, config config.Component, statsdClient ddgostatsd.ClientInterface, wmeta workloadmeta.Component) (status.InformationProvider, *agent.RuntimeSecurityAgent, error) { + fx.Provide(func(stopper startstop.Stopper, log log.Component, config config.Component, statsdClient ddgostatsd.ClientInterface, wmeta workloadmeta.Component, compression logscompression.Component) (status.InformationProvider, *agent.RuntimeSecurityAgent, error) { hostnameDetected, err := hostnameutils.GetHostnameWithContextAndFallback(context.TODO()) if err != nil { return status.NewInformationProvider(nil), nil, err } - runtimeAgent, err := runtime.StartRuntimeSecurity(log, config, hostnameDetected, stopper, statsdClient, wmeta) + runtimeAgent, err := runtime.StartRuntimeSecurity(log, config, hostnameDetected, stopper, statsdClient, wmeta, compression) if err != nil { return status.NewInformationProvider(nil), nil, err } @@ -141,14 +143,14 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { // TODO - components: Do not remove runtimeAgent ref until "github.com/DataDog/datadog-agent/pkg/security/agent" is a component so they're not GCed return status.NewInformationProvider(runtimeAgent.StatusProvider()), runtimeAgent, nil }), - fx.Provide(func(stopper startstop.Stopper, log log.Component, config config.Component, statsdClient ddgostatsd.ClientInterface, sysprobeconfig sysprobeconfig.Component, wmeta workloadmeta.Component) (status.InformationProvider, *pkgCompliance.Agent, error) { + fx.Provide(func(stopper startstop.Stopper, log log.Component, config config.Component, statsdClient ddgostatsd.ClientInterface, sysprobeconfig sysprobeconfig.Component, wmeta workloadmeta.Component, compression logscompression.Component) (status.InformationProvider, *pkgCompliance.Agent, error) { hostnameDetected, err := hostnameutils.GetHostnameWithContextAndFallback(context.TODO()) if err != nil { return status.NewInformationProvider(nil), nil, err } // start compliance security agent - complianceAgent, err := compliance.StartCompliance(log, config, sysprobeconfig, hostnameDetected, stopper, statsdClient, wmeta) + complianceAgent, err := compliance.StartCompliance(log, config, sysprobeconfig, hostnameDetected, stopper, statsdClient, wmeta, compression) if err != nil { return status.NewInformationProvider(nil), nil, err } @@ -184,6 +186,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { } }), settingsimpl.Module(), + logscompressionfx.Module(), ) }, } diff --git a/cmd/serverless-init/log/log.go b/cmd/serverless-init/log/log.go index 3065af97ce498a..77e69704ff8979 100644 --- a/cmd/serverless-init/log/log.go +++ b/cmd/serverless-init/log/log.go @@ -16,6 +16,7 @@ import ( tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent" logConfig "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/logs/sources" serverlessLogs "github.com/DataDog/datadog-agent/pkg/serverless/logs" serverlessTag "github.com/DataDog/datadog-agent/pkg/serverless/tags" @@ -53,8 +54,8 @@ func CreateConfig(origin string) *Config { } // SetupLogAgent creates the log agent and sets the base tags -func SetupLogAgent(conf *Config, tags map[string]string, tagger tagger.Component) logsAgent.ServerlessLogsAgent { - logsAgent, _ := serverlessLogs.SetupLogAgent(conf.Channel, sourceName, conf.source, tagger) +func SetupLogAgent(conf *Config, tags map[string]string, tagger tagger.Component, compression logscompression.Component) logsAgent.ServerlessLogsAgent { + logsAgent, _ := serverlessLogs.SetupLogAgent(conf.Channel, sourceName, conf.source, tagger, compression) tagsArray := serverlessTag.MapToArray(tags) diff --git a/cmd/serverless-init/main.go b/cmd/serverless-init/main.go index bb60255362ae11..2fe4a799017da5 100644 --- a/cmd/serverless-init/main.go +++ b/cmd/serverless-init/main.go @@ -31,6 +31,8 @@ import ( tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" localTaggerFx "github.com/DataDog/datadog-agent/comp/core/tagger/fx" nooptelemetry "github.com/DataDog/datadog-agent/comp/core/telemetry/noopsimpl" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" @@ -80,6 +82,7 @@ func main() { workloadmetafx.Module(workloadmeta.NewParams()), fx.Supply(coreconfig.NewParams("", coreconfig.WithConfigMissingOK(true))), coreconfig.Module(), + logscompressionfx.Module(), fx.Supply(secrets.NewEnabledParams()), secretsimpl.Module(), fx.Provide(func(secrets secrets.Component) option.Option[secrets.Component] { return option.New(secrets) }), @@ -97,8 +100,8 @@ func main() { } // removing these unused dependencies will cause silent crash due to fx framework -func run(_ secrets.Component, _ autodiscovery.Component, _ healthprobeDef.Component, tagger tagger.Component) error { - cloudService, logConfig, traceAgent, metricAgent, logsAgent := setup(modeConf, tagger) +func run(_ secrets.Component, _ autodiscovery.Component, _ healthprobeDef.Component, tagger tagger.Component, compression logscompression.Component) error { + cloudService, logConfig, traceAgent, metricAgent, logsAgent := setup(modeConf, tagger, compression) err := modeConf.Runner(logConfig) @@ -108,7 +111,7 @@ func run(_ secrets.Component, _ autodiscovery.Component, _ healthprobeDef.Compon return err } -func setup(_ mode.Conf, tagger tagger.Component) (cloudservice.CloudService, *serverlessInitLog.Config, trace.ServerlessTraceAgent, *metrics.ServerlessMetricAgent, logsAgent.ServerlessLogsAgent) { +func setup(_ mode.Conf, tagger tagger.Component, compression logscompression.Component) (cloudservice.CloudService, *serverlessInitLog.Config, trace.ServerlessTraceAgent, *metrics.ServerlessMetricAgent, logsAgent.ServerlessLogsAgent) { tracelog.SetLogger(corelogger{}) // load proxy settings @@ -139,7 +142,7 @@ func setup(_ mode.Conf, tagger tagger.Component) (cloudservice.CloudService, *se if err != nil { log.Debugf("Error loading config: %v\n", err) } - logsAgent := serverlessInitLog.SetupLogAgent(agentLogConfig, tags, tagger) + logsAgent := serverlessInitLog.SetupLogAgent(agentLogConfig, tags, tagger, compression) traceAgent := setupTraceAgent(tags, tagger) diff --git a/cmd/serverless-init/main_test.go b/cmd/serverless-init/main_test.go index 2b99ef0d0be46f..ca3dbb076cdad1 100644 --- a/cmd/serverless-init/main_test.go +++ b/cmd/serverless-init/main_test.go @@ -19,6 +19,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/serverless-init/mode" "github.com/DataDog/datadog-agent/comp/core/tagger/mock" "github.com/DataDog/datadog-agent/comp/logs/agent/agentimpl" + compressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" configmock "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/serverless/logs" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -29,6 +30,7 @@ func TestTagsSetup(t *testing.T) { t.Skip() fakeTagger := mock.SetupFakeTagger(t) + fakeCompression := compressionmock.NewMockCompressor() configmock.New(t) @@ -41,7 +43,7 @@ func TestTagsSetup(t *testing.T) { allTags := append(ddTags, ddExtraTags...) - _, _, traceAgent, metricAgent, _ := setup(mode.Conf{}, fakeTagger) + _, _, traceAgent, metricAgent, _ := setup(mode.Conf{}, fakeTagger, fakeCompression) defer traceAgent.Stop() defer metricAgent.Stop() assert.Subset(t, metricAgent.GetExtraTags(), allTags) diff --git a/cmd/serverless-init/metric/metric_test.go b/cmd/serverless-init/metric/metric_test.go index 6c9badf1beac59..4aafa0cb19481b 100644 --- a/cmd/serverless-init/metric/metric_test.go +++ b/cmd/serverless-init/metric/metric_test.go @@ -17,7 +17,8 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -76,5 +77,5 @@ func TestNilDemuxDoesNotPanic(t *testing.T) { } func createDemultiplexer(t *testing.T) demultiplexer.FakeSamplerMock { - return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), compressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) + return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), logscompression.MockModule(), metricscompression.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) } diff --git a/cmd/serverless/main.go b/cmd/serverless/main.go index a679ceef10181b..e8ab54a8b74dc0 100644 --- a/cmd/serverless/main.go +++ b/cmd/serverless/main.go @@ -19,6 +19,8 @@ import ( tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" taggernoop "github.com/DataDog/datadog-agent/comp/core/tagger/fx-noop" logConfig "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" "github.com/DataDog/datadog-agent/pkg/config/model" remoteconfig "github.com/DataDog/datadog-agent/pkg/config/remote/service" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -78,6 +80,7 @@ func main() { err := fxutil.OneShot( runAgent, taggernoop.Module(), + logscompressionfx.Module(), ) if err != nil { @@ -86,7 +89,8 @@ func main() { } } -func runAgent(tagger tagger.Component) { +func runAgent(tagger tagger.Component, compression logscompression.Component) { + startTime := time.Now() setupLambdaAgentOverrides() @@ -125,7 +129,7 @@ func runAgent(tagger tagger.Component) { go startTraceAgent(&wg, lambdaSpanChan, coldStartSpanId, serverlessDaemon, tagger, rcService) go startOtlpAgent(&wg, metricAgent, serverlessDaemon, tagger) - go startTelemetryCollection(&wg, serverlessID, logChannel, serverlessDaemon, tagger) + go startTelemetryCollection(&wg, serverlessID, logChannel, serverlessDaemon, tagger, compression) // start appsec appsecProxyProcessor := startAppSec(serverlessDaemon) @@ -298,7 +302,7 @@ func startAppSec(serverlessDaemon *daemon.Daemon) *httpsec.ProxyLifecycleProcess return appsecProxyProcessor } -func startTelemetryCollection(wg *sync.WaitGroup, serverlessID registration.ID, logChannel chan *logConfig.ChannelMessage, serverlessDaemon *daemon.Daemon, tagger tagger.Component) { +func startTelemetryCollection(wg *sync.WaitGroup, serverlessID registration.ID, logChannel chan *logConfig.ChannelMessage, serverlessDaemon *daemon.Daemon, tagger tagger.Component, compression logscompression.Component) { defer wg.Done() if os.Getenv(daemon.LocalTestEnvVar) == "true" || os.Getenv(daemon.LocalTestEnvVar) == "1" { log.Debug("Running in local test mode. Telemetry collection HTTP route won't be enabled") @@ -322,7 +326,7 @@ func startTelemetryCollection(wg *sync.WaitGroup, serverlessID registration.ID, if logRegistrationError != nil { log.Error("Can't subscribe to logs:", logRegistrationError) } else { - logsAgent, err := serverlessLogs.SetupLogAgent(logChannel, "AWS Logs", "lambda", tagger) + logsAgent, err := serverlessLogs.SetupLogAgent(logChannel, "AWS Logs", "lambda", tagger, compression) if err != nil { log.Errorf("Error setting up the logs agent: %s", err) } diff --git a/cmd/system-probe/api/module/common.go b/cmd/system-probe/api/module/common.go index cec9bf608de45b..8bbad735a27e82 100644 --- a/cmd/system-probe/api/module/common.go +++ b/cmd/system-probe/api/module/common.go @@ -12,6 +12,7 @@ import ( tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "go.uber.org/fx" ) @@ -30,7 +31,8 @@ type Module interface { type FactoryDependencies struct { fx.In - WMeta workloadmeta.Component - Tagger tagger.Component - Telemetry telemetry.Component + WMeta workloadmeta.Component + Tagger tagger.Component + Telemetry telemetry.Component + Compression logscompression.Component } diff --git a/cmd/system-probe/modules/eventmonitor.go b/cmd/system-probe/modules/eventmonitor.go index 41707ad513dec5..f0a9288f14f2af 100644 --- a/cmd/system-probe/modules/eventmonitor.go +++ b/cmd/system-probe/modules/eventmonitor.go @@ -53,7 +53,7 @@ func createEventMonitorModule(_ *sysconfigtypes.Config, deps module.FactoryDepen } if secconfig.RuntimeSecurity.IsRuntimeEnabled() { - cws, err := secmodule.NewCWSConsumer(evm, secconfig.RuntimeSecurity, deps.WMeta, secmoduleOpts) + cws, err := secmodule.NewCWSConsumer(evm, secconfig.RuntimeSecurity, deps.WMeta, secmoduleOpts, deps.Compression) if err != nil { return nil, err } diff --git a/cmd/systray/command/command.go b/cmd/systray/command/command.go index ccba66dc29ee9f..a7eed7024a2442 100644 --- a/cmd/systray/command/command.go +++ b/cmd/systray/command/command.go @@ -29,7 +29,8 @@ import ( workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent/inventoryagentimpl" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/comp/systray/systray" "github.com/DataDog/datadog-agent/comp/systray/systray/systrayimpl" "github.com/DataDog/datadog-agent/pkg/serializer" @@ -110,7 +111,8 @@ func MakeCommand() *cobra.Command { noopAutodiscover.Module(), fx.Supply(option.None[workloadmeta.Component]()), fx.Supply(option.None[collector.Component]()), - compressionfx.Module(), + logscompressionfx.Module(), + metricscompressionfx.Module(), diagnosesendermanagerimpl.Module(), nooptagger.Module(), authtokenimpl.Module(), diff --git a/comp/README.md b/comp/README.md index f011bc57fe71d1..e70d8c101ef337 100644 --- a/comp/README.md +++ b/comp/README.md @@ -615,11 +615,17 @@ Package client implements a component to send process metadata to the Cluster-Ag Package rdnsquerier provides the reverse DNS querier component. -### [comp/serializer/compression](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/serializer/compression) +### [comp/serializer/logscompression](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/serializer/logscompression) -*Datadog Team*: agent-metrics-logs +*Datadog Team*: agent-processing-and-routing + +Package logscompression provides the component for logs compression + +### [comp/serializer/metricscompression](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/serializer/metricscompression) + +*Datadog Team*: agent-processing-and-routing -Package compression provides a compression implementation based on the configuration or available build tags. +Package metricscompression provides the component for metrics compression ### [comp/snmpscan](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/snmpscan) diff --git a/comp/agent/bundle_test.go b/comp/agent/bundle_test.go index 8a1820206ee813..44618341d203b0 100644 --- a/comp/agent/bundle_test.go +++ b/comp/agent/bundle_test.go @@ -16,7 +16,6 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -24,7 +23,6 @@ func TestBundleDependencies(t *testing.T) { fxutil.TestBundle(t, Bundle(jmxloggerimpl.NewDefaultParams()), core.MockBundle(), - compressionmock.MockModule(), defaultforwarder.MockModule(), orchestratorimpl.MockModule(), eventplatformimpl.MockModule(), diff --git a/comp/aggregator/bundle_test.go b/comp/aggregator/bundle_test.go index 2a38d5a0ac6037..8b5b5c35929a58 100644 --- a/comp/aggregator/bundle_test.go +++ b/comp/aggregator/bundle_test.go @@ -15,18 +15,20 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) func TestBundleDependencies(t *testing.T) { fxutil.TestBundle(t, Bundle(demultiplexerimpl.Params{}), core.MockBundle(), - compressionmock.MockModule(), defaultforwarder.MockModule(), orchestratorForwarderImpl.MockModule(), eventplatformimpl.MockModule(), nooptagger.Module(), haagentmock.Module(), + logscompression.MockModule(), + metricscompression.MockModule(), ) } diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go index b6b1dd7d563106..6011623caae835 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go @@ -21,7 +21,7 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" orchestratorforwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator" haagent "github.com/DataDog/datadog-agent/comp/haagent/def" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/util/fxutil" diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_fake_sampler_mock.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_fake_sampler_mock.go index 53e3beb5f77d6e..a16b2ac7c4860c 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_fake_sampler_mock.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_fake_sampler_mock.go @@ -16,7 +16,8 @@ import ( demultiplexerComp "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/core/hostname" log "github.com/DataDog/datadog-agent/comp/core/log/def" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -33,10 +34,11 @@ func FakeSamplerMockModule() fxutil.Module { type fakeSamplerMockDependencies struct { fx.In - Lc fx.Lifecycle - Log log.Component - Hostname hostname.Component - Compressor compression.Component + Lc fx.Lifecycle + Log log.Component + Hostname hostname.Component + LogsCompression logscompression.Component + MetricsCompression metricscompression.Component } type fakeSamplerMock struct { @@ -56,7 +58,7 @@ func (f *fakeSamplerMock) Stop(flush bool) { } func newFakeSamplerMock(deps fakeSamplerMockDependencies) demultiplexerComp.FakeSamplerMock { - demux := initTestAgentDemultiplexerWithFlushInterval(deps.Log, deps.Hostname, deps.Compressor, time.Hour) + demux := initTestAgentDemultiplexerWithFlushInterval(deps.Log, deps.Hostname, deps.LogsCompression, deps.MetricsCompression, time.Hour) mock := &fakeSamplerMock{ TestAgentDemultiplexer: demux, } diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go index 942c10771a63a9..8f609ac9f7d73c 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer_mock.go @@ -16,7 +16,8 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -73,11 +74,12 @@ func newMock(deps mockDependencies) MockProvides { opts.DontStartForwarders = true aggDeps := aggregator.TestDeps{ - Log: deps.Log, - Hostname: deps.Hostname, - SharedForwarder: defaultforwarder.NoopForwarder{}, - Compressor: compressionmock.NewMockCompressor(), - HaAgent: haagentmock.NewMockHaAgent(), + Log: deps.Log, + Hostname: deps.Hostname, + SharedForwarder: defaultforwarder.NoopForwarder{}, + LogsCompression: logscompressionmock.NewMockCompressor(), + MetricsCompression: metricscompressionmock.NewMockCompressor(), + HaAgent: haagentmock.NewMockHaAgent(), } instance := &mock{AgentDemultiplexer: aggregator.InitAndStartAgentDemultiplexerForTest(aggDeps, opts, "")} diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go b/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go index 0e60ca002a79e3..2470cfd351a46d 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go @@ -21,7 +21,8 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -60,11 +61,12 @@ func TestStatusOutPut(t *testing.T) { deps := fxutil.Test[dependencies](t, fx.Options( core.MockBundle(), - compressionmock.MockModule(), defaultforwarder.MockModule(), haagentmock.Module(), orchestratorimpl.MockModule(), eventplatformimpl.MockModule(), + logscompression.MockModule(), + metricscompression.MockModule(), fx.Provide(func() tagger.Component { return mockTagger }), diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/test_agent_demultiplexer.go b/comp/aggregator/demultiplexer/demultiplexerimpl/test_agent_demultiplexer.go index c686630695254e..8f103d19b22fe8 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/test_agent_demultiplexer.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/test_agent_demultiplexer.go @@ -19,7 +19,8 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/logs/message" @@ -176,7 +177,7 @@ func (a *TestAgentDemultiplexer) Reset() { } // initTestAgentDemultiplexerWithFlushInterval inits a TestAgentDemultiplexer with the given flush interval. -func initTestAgentDemultiplexerWithFlushInterval(log log.Component, hostname hostname.Component, compressor compression.Component, flushInterval time.Duration) *TestAgentDemultiplexer { +func initTestAgentDemultiplexerWithFlushInterval(log log.Component, hostname hostname.Component, logscompressor logscompression.Component, metricscompressor metricscompression.Component, flushInterval time.Duration) *TestAgentDemultiplexer { opts := aggregator.DefaultAgentDemultiplexerOptions() opts.FlushInterval = flushInterval opts.DontStartForwarders = true @@ -184,8 +185,9 @@ func initTestAgentDemultiplexerWithFlushInterval(log log.Component, hostname hos sharedForwarderOptions := defaultforwarder.NewOptions(pkgconfigsetup.Datadog(), log, nil) sharedForwarder := defaultforwarder.NewDefaultForwarder(pkgconfigsetup.Datadog(), log, sharedForwarderOptions) + orchestratorForwarder := option.New[defaultforwarder.Forwarder](defaultforwarder.NoopForwarder{}) - eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(hostname)) - demux := aggregator.InitAndStartAgentDemultiplexer(log, sharedForwarder, &orchestratorForwarder, opts, eventPlatformForwarder, haagentmock.NewMockHaAgent(), compressor, noopimpl.NewComponent(), "hostname") + eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(hostname, logscompressor)) + demux := aggregator.InitAndStartAgentDemultiplexer(log, sharedForwarder, &orchestratorForwarder, opts, eventPlatformForwarder, haagentmock.NewMockHaAgent(), metricscompressor, noopimpl.NewComponent(), "hostname") return NewTestAgentDemultiplexer(demux) } diff --git a/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl/sendermanager.go b/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl/sendermanager.go index 58bbb23682b542..0f290c2bb216ba 100644 --- a/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl/sendermanager.go +++ b/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl/sendermanager.go @@ -20,7 +20,8 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" haagent "github.com/DataDog/datadog-agent/comp/haagent/def" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -35,12 +36,13 @@ func Module() fxutil.Module { type dependencies struct { fx.In - Log log.Component - Config config.Component - Hostname hostname.Component - Compressor compression.Component - Tagger tagger.Component - HaAgent haagent.Component + Log log.Component + Config config.Component + Hostname hostname.Component + LogsCompressor logscompression.Component + MetricsCompressor metricscompression.Component + Tagger tagger.Component + HaAgent haagent.Component } type diagnoseSenderManager struct { @@ -74,7 +76,7 @@ func (sender *diagnoseSenderManager) LazyGetSenderManager() (sender.SenderManage haAgent := sender.deps.HaAgent forwarder := defaultforwarder.NewDefaultForwarder(config, log, defaultforwarder.NewOptions(config, log, nil)) orchestratorForwarder := option.NewPtr[defaultforwarder.Forwarder](defaultforwarder.NoopForwarder{}) - eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(sender.deps.Hostname)) + eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(sender.deps.Hostname, sender.deps.LogsCompressor)) senderManager = aggregator.InitAndStartAgentDemultiplexer( log, forwarder, @@ -82,7 +84,7 @@ func (sender *diagnoseSenderManager) LazyGetSenderManager() (sender.SenderManage opts, eventPlatformForwarder, haAgent, - sender.deps.Compressor, + sender.deps.MetricsCompressor, sender.deps.Tagger, hostnameDetected) diff --git a/comp/collector/collector/collectorimpl/collector_demux_test.go b/comp/collector/collector/collectorimpl/collector_demux_test.go index 154f64c84f006b..e6210a69cf0d8d 100644 --- a/comp/collector/collector/collectorimpl/collector_demux_test.go +++ b/comp/collector/collector/collectorimpl/collector_demux_test.go @@ -22,7 +22,8 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" @@ -82,7 +83,7 @@ func (s *SenderManagerProxy) GetDefaultSender() (sender.Sender, error) { } func (suite *CollectorDemuxTestSuite) SetupTest() { - suite.demux = fxutil.Test[demultiplexer.FakeSamplerMock](suite.T(), fx.Provide(func() log.Component { return logmock.New(suite.T()) }), compressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) + suite.demux = fxutil.Test[demultiplexer.FakeSamplerMock](suite.T(), fx.Provide(func() log.Component { return logmock.New(suite.T()) }), metricscompressionmock.MockModule(), logscompressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) suite.SenderManagerMock = NewSenderManagerMock(suite.demux) suite.c = newCollector(fxutil.Test[dependencies](suite.T(), core.MockBundle(), diff --git a/comp/dogstatsd/server/server_util_test.go b/comp/dogstatsd/server/server_util_test.go index a84bdbb0f07413..d6080b0f2cb47f 100644 --- a/comp/dogstatsd/server/server_util_test.go +++ b/comp/dogstatsd/server/server_util_test.go @@ -35,7 +35,8 @@ import ( replaymock "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/fx-mock" serverdebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug" "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug/serverdebugimpl" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/option" @@ -82,10 +83,12 @@ func fulfillDepsWithConfigOverride(t testing.TB, overrides map[string]interface{ Overrides: overrides, }), replaymock.MockModule(), - compressionmock.MockModule(), pidmapimpl.Module(), demultiplexerimpl.FakeSamplerMockModule(), workloadmetafxmock.MockModule(workloadmeta.NewParams()), + logscompression.MockModule(), + metricscompression.MockModule(), + Module(Params{Serverless: false}), )) } @@ -98,7 +101,8 @@ func fulfillDepsWithConfigYaml(t testing.TB, yaml string) serverDeps { hostnameimpl.MockModule(), serverdebugimpl.MockModule(), replaymock.MockModule(), - compressionmock.MockModule(), + metricscompression.MockModule(), + logscompression.MockModule(), pidmapimpl.Module(), demultiplexerimpl.FakeSamplerMockModule(), workloadmetafxmock.MockModule(workloadmeta.NewParams()), @@ -117,10 +121,11 @@ func fulfillDepsWithInactiveServer(t *testing.T, cfg map[string]interface{}) (de }), fx.Supply(Params{Serverless: false}), replaymock.MockModule(), - compressionmock.MockModule(), pidmapimpl.Module(), demultiplexerimpl.FakeSamplerMockModule(), workloadmetafxmock.MockModule(workloadmeta.NewParams()), + metricscompression.MockModule(), + logscompression.MockModule(), )) s := newServerCompat(deps.Config, deps.Log, deps.Replay, deps.Debug, false, deps.Demultiplexer, deps.WMeta, deps.PidMap, deps.Telemetry) diff --git a/comp/forwarder/eventplatform/eventplatformimpl/epforwarder.go b/comp/forwarder/eventplatform/eventplatformimpl/epforwarder.go index 751c365ae91559..a06a3a55c7ef28 100644 --- a/comp/forwarder/eventplatform/eventplatformimpl/epforwarder.go +++ b/comp/forwarder/eventplatform/eventplatformimpl/epforwarder.go @@ -21,6 +21,7 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/diagnose/diagnosis" @@ -30,6 +31,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/logs/metrics" "github.com/DataDog/datadog-agent/pkg/logs/sender" + compressioncommon "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/option" @@ -373,7 +375,7 @@ type passthroughPipelineDesc struct { // newHTTPPassthroughPipeline creates a new HTTP-only event platform pipeline that sends messages directly to intake // without any of the processing that exists in regular logs pipelines. -func newHTTPPassthroughPipeline(coreConfig model.Reader, eventPlatformReceiver eventplatformreceiver.Component, desc passthroughPipelineDesc, destinationsContext *client.DestinationsContext, pipelineID int) (p *passthroughPipeline, err error) { +func newHTTPPassthroughPipeline(coreConfig model.Reader, eventPlatformReceiver eventplatformreceiver.Component, compressor logscompression.Component, desc passthroughPipelineDesc, destinationsContext *client.DestinationsContext, pipelineID int) (p *passthroughPipeline, err error) { configKeys := config.NewLogsConfigKeys(desc.endpointsConfigPrefix, pkgconfigsetup.Datadog()) endpoints, err := config.BuildHTTPEndpointsWithConfig(pkgconfigsetup.Datadog(), configKeys, desc.hostnameEndpointPrefix, desc.intakeTrackType, config.DefaultIntakeProtocol, config.DefaultIntakeOrigin) if err != nil { @@ -412,9 +414,10 @@ func newHTTPPassthroughPipeline(coreConfig model.Reader, eventPlatformReceiver e inputChan := make(chan *message.Message, endpoints.InputChanSize) senderInput := make(chan *message.Payload, 1) // Only buffer 1 message since payloads can be large - encoder := sender.IdentityContentType + var encoder compressioncommon.Compressor + encoder = compressor.NewCompressor("none", 0) if endpoints.Main.UseCompression { - encoder = sender.NewGzipContentEncoding(endpoints.Main.CompressionLevel) + encoder = compressor.NewCompressor(endpoints.Main.CompressionKind, endpoints.Main.CompressionLevel) } var strategy sender.Strategy @@ -471,12 +474,12 @@ func joinHosts(endpoints []config.Endpoint) string { return strings.Join(additionalHosts, ",") } -func newDefaultEventPlatformForwarder(config model.Reader, eventPlatformReceiver eventplatformreceiver.Component) *defaultEventPlatformForwarder { +func newDefaultEventPlatformForwarder(config model.Reader, eventPlatformReceiver eventplatformreceiver.Component, compression logscompression.Component) *defaultEventPlatformForwarder { destinationsCtx := client.NewDestinationsContext() destinationsCtx.Start() pipelines := make(map[string]*passthroughPipeline) for i, desc := range passthroughPipelineDescs { - p, err := newHTTPPassthroughPipeline(config, eventPlatformReceiver, desc, destinationsCtx, i) + p, err := newHTTPPassthroughPipeline(config, eventPlatformReceiver, compression, desc, destinationsCtx, i) if err != nil { log.Errorf("Failed to initialize event platform forwarder pipeline. eventType=%s, error=%s", desc.eventType, err.Error()) continue @@ -496,6 +499,7 @@ type dependencies struct { Lc fx.Lifecycle EventPlatformReceiver eventplatformreceiver.Component Hostname hostnameinterface.Component + Compression logscompression.Component } // newEventPlatformForwarder creates a new EventPlatformForwarder @@ -503,9 +507,9 @@ func newEventPlatformForwarder(deps dependencies) eventplatform.Component { var forwarder *defaultEventPlatformForwarder if deps.Params.UseNoopEventPlatformForwarder { - forwarder = newNoopEventPlatformForwarder(deps.Hostname) + forwarder = newNoopEventPlatformForwarder(deps.Hostname, deps.Compression) } else if deps.Params.UseEventPlatformForwarder { - forwarder = newDefaultEventPlatformForwarder(deps.Config, deps.EventPlatformReceiver) + forwarder = newDefaultEventPlatformForwarder(deps.Config, deps.EventPlatformReceiver, deps.Compression) } if forwarder == nil { return option.NonePtr[eventplatform.Forwarder]() @@ -525,12 +529,12 @@ func newEventPlatformForwarder(deps dependencies) eventplatform.Component { // NewNoopEventPlatformForwarder returns the standard event platform forwarder with sending disabled, meaning events // will build up in each pipeline channel without being forwarded to the intake -func NewNoopEventPlatformForwarder(hostname hostnameinterface.Component) eventplatform.Forwarder { - return newNoopEventPlatformForwarder(hostname) +func NewNoopEventPlatformForwarder(hostname hostnameinterface.Component, compression logscompression.Component) eventplatform.Forwarder { + return newNoopEventPlatformForwarder(hostname, compression) } -func newNoopEventPlatformForwarder(hostname hostnameinterface.Component) *defaultEventPlatformForwarder { - f := newDefaultEventPlatformForwarder(pkgconfigsetup.Datadog(), eventplatformreceiverimpl.NewReceiver(hostname).Comp) +func newNoopEventPlatformForwarder(hostname hostnameinterface.Component, compression logscompression.Component) *defaultEventPlatformForwarder { + f := newDefaultEventPlatformForwarder(pkgconfigsetup.Datadog(), eventplatformreceiverimpl.NewReceiver(hostname).Comp, compression) // remove the senders for _, p := range f.pipelines { p.strategy = nil diff --git a/comp/forwarder/eventplatform/eventplatformimpl/epforwarder_mock.go b/comp/forwarder/eventplatform/eventplatformimpl/epforwarder_mock.go index c2ac206b03f326..a56c7b6a94ebd6 100644 --- a/comp/forwarder/eventplatform/eventplatformimpl/epforwarder_mock.go +++ b/comp/forwarder/eventplatform/eventplatformimpl/epforwarder_mock.go @@ -10,6 +10,7 @@ package eventplatformimpl import ( "github.com/DataDog/datadog-agent/comp/core/hostname" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/option" "go.uber.org/fx" @@ -22,6 +23,6 @@ func MockModule() fxutil.Module { ) } -func newMockComponent(hostname hostname.Component) eventplatform.Component { - return option.NewPtr[eventplatform.Forwarder](NewNoopEventPlatformForwarder(hostname)) +func newMockComponent(hostname hostname.Component, compression logscompression.Component) eventplatform.Component { + return option.NewPtr[eventplatform.Forwarder](NewNoopEventPlatformForwarder(hostname, compression)) } diff --git a/comp/logs/agent/agentimpl/agent.go b/comp/logs/agent/agentimpl/agent.go index a9b7229811f4a3..110fee063ae9cc 100644 --- a/comp/logs/agent/agentimpl/agent.go +++ b/comp/logs/agent/agentimpl/agent.go @@ -31,6 +31,7 @@ import ( integrationsimpl "github.com/DataDog/datadog-agent/comp/logs/integrations/impl" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent" rctypes "github.com/DataDog/datadog-agent/comp/remote-config/rcclient/types" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/logs/auditor" "github.com/DataDog/datadog-agent/pkg/logs/client" @@ -82,6 +83,7 @@ type dependencies struct { WMeta option.Option[workloadmeta.Component] SchedulerProviders []schedulers.Scheduler `group:"log-agent-scheduler"` Tagger tagger.Component + Compression logscompression.Component } type provides struct { @@ -119,6 +121,7 @@ type logAgent struct { wmeta option.Option[workloadmeta.Component] schedulerProviders []schedulers.Scheduler integrationsLogs integrations.Component + compression logscompression.Component // make sure this is done only once, when we're ready prepareSchedulers sync.Once @@ -150,6 +153,7 @@ func newLogsAgent(deps dependencies) provides { schedulerProviders: deps.SchedulerProviders, integrationsLogs: integrationsLogs, tagger: deps.Tagger, + compression: deps.Compression, } deps.Lc.Append(fx.Hook{ OnStart: logsAgent.start, diff --git a/comp/logs/agent/agentimpl/agent_core_init.go b/comp/logs/agent/agentimpl/agent_core_init.go index 28ad534b8bd283..87e2b00810b246 100644 --- a/comp/logs/agent/agentimpl/agent_core_init.go +++ b/comp/logs/agent/agentimpl/agent_core_init.go @@ -46,7 +46,7 @@ func (a *logAgent) SetupPipeline(processingRules []*config.ProcessingRule, wmeta diagnosticMessageReceiver := diagnostic.NewBufferedMessageReceiver(nil, a.hostname) // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(a.config.GetInt("logs_config.pipelines"), auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) + pipelineProvider := pipeline.NewProvider(a.config.GetInt("logs_config.pipelines"), auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config, a.compression) // setup the launchers lnchrs := launchers.NewLaunchers(a.sources, pipelineProvider, auditor, a.tracker) diff --git a/comp/logs/agent/agentimpl/agent_serverless_init.go b/comp/logs/agent/agentimpl/agent_serverless_init.go index 47cc71943ac469..dabe8ab0601351 100644 --- a/comp/logs/agent/agentimpl/agent_serverless_init.go +++ b/comp/logs/agent/agentimpl/agent_serverless_init.go @@ -49,7 +49,7 @@ func (a *logAgent) SetupPipeline( destinationsCtx := client.NewDestinationsContext() // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewServerlessProvider(a.config.GetInt("logs_config.pipelines"), a.auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) + pipelineProvider := pipeline.NewServerlessProvider(a.config.GetInt("logs_config.pipelines"), a.auditor, diagnosticMessageReceiver, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config, a.compression) lnchrs := launchers.NewLaunchers(a.sources, pipelineProvider, a.auditor, a.tracker) lnchrs.AddLauncher(channel.NewLauncher()) diff --git a/comp/logs/agent/agentimpl/agent_test.go b/comp/logs/agent/agentimpl/agent_test.go index 6af1f1781419ef..df067c7f2fc737 100644 --- a/comp/logs/agent/agentimpl/agent_test.go +++ b/comp/logs/agent/agentimpl/agent_test.go @@ -36,6 +36,7 @@ import ( flareController "github.com/DataDog/datadog-agent/comp/logs/agent/flare" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent/inventoryagentimpl" + compressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/config/env" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/logs/client/http" @@ -134,11 +135,12 @@ func createAgent(suite *AgentTestSuite, endpoints *config.Endpoints) (*logAgent, started: atomic.NewUint32(0), integrationsLogs: integrationsimpl.NewLogsIntegration(), - sources: sources, - services: services, - tracker: tailers.NewTailerTracker(), - endpoints: endpoints, - tagger: fakeTagger, + sources: sources, + services: services, + tracker: tailers.NewTailerTracker(), + endpoints: endpoints, + tagger: fakeTagger, + compression: compressionfx.NewMockCompressor(), } agent.setupAgent() @@ -405,6 +407,7 @@ func (suite *AgentTestSuite) createDeps() dependencies { fx.Replace(configComponent.MockParams{Overrides: suite.configOverrides}), inventoryagentimpl.MockModule(), workloadmetafxmock.MockModule(workloadmeta.NewParams()), + compressionfx.MockModule(), fx.Provide(func() tagger.Component { return suite.tagger }), diff --git a/comp/logs/agent/agentimpl/serverless.go b/comp/logs/agent/agentimpl/serverless.go index 44752f65726cc3..32d725da3fa0ce 100644 --- a/comp/logs/agent/agentimpl/serverless.go +++ b/comp/logs/agent/agentimpl/serverless.go @@ -14,6 +14,7 @@ import ( tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/comp/logs/agent" flareController "github.com/DataDog/datadog-agent/comp/logs/agent/flare" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/logs/service" "github.com/DataDog/datadog-agent/pkg/logs/sources" @@ -21,7 +22,8 @@ import ( ) // NewServerlessLogsAgent creates a new instance of the logs agent for serverless -func NewServerlessLogsAgent(tagger tagger.Component) agent.ServerlessLogsAgent { +func NewServerlessLogsAgent(tagger tagger.Component, compression logscompression.Component) agent.ServerlessLogsAgent { + logsAgent := &logAgent{ log: logComponent.NewTemporaryLoggerWithoutInit(), config: pkgconfigsetup.Datadog(), @@ -32,6 +34,7 @@ func NewServerlessLogsAgent(tagger tagger.Component) agent.ServerlessLogsAgent { tracker: tailers.NewTailerTracker(), flarecontroller: flareController.NewFlareController(), tagger: tagger, + compression: compression, } return logsAgent } diff --git a/comp/logs/agent/config/config_keys.go b/comp/logs/agent/config/config_keys.go index 0f9dbbf2b36de8..ed27929cd5be08 100644 --- a/comp/logs/agent/config/config_keys.go +++ b/comp/logs/agent/config/config_keys.go @@ -109,7 +109,15 @@ func (l *LogsConfigKeys) devModeUseProto() bool { return l.getConfig().GetBool(l.getConfigKey("dev_mode_use_proto")) } +func (l *LogsConfigKeys) compressionKind() string { + return l.getConfig().GetString(l.getConfigKey("compression_kind")) +} + func (l *LogsConfigKeys) compressionLevel() int { + if l.compressionKind() == "zstd" { + return l.getConfig().GetInt(l.getConfigKey("zstd_compression_level")) + } + return l.getConfig().GetInt(l.getConfigKey("compression_level")) } diff --git a/comp/logs/agent/config/endpoints.go b/comp/logs/agent/config/endpoints.go index 58ce82d31255c4..192c5c0bf52775 100644 --- a/comp/logs/agent/config/endpoints.go +++ b/comp/logs/agent/config/endpoints.go @@ -41,8 +41,9 @@ type Endpoint struct { Host string `mapstructure:"host" json:"host"` Port int - UseCompression bool `mapstructure:"use_compression" json:"use_compression"` - CompressionLevel int `mapstructure:"compression_level" json:"compression_level"` + UseCompression bool `mapstructure:"use_compression" json:"use_compression"` + CompressionKind string `mapstructure:"compression_kind" json:"compression_kind"` + CompressionLevel int `mapstructure:"compression_level" json:"compression_level"` ProxyAddress string IsMRF bool `mapstructure:"-" json:"-"` ConnectionResetInterval time.Duration @@ -99,6 +100,7 @@ func NewHTTPEndpoint(logsConfig *LogsConfigKeys) Endpoint { return Endpoint{ apiKeyGetter: logsConfig.getAPIKeyGetter(), UseCompression: logsConfig.useCompression(), + CompressionKind: logsConfig.compressionKind(), CompressionLevel: logsConfig.compressionLevel(), ConnectionResetInterval: logsConfig.connectionResetInterval(), BackoffBase: logsConfig.senderBackoffBase(), diff --git a/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go b/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go index a0649ec8bef2ea..535833cd53e6c6 100644 --- a/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go +++ b/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go @@ -27,6 +27,7 @@ import ( logagent "github.com/DataDog/datadog-agent/comp/logs/agent" logConfig "github.com/DataDog/datadog-agent/comp/logs/agent/config" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent/inventoryagentimpl" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/logs/sources" @@ -159,6 +160,7 @@ func TestGetPayload(t *testing.T) { logsBundle.MockBundle(), core.MockBundle(), inventoryagentimpl.MockModule(), + logscompression.MockModule(), workloadmetafxmock.MockModule(workloadmeta.NewParams()), fx.Provide(func() tagger.Component { return fakeTagger diff --git a/comp/networkpath/bundle_test.go b/comp/networkpath/bundle_test.go index 6eb6868e8acbaa..1e26b5677a53ce 100644 --- a/comp/networkpath/bundle_test.go +++ b/comp/networkpath/bundle_test.go @@ -11,6 +11,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" rdnsquerier "github.com/DataDog/datadog-agent/comp/rdnsquerier/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -19,5 +20,6 @@ func TestBundleDependencies(t *testing.T) { core.MockBundle(), eventplatformimpl.MockModule(), rdnsquerier.MockModule(), + logscompression.MockModule(), ) } diff --git a/comp/networkpath/npcollector/npcollectorimpl/npcollector_testutils.go b/comp/networkpath/npcollector/npcollectorimpl/npcollector_testutils.go index 16d777c11fd79d..a5530698426748 100644 --- a/comp/networkpath/npcollector/npcollectorimpl/npcollector_testutils.go +++ b/comp/networkpath/npcollector/npcollectorimpl/npcollector_testutils.go @@ -23,6 +23,7 @@ import ( "github.com/DataDog/datadog-agent/comp/ndmtmp/forwarder/forwarderimpl" "github.com/DataDog/datadog-agent/comp/networkpath/npcollector" rdnsqueriermock "github.com/DataDog/datadog-agent/comp/rdnsquerier/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/stretchr/testify/require" "go.uber.org/fx" "go.uber.org/fx/fxtest" @@ -45,6 +46,7 @@ var testOptions = fx.Options( core.MockBundle(), eventplatformimpl.MockModule(), rdnsqueriermock.MockModule(), + logscompression.MockModule(), ) func newTestNpCollector(t fxtest.TB, agentConfigs map[string]any) (*fxtest.App, *npCollectorImpl) { diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index c1607bf06d3051..e39662c20b5e57 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -34,7 +34,8 @@ replace ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor => ../../otlp/components/processor/infraattributesprocessor github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ../../otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil => ../../../otelcol/otlp/testutil - github.com/DataDog/datadog-agent/comp/serializer/compression => ../../../serializer/compression + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../../serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../../serializer/metricscompression github.com/DataDog/datadog-agent/comp/trace/agent/def => ../../../trace/agent/def github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../trace/compression/impl-gzip @@ -79,6 +80,7 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/buf => ../../../../pkg/util/buf github.com/DataDog/datadog-agent/pkg/util/cgroups => ../../../../pkg/util/cgroups github.com/DataDog/datadog-agent/pkg/util/common => ../../../../pkg/util/common + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../../pkg/util/compression github.com/DataDog/datadog-agent/pkg/util/containers/image => ../../../../pkg/util/containers/image github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../../../pkg/util/executable @@ -165,6 +167,8 @@ require ( require ( github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.62.0-rc.1 // indirect + github.com/DataDog/datadog-agent/comp/serializer/metricscompression v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect @@ -216,7 +220,7 @@ require ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient v0.60.1 // indirect github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor v0.60.1 // indirect github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil v0.57.0-devel.0.20240718200853-81bf3b2e412d // indirect - github.com/DataDog/datadog-agent/comp/serializer/compression v0.59.0-rc.6 // indirect + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.59.0-rc.6 // indirect github.com/DataDog/datadog-agent/comp/trace/agent/def v0.59.0-rc.6 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.60.1 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.60.1 // indirect diff --git a/comp/otelcol/logsagentpipeline/go.mod b/comp/otelcol/logsagentpipeline/go.mod index b4797cb0c91060..c987e4a80dce12 100644 --- a/comp/otelcol/logsagentpipeline/go.mod +++ b/comp/otelcol/logsagentpipeline/go.mod @@ -14,6 +14,8 @@ replace ( github.com/DataDog/datadog-agent/comp/core/telemetry => ../../core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../def github.com/DataDog/datadog-agent/comp/logs/agent/config => ../../logs/agent/config + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../serializer/metricscompression github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../pkg/collector/check/defaults github.com/DataDog/datadog-agent/pkg/config/env => ../../../pkg/config/env github.com/DataDog/datadog-agent/pkg/config/mock => ../../../pkg/config/mock @@ -39,6 +41,7 @@ replace ( github.com/DataDog/datadog-agent/pkg/status/health => ../../../pkg/status/health github.com/DataDog/datadog-agent/pkg/telemetry => ../../../pkg/telemetry github.com/DataDog/datadog-agent/pkg/util/backoff => ../../../pkg/util/backoff + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../pkg/util/compression github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../../pkg/util/executable github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../pkg/util/filesystem @@ -67,8 +70,9 @@ require ( github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 // indirect @@ -91,9 +95,10 @@ require ( github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 // indirect diff --git a/comp/otelcol/logsagentpipeline/go.sum b/comp/otelcol/logsagentpipeline/go.sum index c2e16fcb1548f3..c2af38433a1ac2 100644 --- a/comp/otelcol/logsagentpipeline/go.sum +++ b/comp/otelcol/logsagentpipeline/go.sum @@ -6,6 +6,8 @@ github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go index 4050b068c210b5..9da32f7bd6739e 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent.go @@ -16,6 +16,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/logs/agent/config" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/logs/auditor" "github.com/DataDog/datadog-agent/pkg/logs/client" @@ -41,17 +42,19 @@ const ( type Dependencies struct { fx.In - Lc fx.Lifecycle - Log log.Component - Config configComponent.Component - Hostname hostnameinterface.Component + Lc fx.Lifecycle + Log log.Component + Config configComponent.Component + Hostname hostnameinterface.Component + Compression compression.Component } // Agent represents the data pipeline that collects, decodes, processes and sends logs to the backend. type Agent struct { - log log.Component - config pkgconfigmodel.Reader - hostname hostnameinterface.Component + log log.Component + config pkgconfigmodel.Reader + hostname hostnameinterface.Component + compression compression.Component endpoints *config.Endpoints auditor auditor.Auditor @@ -77,9 +80,10 @@ func NewLogsAgent(deps Dependencies) logsagentpipeline.LogsAgent { } logsAgent := &Agent{ - log: deps.Log, - config: deps.Config, - hostname: deps.Hostname, + log: deps.Log, + config: deps.Config, + hostname: deps.Hostname, + compression: deps.Compression, } if deps.Lc != nil { deps.Lc.Append(fx.Hook{ @@ -210,7 +214,7 @@ func (a *Agent) SetupPipeline( destinationsCtx := client.NewDestinationsContext() // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(a.config.GetInt("logs_config.pipelines"), auditor, &diagnostic.NoopMessageReceiver{}, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config) + pipelineProvider := pipeline.NewProvider(a.config.GetInt("logs_config.pipelines"), auditor, &diagnostic.NoopMessageReceiver{}, processingRules, a.endpoints, destinationsCtx, NewStatusProvider(), a.hostname, a.config, a.compression) a.auditor = auditor a.destinationsCtx = destinationsCtx diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent_test.go b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent_test.go index da2be9d72e4d70..420a8138610836 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent_test.go +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/agent_test.go @@ -16,6 +16,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + compressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/logs/client/http" "github.com/DataDog/datadog-agent/pkg/logs/client/mock" @@ -71,9 +72,10 @@ func createAgent(suite *AgentTestSuite, endpoints *config.Endpoints) *Agent { )) agent := &Agent{ - log: deps.Log, - config: deps.Config, - endpoints: endpoints, + log: deps.Log, + config: deps.Config, + endpoints: endpoints, + compression: compressionfx.NewMockCompressor(), } agent.setupAgent() diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod index 8dd13d8f235aeb..421fdd40e93a63 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod @@ -15,6 +15,8 @@ replace ( github.com/DataDog/datadog-agent/comp/def => ../../../def github.com/DataDog/datadog-agent/comp/logs/agent/config => ../../../logs/agent/config github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline => ../../logsagentpipeline + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../../serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../../serializer/metricscompression github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../../pkg/collector/check/defaults github.com/DataDog/datadog-agent/pkg/config/env => ../../../../pkg/config/env github.com/DataDog/datadog-agent/pkg/config/mock => ../../../../pkg/config/mock @@ -40,6 +42,7 @@ replace ( github.com/DataDog/datadog-agent/pkg/status/health => ../../../../pkg/status/health github.com/DataDog/datadog-agent/pkg/telemetry => ../../../../pkg/telemetry github.com/DataDog/datadog-agent/pkg/util/backoff => ../../../../pkg/util/backoff + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../../pkg/util/compression github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../../../pkg/util/executable github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../../pkg/util/filesystem @@ -62,12 +65,13 @@ replace ( ) require ( - github.com/DataDog/datadog-agent/comp/core/config v0.56.0-rc.3 + github.com/DataDog/datadog-agent/comp/core/config v0.56.2 github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/core/log/mock v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline v0.56.0-rc.3 + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/config/setup v0.59.0 github.com/DataDog/datadog-agent/pkg/logs/auditor v0.56.0-rc.3 @@ -79,7 +83,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/testutil v0.56.0-rc.3 @@ -90,11 +94,11 @@ require ( require ( github.com/DataDog/agent-payload/v5 v5.0.140 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 // indirect @@ -108,6 +112,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect @@ -123,6 +128,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/viper v1.14.0 // indirect + github.com/DataDog/zstd v1.5.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum index c2e16fcb1548f3..c2af38433a1ac2 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum @@ -6,6 +6,8 @@ github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index 4f3e4c31434575..986e7567ac5039 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -23,7 +23,8 @@ replace ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter => ../../../../otlp/components/exporter/serializerexporter github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../../metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil => ../../../../otlp/testutil - github.com/DataDog/datadog-agent/comp/serializer/compression => ../../../../../serializer/compression/ + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../../../../serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../../../../serializer/metricscompression github.com/DataDog/datadog-agent/comp/trace/agent/def => ../../../../../../comp/trace/agent/def github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../../../../comp/trace/compression/impl-gzip @@ -68,6 +69,7 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/buf => ../../../../../../pkg/util/buf/ github.com/DataDog/datadog-agent/pkg/util/cgroups => ../../../../../../pkg/util/cgroups/ github.com/DataDog/datadog-agent/pkg/util/common => ../../../../../../pkg/util/common/ + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../../../../pkg/util/compression github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../../../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../../../../../pkg/util/executable/ github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../../../../pkg/util/filesystem/ @@ -140,8 +142,8 @@ require ( require ( github.com/DataDog/agent-payload/v5 v5.0.140 // indirect github.com/DataDog/datadog-agent/comp/core/config v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/builder v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/types v0.57.1 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect @@ -152,7 +154,8 @@ require ( github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/serializer/compression v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.56.2 // indirect + github.com/DataDog/datadog-agent/comp/serializer/metricscompression v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/aggregator/ckey v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/api v0.59.0 // indirect @@ -189,6 +192,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/buf v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/cgroups v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/common v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod index 1b7db57401f862..521e61c80b506e 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod @@ -16,7 +16,8 @@ replace ( github.com/DataDog/datadog-agent/comp/def => ../../../../../def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../../../../forwarder/defaultforwarder github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface => ../../../../../forwarder/orchestrator/orchestratorinterface - github.com/DataDog/datadog-agent/comp/serializer/compression => ../../../../../serializer/compression/ + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../../../../serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../../../../serializer/metricscompression github.com/DataDog/datadog-agent/pkg/aggregator/ckey => ../../../../../../pkg/aggregator/ckey github.com/DataDog/datadog-agent/pkg/api => ../../../../../../pkg/api github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../../../../pkg/collector/check/defaults @@ -43,6 +44,7 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/backoff => ../../../../../../pkg/util/backoff/ github.com/DataDog/datadog-agent/pkg/util/buf => ../../../../../../pkg/util/buf/ github.com/DataDog/datadog-agent/pkg/util/common => ../../../../../../pkg/util/common/ + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../../../../pkg/util/compression/ github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../../../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../../../../../pkg/util/executable/ github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../../../../pkg/util/filesystem/ @@ -93,16 +95,15 @@ require ( require ( github.com/DataDog/agent-payload/v5 v5.0.140 // indirect github.com/DataDog/datadog-agent/comp/core/config v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/builder v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/types v0.57.1 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/def v0.57.1 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/serializer/compression v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/aggregator/ckey v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/api v0.57.1 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect @@ -124,7 +125,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/common v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/json v0.56.0-rc.3 // indirect @@ -242,6 +243,8 @@ require ( require ( github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.62.0-rc.1 // indirect + github.com/DataDog/datadog-agent/comp/serializer/metricscompression v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect diff --git a/comp/otelcol/otlp/integrationtest/integration_test.go b/comp/otelcol/otlp/integrationtest/integration_test.go index 0ac554d4d0b630..8e9cbdf5204d6f 100644 --- a/comp/otelcol/otlp/integrationtest/integration_test.go +++ b/comp/otelcol/otlp/integrationtest/integration_test.go @@ -63,8 +63,9 @@ import ( "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - implzlib "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zlib" + logscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + metricscompressionfx "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-otel" tracecomp "github.com/DataDog/datadog-agent/comp/trace" traceagentcomp "github.com/DataDog/datadog-agent/comp/trace/agent/impl" gzipfx "github.com/DataDog/datadog-agent/comp/trace/compression/fx-gzip" @@ -74,6 +75,7 @@ import ( pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/trace/telemetry" + "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/option" ) @@ -118,10 +120,12 @@ func runTestOTelAgent(ctx context.Context, params *subcommands.GlobalParams) err return logdef.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) }), logsagentpipelineimpl.Module(), - // We create strategy.ZlibStrategy directly to avoid build tags - fx.Provide(implzlib.NewComponent), - fx.Provide(func(s implzlib.Provides) compression.Component { - return s.Comp + logscompressionfx.Module(), + metricscompressionfx.Module(), + // For FX to provide the compression.Compressor interface (used by serializer.NewSerializer) + // implemented by the metricsCompression.Component + fx.Provide(func(c metricscompression.Component) compression.Compressor { + return c }), fx.Provide(serializer.NewSerializer), // For FX to provide the serializer.MetricSerializer from the serializer.Serializer diff --git a/comp/process/bundle.go b/comp/process/bundle.go index 96802180c2cce1..b30a1bf04c11c7 100644 --- a/comp/process/bundle.go +++ b/comp/process/bundle.go @@ -26,6 +26,7 @@ import ( "github.com/DataDog/datadog-agent/comp/process/rtcontainercheck/rtcontainercheckimpl" "github.com/DataDog/datadog-agent/comp/process/runner/runnerimpl" "github.com/DataDog/datadog-agent/comp/process/submitter/submitterimpl" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -55,5 +56,6 @@ func Bundle() fxutil.BundleOptions { apiserver.Module(), forwardersimpl.Module(), + logscompression.Module(), ) } diff --git a/comp/serializer/compression/common/common.go b/comp/serializer/compression/common/common.go deleted file mode 100644 index 65b86991528599..00000000000000 --- a/comp/serializer/compression/common/common.go +++ /dev/null @@ -1,16 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -// Package common provides a set of constants describing the compression options -package common - -// ZlibKind defines a const value for the zlib compressor -const ZlibKind = "zlib" - -// ZstdKind defines a const value for the zstd compressor -const ZstdKind = "zstd" - -// NoneKind defines a const value for disabling compression -const NoneKind = "none" diff --git a/comp/serializer/compression/fx-zlib/fx.go b/comp/serializer/compression/fx-zlib/fx.go deleted file mode 100644 index f10af59cfc6d8a..00000000000000 --- a/comp/serializer/compression/fx-zlib/fx.go +++ /dev/null @@ -1,21 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -// Package fx provides the fx module for the serializer/compression component -package fx - -import ( - compressionimpl "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zlib" - "github.com/DataDog/datadog-agent/pkg/util/fxutil" -) - -// Module defines the fx options for the component. -func Module() fxutil.Module { - return fxutil.Component( - fxutil.ProvideComponentConstructor( - compressionimpl.NewComponent, - ), - ) -} diff --git a/comp/serializer/compression/mock/component_mock.go b/comp/serializer/compression/mock/component_mock.go deleted file mode 100644 index 726e179142b0af..00000000000000 --- a/comp/serializer/compression/mock/component_mock.go +++ /dev/null @@ -1,16 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024-present Datadog, Inc. - -//go:build test - -// Package mock provides the mock component for serializer/compression -package mock - -import compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - -// Mock implements mock-specific methods. -type Mock interface { - compression.Component -} diff --git a/comp/serializer/compression/selector/def.go b/comp/serializer/compression/selector/def.go deleted file mode 100644 index 0e8b5837b57e9d..00000000000000 --- a/comp/serializer/compression/selector/def.go +++ /dev/null @@ -1,21 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package selector - -import ( - "github.com/DataDog/datadog-agent/comp/core/config" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" -) - -// Requires contains the config for Compression -type Requires struct { - Cfg config.Component -} - -// Provides contains the compression component -type Provides struct { - Comp compression.Component -} diff --git a/comp/serializer/compression/selector/no-zlib-no-zstd.go b/comp/serializer/compression/selector/no-zlib-no-zstd.go deleted file mode 100644 index f370ab640200be..00000000000000 --- a/comp/serializer/compression/selector/no-zlib-no-zstd.go +++ /dev/null @@ -1,27 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build !zlib && !zstd - -// Package selector provides correct compression impl to fx -package selector - -import ( - "github.com/DataDog/datadog-agent/comp/core/config" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - implnoop "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-noop" -) - -// NewCompressorReq returns a new Compressor based on serializer_compressor_kind -// This function is called only when there is no zlib or zstd tag -func NewCompressorReq(_ Requires) Provides { - return Provides{Comp: implnoop.NewComponent().Comp} -} - -// NewCompressor returns a new Compressor based on serializer_compressor_kind -// This function is called only when there is no zlib or zstd tag -func NewCompressor(cfg config.Component) compression.Component { - return NewCompressorReq(Requires{Cfg: cfg}).Comp -} diff --git a/comp/serializer/compression/selector/zlib-and-zstd.go b/comp/serializer/compression/selector/zlib-and-zstd.go deleted file mode 100644 index f912f30e816f08..00000000000000 --- a/comp/serializer/compression/selector/zlib-and-zstd.go +++ /dev/null @@ -1,43 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build zlib && zstd - -// Package selector provides correct compression impl to fx -package selector - -import ( - "github.com/DataDog/datadog-agent/comp/core/config" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - implnoop "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-noop" - implzlib "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zlib" - implzstd "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zstd" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -// NewCompressorReq returns a new Compressor based on serializer_compressor_kind -// This function is called when both zlib and zstd build tags are included -func NewCompressorReq(req Requires) Provides { - switch req.Cfg.GetString("serializer_compressor_kind") { - case common.ZlibKind: - return Provides{implzlib.NewComponent().Comp} - case common.ZstdKind: - level := req.Cfg.GetInt("serializer_zstd_compressor_level") - return Provides{implzstd.NewComponent(implzstd.Requires{Level: compression.ZstdCompressionLevel(level)}).Comp} - case common.NoneKind: - log.Warn("no serializer_compressor_kind set. use zlib or zstd") - return Provides{implnoop.NewComponent().Comp} - default: - log.Warn("invalid serializer_compressor_kind detected. use one of 'zlib', 'zstd'") - return Provides{implnoop.NewComponent().Comp} - } -} - -// NewCompressor returns a new Compressor based on serializer_compressor_kind -// This function is called when both zlib and zstd build tags are included -func NewCompressor(cfg config.Component) compression.Component { - return NewCompressorReq(Requires{Cfg: cfg}).Comp -} diff --git a/comp/serializer/compression/selector/zlib-no-zstd.go b/comp/serializer/compression/selector/zlib-no-zstd.go deleted file mode 100644 index febf91e379e942..00000000000000 --- a/comp/serializer/compression/selector/zlib-no-zstd.go +++ /dev/null @@ -1,42 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build zlib && !zstd - -// Package selector provides correct compression impl to fx -package selector - -import ( - "github.com/DataDog/datadog-agent/comp/core/config" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - implnoop "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-noop" - implzlib "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zlib" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -// NewCompressorReq returns a new Compressor based on serializer_compressor_kind -// This function is called only when the zlib build tag is included -func NewCompressorReq(req Requires) Provides { - switch req.Cfg.GetString("serializer_compressor_kind") { - case common.ZlibKind: - return Provides{implzlib.NewComponent().Comp} - case common.ZstdKind: - log.Warn("zstd build tag not included. using zlib") - return Provides{implzlib.NewComponent().Comp} - case common.NoneKind: - log.Warn("no serializer_compressor_kind set. use zlib or zstd") - return Provides{implnoop.NewComponent().Comp} - default: - log.Warn("invalid serializer_compressor_kind detected. use zlib or zstd") - return Provides{implnoop.NewComponent().Comp} - } -} - -// NewCompressor returns a new Compressor based on serializer_compressor_kind -// This function is called only when the zlib build tag is included -func NewCompressor(cfg config.Component) compression.Component { - return NewCompressorReq(Requires{Cfg: cfg}).Comp -} diff --git a/comp/serializer/logscompression/def/component.go b/comp/serializer/logscompression/def/component.go new file mode 100644 index 00000000000000..5f959aea6ffb03 --- /dev/null +++ b/comp/serializer/logscompression/def/component.go @@ -0,0 +1,22 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package logscompression provides the component for logs compression +package logscompression + +// team: agent-processing-and-routing + +import ( + "github.com/DataDog/datadog-agent/pkg/util/compression" +) + +// Component is the component type. +// The logscompression component provides a factory that returns a requested Compressor +// used when setting up the endpoints. +// (This is different from the metrics compressor which returns the requested Compressor +// by reading the configuration at load time). +type Component interface { + NewCompressor(kind string, level int) compression.Compressor +} diff --git a/comp/serializer/compression/doc.go b/comp/serializer/logscompression/doc.go similarity index 73% rename from comp/serializer/compression/doc.go rename to comp/serializer/logscompression/doc.go index e15c0ed472a27e..d711ccdd6bf28e 100644 --- a/comp/serializer/compression/doc.go +++ b/comp/serializer/logscompression/doc.go @@ -3,5 +3,5 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package compression contains the serializer compression component -package compression +// Package logs contains the serializer compression component for metrics +package logs diff --git a/comp/serializer/logscompression/fx-mock/fx.go b/comp/serializer/logscompression/fx-mock/fx.go new file mode 100644 index 00000000000000..ceac5e34ae2bf2 --- /dev/null +++ b/comp/serializer/logscompression/fx-mock/fx.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build test + +// Package fx provides the fx module for the serializer/compression component +package fx + +import ( + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" +) + +type component struct{} + +func (*component) NewCompressor(_kind string, _level int) compression.Compressor { + return selector.NewNoopCompressor() +} + +// NewMockCompressor returns a mock component that will always return a noop compressor. +func NewMockCompressor() logscompression.Component { + return &component{} +} + +// MockModule defines the fx options for the mock component. +func MockModule() fxutil.Module { + return fxutil.Component( + fxutil.ProvideComponentConstructor( + NewMockCompressor, + ), + ) +} diff --git a/comp/serializer/compression/fx/fx.go b/comp/serializer/logscompression/fx/fx.go similarity index 71% rename from comp/serializer/compression/fx/fx.go rename to comp/serializer/logscompression/fx/fx.go index 093ea3f4d6e672..e135cbd5b56141 100644 --- a/comp/serializer/compression/fx/fx.go +++ b/comp/serializer/logscompression/fx/fx.go @@ -3,11 +3,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package fx provides the fx module for the serializer/compression component +// Package fx provides the fx module for the serializer/logscompression component package fx import ( - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + "github.com/DataDog/datadog-agent/comp/serializer/logscompression/impl" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -15,7 +15,7 @@ import ( func Module() fxutil.Module { return fxutil.Component( fxutil.ProvideComponentConstructor( - selector.NewCompressorReq, + logscompressionimpl.NewComponent, ), ) } diff --git a/comp/serializer/logscompression/go.mod b/comp/serializer/logscompression/go.mod new file mode 100644 index 00000000000000..954ae2d7315202 --- /dev/null +++ b/comp/serializer/logscompression/go.mod @@ -0,0 +1,133 @@ +module github.com/DataDog/datadog-agent/comp/serializer/logscompression + +go 1.22.0 + +replace ( + github.com/DataDog/datadog-agent/comp/core/config => ../../core/config + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../pkg/util/compression + github.com/DataDog/datadog-agent/pkg/util/compression/selector => ../../../pkg/util/compression/selector + github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../pkg/util/defaultpaths + github.com/DataDog/datadog-agent/pkg/util/fxutil => ../../../pkg/util/fxutil + github.com/DataDog/datadog-agent/pkg/util/option => ../../../pkg/util/option +) + +require ( + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 +) + +require ( + github.com/DataDog/datadog-agent/comp/core/config v0.0.0-00010101000000-000000000000 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/setup v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/structure v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/pointer v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/system v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect + github.com/DataDog/viper v1.14.0 // indirect + github.com/DataDog/zstd v1.5.6 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/ebitengine/purego v0.8.1 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.24.12 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/dig v1.18.0 // indirect + go.uber.org/fx v1.23.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/DataDog/datadog-agent/comp/def => ../../def + +replace github.com/DataDog/datadog-agent/pkg/util/log => ../../../pkg/util/log + +replace github.com/DataDog/datadog-agent/pkg/util/scrubber => ../../../pkg/util/scrubber + +replace github.com/DataDog/datadog-agent/pkg/version => ../../../pkg/version + +replace github.com/DataDog/datadog-agent/comp/api/api/def => ../../api/api/def + +replace github.com/DataDog/datadog-agent/comp/core/flare/builder => ../../core/flare/builder + +replace github.com/DataDog/datadog-agent/comp/core/flare/types => ../../core/flare/types + +replace github.com/DataDog/datadog-agent/comp/core/secrets => ../../core/secrets + +replace github.com/DataDog/datadog-agent/comp/core/telemetry => ../../core/telemetry + +replace github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../pkg/collector/check/defaults + +replace github.com/DataDog/datadog-agent/pkg/config/env => ../../../pkg/config/env + +replace github.com/DataDog/datadog-agent/pkg/config/mock => ../../../pkg/config/mock + +replace github.com/DataDog/datadog-agent/pkg/config/model => ../../../pkg/config/model + +replace github.com/DataDog/datadog-agent/pkg/config/nodetreemodel => ../../../pkg/config/nodetreemodel + +replace github.com/DataDog/datadog-agent/pkg/config/setup => ../../../pkg/config/setup + +replace github.com/DataDog/datadog-agent/pkg/config/teeconfig => ../../../pkg/config/teeconfig + +replace github.com/DataDog/datadog-agent/pkg/util/executable => ../../../pkg/util/executable + +replace github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../pkg/util/filesystem + +replace github.com/DataDog/datadog-agent/pkg/util/hostname/validate => ../../../pkg/util/hostname/validate + +replace github.com/DataDog/datadog-agent/pkg/util/pointer => ../../../pkg/util/pointer + +replace github.com/DataDog/datadog-agent/pkg/util/system => ../../../pkg/util/system + +replace github.com/DataDog/datadog-agent/pkg/util/system/socket => ../../../pkg/util/system/socket + +replace github.com/DataDog/datadog-agent/pkg/util/testutil => ../../../pkg/util/testutil + +replace github.com/DataDog/datadog-agent/pkg/util/winutil => ../../../pkg/util/winutil + +replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../../pkg/config/structure diff --git a/comp/serializer/logscompression/go.sum b/comp/serializer/logscompression/go.sum new file mode 100644 index 00000000000000..2d48a8a02f7f31 --- /dev/null +++ b/comp/serializer/logscompression/go.sum @@ -0,0 +1,333 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= +github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE= +github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a h1:3Bm7EwfUQUvhNeKIkUct/gl9eod1TcXuj8stxvi/GoI= +github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= +github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v4 v4.24.12 h1:qvePBOk20e0IKA1QXrIIU+jmk+zEiYVVx06WjBRlZo4= +github.com/shirou/gopsutil/v4 v4.24.12/go.mod h1:DCtMPAad2XceTeIAbGyVfycbYQNBGk2P8cvDi7/VN9o= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= +go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.23.0 h1:lIr/gYWQGfTwGcSXWXu4vP5Ws6iqnNEIY+F/aFzCKTg= +go.uber.org/fx v1.23.0/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190529164535-6a60838ec259/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/comp/serializer/logscompression/impl/logscompressionimpl.go b/comp/serializer/logscompression/impl/logscompressionimpl.go new file mode 100644 index 00000000000000..27d57f9fa1e1dd --- /dev/null +++ b/comp/serializer/logscompression/impl/logscompressionimpl.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package logscompressionimpl provides the implementation for the serializer/logscompression component +package logscompressionimpl + +import ( + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" +) + +type component struct{} + +// Provides contains the compression component +type Provides struct { + Comp logscompression.Component +} + +func (*component) NewCompressor(kind string, level int) compression.Compressor { + return selector.NewCompressor(kind, level) +} + +// NewComponent creates a new logscompression component. +func NewComponent() logscompression.Component { + return &component{} +} diff --git a/comp/serializer/metricscompression/def/component.go b/comp/serializer/metricscompression/def/component.go new file mode 100644 index 00000000000000..9c1e6a03fa0c5c --- /dev/null +++ b/comp/serializer/metricscompression/def/component.go @@ -0,0 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package metricscompression provides the component for metrics compression +package metricscompression + +// team: agent-processing-and-routing + +import ( + "github.com/DataDog/datadog-agent/pkg/util/compression" +) + +// Component is the component type. +type Component interface { + compression.Compressor +} diff --git a/comp/serializer/metricscompression/doc.go b/comp/serializer/metricscompression/doc.go new file mode 100644 index 00000000000000..ea66f65b8953ca --- /dev/null +++ b/comp/serializer/metricscompression/doc.go @@ -0,0 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package metrics contains the serializer compression component for metrics +package metrics diff --git a/comp/serializer/compression/fx-mock/fx.go b/comp/serializer/metricscompression/fx-mock/fx.go similarity index 67% rename from comp/serializer/compression/fx-mock/fx.go rename to comp/serializer/metricscompression/fx-mock/fx.go index 93c27867bc9c78..f9cdc61f63cd18 100644 --- a/comp/serializer/compression/fx-mock/fx.go +++ b/comp/serializer/metricscompression/fx-mock/fx.go @@ -9,8 +9,9 @@ package fx import ( - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - compressionnoop "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-noop" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + common "github.com/DataDog/datadog-agent/pkg/util/compression" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -23,7 +24,7 @@ func MockModule() fxutil.Module { ) } -// NewMockCompressor returns a new Mock +// NewMockCompressor returns a noop compressor. func NewMockCompressor() compression.Component { - return compressionnoop.NewComponent().Comp + return selector.NewCompressor(common.NoneKind, 1) } diff --git a/comp/serializer/compression/fx-zstd-nocgo/fx.go b/comp/serializer/metricscompression/fx-otel/fx.go similarity index 67% rename from comp/serializer/compression/fx-zstd-nocgo/fx.go rename to comp/serializer/metricscompression/fx-otel/fx.go index 1666286a9e15c2..c383efd2442993 100644 --- a/comp/serializer/compression/fx-zstd-nocgo/fx.go +++ b/comp/serializer/metricscompression/fx-otel/fx.go @@ -3,11 +3,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package fx provides the fx module for the serializer/compression component +// Package fx provides the fx module that will be used by otel for the serializer/metricscompression component package fx import ( - compressionimpl "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zstd-nocgo" + "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -15,7 +15,7 @@ import ( func Module() fxutil.Module { return fxutil.Component( fxutil.ProvideComponentConstructor( - compressionimpl.NewComponent, + metricscompressionimpl.NewCompressorReqOtel, ), ) } diff --git a/comp/serializer/compression/fx-zstd/fx.go b/comp/serializer/metricscompression/fx/fx.go similarity index 70% rename from comp/serializer/compression/fx-zstd/fx.go rename to comp/serializer/metricscompression/fx/fx.go index 9c7e35a7a3d907..9dd06857e5b53b 100644 --- a/comp/serializer/compression/fx-zstd/fx.go +++ b/comp/serializer/metricscompression/fx/fx.go @@ -3,11 +3,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package fx provides the fx module for the serializer/compression component +// Package fx provides the fx module for the serializer/metricscompression component package fx import ( - compressionimpl "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-zstd" + "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -15,7 +15,7 @@ import ( func Module() fxutil.Module { return fxutil.Component( fxutil.ProvideComponentConstructor( - compressionimpl.NewComponent, + metricscompressionimpl.NewCompressorReq, ), ) } diff --git a/comp/serializer/compression/go.mod b/comp/serializer/metricscompression/go.mod similarity index 69% rename from comp/serializer/compression/go.mod rename to comp/serializer/metricscompression/go.mod index a51ed6f3d4347a..5c70170b4a17bd 100644 --- a/comp/serializer/compression/go.mod +++ b/comp/serializer/metricscompression/go.mod @@ -1,26 +1,12 @@ -module github.com/DataDog/datadog-agent/comp/serializer/compression +module github.com/DataDog/datadog-agent/comp/serializer/metricscompression go 1.22.0 replace ( - github.com/DataDog/datadog-agent/comp/api/api/def => ../../api/api/def github.com/DataDog/datadog-agent/comp/core/config => ../../core/config - github.com/DataDog/datadog-agent/comp/core/flare/builder => ../../core/flare/builder - github.com/DataDog/datadog-agent/comp/core/flare/types => ../../core/flare/types - github.com/DataDog/datadog-agent/comp/core/secrets => ../../core/secrets - github.com/DataDog/datadog-agent/comp/core/telemetry => ../../core/telemetry - github.com/DataDog/datadog-agent/comp/def => ../../def - github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../pkg/collector/check/defaults - github.com/DataDog/datadog-agent/pkg/config/env => ../../../pkg/config/env - github.com/DataDog/datadog-agent/pkg/config/mock => ../../../pkg/config/mock - github.com/DataDog/datadog-agent/pkg/config/model => ../../../pkg/config/model - github.com/DataDog/datadog-agent/pkg/config/nodetreemodel => ../../../pkg/config/nodetreemodel - github.com/DataDog/datadog-agent/pkg/config/setup => ../../../pkg/config/setup - github.com/DataDog/datadog-agent/pkg/config/teeconfig => ../../../pkg/config/teeconfig - github.com/DataDog/datadog-agent/pkg/telemetry => ../../../pkg/telemetry + github.com/DataDog/datadog-agent/pkg/util/compression => ../../../pkg/util/compression + github.com/DataDog/datadog-agent/pkg/util/compression/impl-zlib => ../../../pkg/util/compression/impl-zlib github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../pkg/util/defaultpaths - github.com/DataDog/datadog-agent/pkg/util/executable => ../../../pkg/util/executable - github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../pkg/util/filesystem github.com/DataDog/datadog-agent/pkg/util/fxutil => ../../../pkg/util/fxutil github.com/DataDog/datadog-agent/pkg/util/hostname/validate => ../../../pkg/util/hostname/validate github.com/DataDog/datadog-agent/pkg/util/log => ../../../pkg/util/log @@ -35,17 +21,15 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/core/config v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 - github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 - github.com/DataDog/zstd v1.5.6 - github.com/klauspost/compress v1.17.11 + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 ) require ( - github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect - github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 // indirect @@ -57,6 +41,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.60.1 // indirect @@ -65,6 +50,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/viper v1.14.0 // indirect + github.com/DataDog/zstd v1.5.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -104,6 +90,36 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../../pkg/config/structure +replace github.com/DataDog/datadog-agent/comp/api/api/def => ../../api/api/def + +replace github.com/DataDog/datadog-agent/comp/core/flare/builder => ../../core/flare/builder + +replace github.com/DataDog/datadog-agent/comp/core/flare/types => ../../core/flare/types + +replace github.com/DataDog/datadog-agent/comp/core/secrets => ../../core/secrets + +replace github.com/DataDog/datadog-agent/comp/core/telemetry => ../../core/telemetry + +replace github.com/DataDog/datadog-agent/comp/def => ../../def + +replace github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../pkg/collector/check/defaults + +replace github.com/DataDog/datadog-agent/pkg/config/env => ../../../pkg/config/env + +replace github.com/DataDog/datadog-agent/pkg/config/mock => ../../../pkg/config/mock + +replace github.com/DataDog/datadog-agent/pkg/config/model => ../../../pkg/config/model + +replace github.com/DataDog/datadog-agent/pkg/config/nodetreemodel => ../../../pkg/config/nodetreemodel + +replace github.com/DataDog/datadog-agent/pkg/config/setup => ../../../pkg/config/setup + +replace github.com/DataDog/datadog-agent/pkg/config/teeconfig => ../../../pkg/config/teeconfig + +replace github.com/DataDog/datadog-agent/pkg/util/executable => ../../../pkg/util/executable + +replace github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../pkg/util/filesystem replace github.com/DataDog/datadog-agent/pkg/version => ../../../pkg/version + +replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../../pkg/config/structure diff --git a/comp/serializer/metricscompression/go.sum b/comp/serializer/metricscompression/go.sum new file mode 100644 index 00000000000000..2d48a8a02f7f31 --- /dev/null +++ b/comp/serializer/metricscompression/go.sum @@ -0,0 +1,333 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= +github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE= +github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= +github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a h1:3Bm7EwfUQUvhNeKIkUct/gl9eod1TcXuj8stxvi/GoI= +github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= +github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v4 v4.24.12 h1:qvePBOk20e0IKA1QXrIIU+jmk+zEiYVVx06WjBRlZo4= +github.com/shirou/gopsutil/v4 v4.24.12/go.mod h1:DCtMPAad2XceTeIAbGyVfycbYQNBGk2P8cvDi7/VN9o= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= +go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.23.0 h1:lIr/gYWQGfTwGcSXWXu4vP5Ws6iqnNEIY+F/aFzCKTg= +go.uber.org/fx v1.23.0/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190529164535-6a60838ec259/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/comp/serializer/metricscompression/impl/metricscompression.go b/comp/serializer/metricscompression/impl/metricscompression.go new file mode 100644 index 00000000000000..82aeefaf2b905f --- /dev/null +++ b/comp/serializer/metricscompression/impl/metricscompression.go @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package metricscompressionimpl provides the implementation for the serializer/metricscompression component +package metricscompressionimpl + +import ( + "github.com/DataDog/datadog-agent/comp/core/config" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + zlib "github.com/DataDog/datadog-agent/pkg/util/compression/impl-zlib" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" +) + +// Requires contains the config for Compression +type Requires struct { + Cfg config.Component +} + +// NewCompressorReq returns the compression component +func NewCompressorReq(req Requires) Provides { + return Provides{ + selector.FromConfig(req.Cfg), + } +} + +// Provides contains the compression component +type Provides struct { + Comp metricscompression.Component +} + +// NewCompressorReqOtel returns the compression component for Otel +func NewCompressorReqOtel() Provides { + return Provides{ + Comp: zlib.New(), + } +} diff --git a/go.mod b/go.mod index b3a60dbc7832a0..378de5743cb4f7 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,8 @@ replace ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor => ./comp/otelcol/otlp/components/processor/infraattributesprocessor github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ./comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil => ./comp/otelcol/otlp/testutil - github.com/DataDog/datadog-agent/comp/serializer/compression => ./comp/serializer/compression + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ./comp/serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ./comp/serializer/metricscompression github.com/DataDog/datadog-agent/comp/trace/agent/def => ./comp/trace/agent/def github.com/DataDog/datadog-agent/comp/trace/compression/def => ./comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ./comp/trace/compression/impl-gzip @@ -115,6 +116,10 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/cache => ./pkg/util/cache github.com/DataDog/datadog-agent/pkg/util/cgroups => ./pkg/util/cgroups github.com/DataDog/datadog-agent/pkg/util/common => ./pkg/util/common + github.com/DataDog/datadog-agent/pkg/util/compression => ./pkg/util/compression + github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop => ./pkg/util/compression/impl-noop + github.com/DataDog/datadog-agent/pkg/util/compression/impl-zlib => ./pkg/util/compression/impl-zlib + github.com/DataDog/datadog-agent/pkg/util/compression/selector => ./pkg/util/compression/selector github.com/DataDog/datadog-agent/pkg/util/containers/image => ./pkg/util/containers/image github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ./pkg/util/defaultpaths/ github.com/DataDog/datadog-agent/pkg/util/executable => ./pkg/util/executable @@ -583,6 +588,7 @@ require ( ) require ( + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 github.com/shirou/gopsutil/v4 v4.24.12 go.opentelemetry.io/collector/component/componenttest v0.117.0 ) @@ -650,7 +656,8 @@ require ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient v0.60.1 github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor v0.59.0 github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil v0.57.0-devel.0.20240718200853-81bf3b2e412d - github.com/DataDog/datadog-agent/comp/serializer/compression v0.59.0-rc.6 + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.59.0-rc.6 + github.com/DataDog/datadog-agent/comp/serializer/metricscompression v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/trace/agent/def v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/trace/compression/def v0.60.1 github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.60.1 diff --git a/go.work b/go.work index 54afd4d1c59f38..82b41caf95f922 100644 --- a/go.work +++ b/go.work @@ -42,7 +42,8 @@ use ( comp/otelcol/otlp/components/processor/infraattributesprocessor comp/otelcol/otlp/components/statsprocessor comp/otelcol/otlp/testutil - comp/serializer/compression + comp/serializer/logscompression + comp/serializer/metricscompression comp/trace/agent/def comp/trace/compression/def comp/trace/compression/impl-gzip @@ -102,6 +103,7 @@ use ( pkg/util/cache pkg/util/cgroups pkg/util/common + pkg/util/compression pkg/util/containers/image pkg/util/defaultpaths pkg/util/executable diff --git a/modules.yml b/modules.yml index 3ee6e3c746a058..5fd9ee98df6ed3 100644 --- a/modules.yml +++ b/modules.yml @@ -77,7 +77,9 @@ modules: used_by_otel: true comp/otelcol/otlp/testutil: used_by_otel: true - comp/serializer/compression: + comp/serializer/logscompression: + used_by_otel: true + comp/serializer/metricscompression: used_by_otel: true comp/trace/agent/def: used_by_otel: true @@ -210,6 +212,8 @@ modules: used_by_otel: true pkg/util/common: used_by_otel: true + pkg/util/compression: + used_by_otel: true pkg/util/containers/image: used_by_otel: true pkg/util/defaultpaths: diff --git a/pkg/aggregator/aggregator_test.go b/pkg/aggregator/aggregator_test.go index 2b447281eea771..97fb6e03c9466f 100644 --- a/pkg/aggregator/aggregator_test.go +++ b/pkg/aggregator/aggregator_test.go @@ -29,7 +29,9 @@ import ( orchestratorforwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator" haagent "github.com/DataDog/datadog-agent/comp/haagent/def" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" configmock "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" @@ -794,10 +796,11 @@ type aggregatorDeps struct { Demultiplexer *AgentDemultiplexer OrchestratorFwd orchestratorforwarder.Component EventPlatformFwd eventplatform.Component + Compressor compression.Component } func createAggrDeps(t *testing.T) aggregatorDeps { - deps := fxutil.Test[TestDeps](t, defaultforwarder.MockModule(), core.MockBundle(), compressionmock.MockModule(), haagentmock.Module()) + deps := fxutil.Test[TestDeps](t, defaultforwarder.MockModule(), core.MockBundle(), logscompressionmock.MockModule(), metricscompressionmock.MockModule(), haagentmock.Module()) opts := demuxTestOptions() return aggregatorDeps{ diff --git a/pkg/aggregator/check_sampler_bench_test.go b/pkg/aggregator/check_sampler_bench_test.go index 0691b474d4fbab..e5c6de05432a43 100644 --- a/pkg/aggregator/check_sampler_bench_test.go +++ b/pkg/aggregator/check_sampler_bench_test.go @@ -18,11 +18,12 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" //nolint:revive // TODO(AML) Fix revive linter forwarder "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator/ckey" "github.com/DataDog/datadog-agent/pkg/aggregator/internal/tags" @@ -52,7 +53,7 @@ func benchmarkAddBucket(bucketValue int64, b *testing.B) { options.DontStartForwarders = true sharedForwarder := forwarder.NewDefaultForwarder(pkgconfigsetup.Datadog(), deps.Log, forwarderOpts) orchestratorForwarder := option.New[defaultforwarder.Forwarder](defaultforwarder.NoopForwarder{}) - eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(deps.Hostname)) + eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(deps.Hostname, logscompressionmock.NewMockCompressor())) haAgent := haagentmock.NewMockHaAgent() demux := InitAndStartAgentDemultiplexer(deps.Log, sharedForwarder, &orchestratorForwarder, options, eventPlatformForwarder, haAgent, deps.Compressor, taggerComponent, "hostname") defer demux.Stop(true) diff --git a/pkg/aggregator/demultiplexer_agent.go b/pkg/aggregator/demultiplexer_agent.go index a519383eea73e9..1e39c83ba5dd40 100644 --- a/pkg/aggregator/demultiplexer_agent.go +++ b/pkg/aggregator/demultiplexer_agent.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" orchestratorforwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator" haagent "github.com/DataDog/datadog-agent/comp/haagent/def" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/aggregator/internal/tags" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" diff --git a/pkg/aggregator/demultiplexer_agent_test.go b/pkg/aggregator/demultiplexer_agent_test.go index ce59d5ea8943f6..052797b0648362 100644 --- a/pkg/aggregator/demultiplexer_agent_test.go +++ b/pkg/aggregator/demultiplexer_agent_test.go @@ -25,8 +25,9 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" haagent "github.com/DataDog/datadog-agent/comp/haagent/def" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -114,7 +115,12 @@ func TestDemuxNoAggOptionEnabled(t *testing.T) { func TestDemuxNoAggOptionIsDisabledByDefault(t *testing.T) { opts := demuxTestOptions() - deps := fxutil.Test[TestDeps](t, defaultforwarder.MockModule(), core.MockBundle(), compressionmock.MockModule(), haagentmock.Module()) + deps := fxutil.Test[TestDeps](t, + defaultforwarder.MockModule(), + core.MockBundle(), + haagentmock.Module(), + logscompression.MockModule(), + metricscompression.MockModule()) demux := InitAndStartAgentDemultiplexerForTest(deps, opts, "") require.False(t, demux.Options().EnableNoAggregationPipeline, "the no aggregation pipeline should be disabled by default") @@ -171,8 +177,9 @@ func createDemultiplexerAgentTestDeps(t *testing.T) DemultiplexerAgentTestDeps { core.MockBundle(), orchestratorimpl.MockModule(), eventplatformimpl.MockModule(), + logscompression.MockModule(), + metricscompression.MockModule(), haagentmock.Module(), - compressionmock.MockModule(), fx.Provide(func() tagger.Component { return taggerComponent }), ) } diff --git a/pkg/aggregator/demultiplexer_mock.go b/pkg/aggregator/demultiplexer_mock.go index 6908e63564a504..66a007db4d4f99 100644 --- a/pkg/aggregator/demultiplexer_mock.go +++ b/pkg/aggregator/demultiplexer_mock.go @@ -17,23 +17,25 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" haagent "github.com/DataDog/datadog-agent/comp/haagent/def" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/util/option" ) // TestDeps contains dependencies for InitAndStartAgentDemultiplexerForTest type TestDeps struct { fx.In - Log log.Component - Hostname hostname.Component - SharedForwarder defaultforwarder.Component - Compressor compression.Component - HaAgent haagent.Component + Log log.Component + Hostname hostname.Component + SharedForwarder defaultforwarder.Component + LogsCompression logscompression.Component + MetricsCompression metricscompression.Component + HaAgent haagent.Component } // InitAndStartAgentDemultiplexerForTest initializes an aggregator for tests. func InitAndStartAgentDemultiplexerForTest(deps TestDeps, options AgentDemultiplexerOptions, hostname string) *AgentDemultiplexer { orchestratorForwarder := option.New[defaultforwarder.Forwarder](defaultforwarder.NoopForwarder{}) - eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(deps.Hostname)) - return InitAndStartAgentDemultiplexer(deps.Log, deps.SharedForwarder, &orchestratorForwarder, options, eventPlatformForwarder, deps.HaAgent, deps.Compressor, nooptagger.NewComponent(), hostname) + eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(deps.Hostname, deps.LogsCompression)) + return InitAndStartAgentDemultiplexer(deps.Log, deps.SharedForwarder, &orchestratorForwarder, options, eventPlatformForwarder, deps.HaAgent, deps.MetricsCompression, nooptagger.NewComponent(), hostname) } diff --git a/pkg/aggregator/demultiplexer_serverless.go b/pkg/aggregator/demultiplexer_serverless.go index c77982bc8be1e6..d80128fa426d06 100644 --- a/pkg/aggregator/demultiplexer_serverless.go +++ b/pkg/aggregator/demultiplexer_serverless.go @@ -14,12 +14,12 @@ import ( logimpl "github.com/DataDog/datadog-agent/comp/core/log/impl" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" forwarder "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" "github.com/DataDog/datadog-agent/pkg/aggregator/internal/tags" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" "github.com/DataDog/datadog-agent/pkg/util/hostname" ) @@ -49,7 +49,7 @@ func InitAndStartServerlessDemultiplexer(keysPerDomain map[string][]string, forw logger := logimpl.NewTemporaryLoggerWithoutInit() forwarder := forwarder.NewSyncForwarder(pkgconfigsetup.Datadog(), logger, keysPerDomain, forwarderTimeout) h, _ := hostname.Get(context.Background()) - serializer := serializer.NewSerializer(forwarder, nil, selector.NewCompressor(pkgconfigsetup.Datadog()), pkgconfigsetup.Datadog(), h) + serializer := serializer.NewSerializer(forwarder, nil, selector.FromConfig(pkgconfigsetup.Datadog()), pkgconfigsetup.Datadog(), h) metricSamplePool := metrics.NewMetricSamplePool(MetricSamplePoolBatchSize, utils.IsTelemetryEnabled(pkgconfigsetup.Datadog())) tagsStore := tags.NewStore(pkgconfigsetup.Datadog().GetBool("aggregator_use_tags_store"), "timesampler") diff --git a/pkg/aggregator/demultiplexer_test.go b/pkg/aggregator/demultiplexer_test.go index 72ed3464eb346c..307eceda9e76ba 100644 --- a/pkg/aggregator/demultiplexer_test.go +++ b/pkg/aggregator/demultiplexer_test.go @@ -22,8 +22,9 @@ import ( orchestratorForwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator" orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -295,7 +296,8 @@ func createDemuxDepsWithOrchestratorFwd( orchestratorForwarderImpl.Module(orchestratorParams), eventplatformimpl.Module(eventPlatformParams), eventplatformreceiverimpl.Module(), - compressionmock.MockModule(), + logscompressionmock.MockModule(), + metricscompressionmock.MockModule(), haagentmock.Module(), ) deps := fxutil.Test[internalDemutiplexerDeps](t, modules) @@ -304,6 +306,7 @@ func createDemuxDepsWithOrchestratorFwd( TestDeps: deps.TestDeps, Demultiplexer: InitAndStartAgentDemultiplexer(deps.Log, deps.SharedForwarder, deps.OrchestratorForwarder, opts, deps.Eventplatform, deps.HaAgent, deps.Compressor, nooptagger.NewComponent(), ""), OrchestratorFwd: deps.OrchestratorForwarder, + Compressor: deps.Compressor, EventPlatformFwd: deps.Eventplatform, } } diff --git a/pkg/aggregator/mocksender/mocksender.go b/pkg/aggregator/mocksender/mocksender.go index 86725c3a114f85..c19b806fb8a2c0 100644 --- a/pkg/aggregator/mocksender/mocksender.go +++ b/pkg/aggregator/mocksender/mocksender.go @@ -20,7 +20,8 @@ import ( nooptagger "github.com/DataDog/datadog-agent/comp/core/tagger/impl-noop" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" @@ -42,9 +43,9 @@ func CreateDefaultDemultiplexer() *aggregator.AgentDemultiplexer { log := logimpl.NewTemporaryLoggerWithoutInit() sharedForwarder := defaultforwarder.NewDefaultForwarder(pkgconfigsetup.Datadog(), log, defaultforwarder.NewOptions(pkgconfigsetup.Datadog(), log, nil)) orchestratorForwarder := option.New[defaultforwarder.Forwarder](defaultforwarder.NoopForwarder{}) - eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(hostnameimpl.NewHostnameService())) + eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(hostnameimpl.NewHostnameService(), logscompressionmock.NewMockCompressor())) taggerComponent := nooptagger.NewComponent() - return aggregator.InitAndStartAgentDemultiplexer(log, sharedForwarder, &orchestratorForwarder, opts, eventPlatformForwarder, haagentmock.NewMockHaAgent(), compressionmock.NewMockCompressor(), taggerComponent, "") + return aggregator.InitAndStartAgentDemultiplexer(log, sharedForwarder, &orchestratorForwarder, opts, eventPlatformForwarder, haagentmock.NewMockHaAgent(), metricscompressionmock.NewMockCompressor(), taggerComponent, "") } // NewMockSenderWithSenderManager returns a functional mocked Sender for testing diff --git a/pkg/aggregator/sender_test.go b/pkg/aggregator/sender_test.go index 06bc8ddd82b004..68a02066f5a9a9 100644 --- a/pkg/aggregator/sender_test.go +++ b/pkg/aggregator/sender_test.go @@ -24,7 +24,8 @@ import ( "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/metrics/event" @@ -58,8 +59,8 @@ func testDemux(log log.Component, hostname hostname.Component) *AgentDemultiplex opts := DefaultAgentDemultiplexerOptions() opts.DontStartForwarders = true orchestratorForwarder := option.New[defaultforwarder.Forwarder](defaultforwarder.NoopForwarder{}) - eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(hostname)) - demux := initAgentDemultiplexer(log, NewForwarderTest(log), &orchestratorForwarder, opts, eventPlatformForwarder, haagentmock.NewMockHaAgent(), compressionmock.NewMockCompressor(), nooptagger.NewComponent(), defaultHostname) + eventPlatformForwarder := option.NewPtr[eventplatform.Forwarder](eventplatformimpl.NewNoopEventPlatformForwarder(hostname, logscompressionmock.NewMockCompressor())) + demux := initAgentDemultiplexer(log, NewForwarderTest(log), &orchestratorForwarder, opts, eventPlatformForwarder, haagentmock.NewMockHaAgent(), metricscompressionmock.NewMockCompressor(), nooptagger.NewComponent(), defaultHostname) return demux } diff --git a/pkg/cli/subcommands/check/command.go b/pkg/cli/subcommands/check/command.go index 0f06d0ef3d25ce..e7ec8a6e9b5b21 100644 --- a/pkg/cli/subcommands/check/command.go +++ b/pkg/cli/subcommands/check/command.go @@ -68,7 +68,8 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/inventorychecks/inventorychecksimpl" "github.com/DataDog/datadog-agent/comp/remote-config/rcservice" "github.com/DataDog/datadog-agent/comp/remote-config/rcservicemrf" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/cli/standalone" pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" @@ -178,12 +179,13 @@ func MakeCommand(globalParamsGetter func() GlobalParams) *cobra.Command { autodiscoveryimpl.Module(), forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithNoopForwarder())), inventorychecksimpl.Module(), + logscompression.Module(), + metricscompression.Module(), // inventorychecksimpl depends on a collector and serializer when created to send payload. // Here we just want to collect metadata to be displayed, so we don't need a collector. collector.NoneModule(), fx.Supply(status.NewInformationProvider(statuscollector.Provider{})), fx.Provide(func() serializer.MetricSerializer { return nil }), - compressionfx.Module(), // Initializing the aggregator with a flush interval of 0 (to disable the flush goroutines) demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams(demultiplexerimpl.WithFlushInterval(0))), orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewNoopParams()), diff --git a/pkg/cli/subcommands/dcaflare/command.go b/pkg/cli/subcommands/dcaflare/command.go index bf828f457375fa..ddf4852e7affec 100644 --- a/pkg/cli/subcommands/dcaflare/command.go +++ b/pkg/cli/subcommands/dcaflare/command.go @@ -20,7 +20,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/flare/helpers" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/secrets" - compressionfx "github.com/DataDog/datadog-agent/comp/serializer/compression/fx" "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/settings" settingshttp "github.com/DataDog/datadog-agent/pkg/config/settings/http" @@ -89,7 +88,6 @@ func MakeCommand(globalParamsGetter func() GlobalParams) *cobra.Command { LogParams: log.ForOneShot(LoggerName, DefaultLogLevel, true), }), core.Bundle(), - compressionfx.Module(), ) }, } diff --git a/pkg/clusteragent/admission/validate/kubernetesadmissionevents/kubernetesadmissionevents_test.go b/pkg/clusteragent/admission/validate/kubernetesadmissionevents/kubernetesadmissionevents_test.go index 7e3385828470a7..2ba0bea1dbe309 100644 --- a/pkg/clusteragent/admission/validate/kubernetesadmissionevents/kubernetesadmissionevents_test.go +++ b/pkg/clusteragent/admission/validate/kubernetesadmissionevents/kubernetesadmissionevents_test.go @@ -29,7 +29,8 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" "github.com/DataDog/datadog-agent/pkg/metrics/event" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -267,5 +268,5 @@ func TestKubernetesAdmissionEvents(t *testing.T) { // createDemultiplexer creates a demultiplexer for testing func createDemultiplexer(t *testing.T) demultiplexer.FakeSamplerMock { - return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), compressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) + return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), logscompression.MockModule(), metricscompression.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) } diff --git a/pkg/collector/corechecks/snmp/snmp_test.go b/pkg/collector/corechecks/snmp/snmp_test.go index 2f1095dc520271..cf17abb2e15d9e 100644 --- a/pkg/collector/corechecks/snmp/snmp_test.go +++ b/pkg/collector/corechecks/snmp/snmp_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/snmp/internal/report" "github.com/gosnmp/gosnmp" @@ -53,7 +52,7 @@ type deps struct { } func createDeps(t *testing.T) deps { - return fxutil.Test[deps](t, compressionmock.MockModule(), demultiplexerimpl.MockModule(), defaultforwarder.MockModule(), core.MockBundle()) + return fxutil.Test[deps](t, demultiplexerimpl.MockModule(), defaultforwarder.MockModule(), core.MockBundle()) } func Test_Run_simpleCase(t *testing.T) { diff --git a/pkg/compliance/reporter.go b/pkg/compliance/reporter.go index cf59ee90434894..9f9fd877105715 100644 --- a/pkg/compliance/reporter.go +++ b/pkg/compliance/reporter.go @@ -14,6 +14,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" "github.com/DataDog/datadog-agent/comp/logs/agent/agentimpl" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" configUtils "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/logs/auditor" @@ -38,13 +39,13 @@ type LogReporter struct { } // NewLogReporter instantiates a new log LogReporter -func NewLogReporter(hostname string, sourceName, sourceType string, endpoints *config.Endpoints, dstcontext *client.DestinationsContext) *LogReporter { +func NewLogReporter(hostname string, sourceName, sourceType string, endpoints *config.Endpoints, dstcontext *client.DestinationsContext, compression logscompression.Component) *LogReporter { // setup the auditor auditor := auditor.NewNullAuditor() auditor.Start() // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(4, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, dstcontext, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog()) + pipelineProvider := pipeline.NewProvider(4, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, dstcontext, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog(), compression) pipelineProvider.Start() logSource := sources.NewLogSource( diff --git a/pkg/compliance/status_provider_test.go b/pkg/compliance/status_provider_test.go index 200084ea995b95..69159b985e7c5c 100644 --- a/pkg/compliance/status_provider_test.go +++ b/pkg/compliance/status_provider_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + compressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/logs/client" ) @@ -19,7 +20,7 @@ func TestStatus(t *testing.T) { provider := statusProvider{ agent: &Agent{ opts: AgentOptions{ - Reporter: NewLogReporter("test", "test", "test", &config.Endpoints{}, &client.DestinationsContext{}), + Reporter: NewLogReporter("test", "test", "test", &config.Endpoints{}, &client.DestinationsContext{}, compressionfx.NewMockCompressor()), }, }, } diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index fb7c911c342a71..e0d9ac31ed5656 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -86,6 +86,9 @@ const ( // in situations where we have a high value for `GOMAXPROCS`. DefaultZstdCompressionLevel = 1 + // DefaultGzipCompressionLevel is the default gzip compression level for logs. + DefaultGzipCompressionLevel = 6 + // DefaultLogsSenderBackoffFactor is the default logs sender backoff randomness factor DefaultLogsSenderBackoffFactor = 2.0 @@ -2425,7 +2428,9 @@ func bindEnvAndSetLogsConfigKeys(config pkgconfigmodel.Setup, prefix string) { config.BindEnv(prefix + "dd_url") config.BindEnv(prefix + "additional_endpoints") config.BindEnvAndSetDefault(prefix+"use_compression", true) - config.BindEnvAndSetDefault(prefix+"compression_level", 6) // Default level for the gzip/deflate algorithm + config.BindEnvAndSetDefault(prefix+"compression_kind", "gzip") + config.BindEnvAndSetDefault(prefix+"zstd_compression_level", DefaultZstdCompressionLevel) // Default level for the zstd algorithm + config.BindEnvAndSetDefault(prefix+"compression_level", DefaultGzipCompressionLevel) // Default level for the gzip algorithm config.BindEnvAndSetDefault(prefix+"batch_wait", DefaultBatchWait) config.BindEnvAndSetDefault(prefix+"connection_reset_interval", 0) // in seconds, 0 means disabled config.BindEnvAndSetDefault(prefix+"logs_no_ssl", false) diff --git a/pkg/logs/pipeline/go.mod b/pkg/logs/pipeline/go.mod index c9e22c26f7c33c..82bcba4d07cb96 100644 --- a/pkg/logs/pipeline/go.mod +++ b/pkg/logs/pipeline/go.mod @@ -13,6 +13,8 @@ replace ( github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../comp/def github.com/DataDog/datadog-agent/comp/logs/agent/config => ../../../comp/logs/agent/config + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../../comp/serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../../comp/serializer/metricscompression github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../collector/check/defaults github.com/DataDog/datadog-agent/pkg/config/env => ../../config/env github.com/DataDog/datadog-agent/pkg/config/mock => ../../config/mock @@ -37,6 +39,8 @@ replace ( github.com/DataDog/datadog-agent/pkg/status/health => ../../status/health github.com/DataDog/datadog-agent/pkg/telemetry => ../../telemetry github.com/DataDog/datadog-agent/pkg/util/backoff => ../../util/backoff + github.com/DataDog/datadog-agent/pkg/util/compression => ../../util/compression + github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../util/executable github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../util/filesystem github.com/DataDog/datadog-agent/pkg/util/fxutil => ../../util/fxutil @@ -59,6 +63,7 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/config/setup v0.59.0 github.com/DataDog/datadog-agent/pkg/logs/auditor v0.56.0-rc.3 @@ -71,6 +76,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/sender v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/status/health v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 github.com/DataDog/datadog-agent/pkg/util/startstop v0.56.0-rc.3 github.com/hashicorp/go-multierror v1.1.1 @@ -80,11 +86,15 @@ require ( require ( github.com/DataDog/agent-payload/v5 v5.0.140 // indirect + github.com/DataDog/datadog-agent/comp/core/config v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/structure v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.59.0 // indirect @@ -95,7 +105,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 // indirect @@ -108,6 +118,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect github.com/DataDog/viper v1.14.0 // indirect + github.com/DataDog/zstd v1.5.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/pkg/logs/pipeline/go.sum b/pkg/logs/pipeline/go.sum index c2e16fcb1548f3..c2af38433a1ac2 100644 --- a/pkg/logs/pipeline/go.sum +++ b/pkg/logs/pipeline/go.sum @@ -6,6 +6,8 @@ github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/pipeline/pipeline.go b/pkg/logs/pipeline/pipeline.go index b0136dac860d94..f2e701598f0418 100644 --- a/pkg/logs/pipeline/pipeline.go +++ b/pkg/logs/pipeline/pipeline.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/logs/client" @@ -24,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/logs/processor" "github.com/DataDog/datadog-agent/pkg/logs/sender" "github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface" + compressioncommon "github.com/DataDog/datadog-agent/pkg/util/compression" ) // Pipeline processes and sends messages to the backend @@ -48,7 +50,9 @@ func NewPipeline(outputChan chan *message.Payload, pipelineID int, status statusinterface.Status, hostname hostnameinterface.Component, - cfg pkgconfigmodel.Reader) *Pipeline { + cfg pkgconfigmodel.Reader, + compression logscompression.Component, +) *Pipeline { var senderDoneChan chan *sync.WaitGroup var flushWg *sync.WaitGroup @@ -77,7 +81,7 @@ func NewPipeline(outputChan chan *message.Payload, encoder = processor.RawEncoder } - strategy := getStrategy(strategyInput, senderInput, flushChan, endpoints, serverless, flushWg, pipelineMonitor) + strategy := getStrategy(strategyInput, senderInput, flushChan, endpoints, serverless, flushWg, pipelineMonitor, compression) logsSender = sender.NewSender(cfg, senderInput, outputChan, mainDestinations, pkgconfigsetup.Datadog().GetInt("logs_config.payload_channel_size"), senderDoneChan, flushWg, pipelineMonitor) inputChan := make(chan *message.Message, pkgconfigsetup.Datadog().GetInt("logs_config.message_channel_size")) @@ -156,13 +160,24 @@ func getDestinations(endpoints *config.Endpoints, destinationsContext *client.De } //nolint:revive // TODO(AML) Fix revive linter -func getStrategy(inputChan chan *message.Message, outputChan chan *message.Payload, flushChan chan struct{}, endpoints *config.Endpoints, serverless bool, flushWg *sync.WaitGroup, pipelineMonitor metrics.PipelineMonitor) sender.Strategy { +func getStrategy( + inputChan chan *message.Message, + outputChan chan *message.Payload, + flushChan chan struct{}, + endpoints *config.Endpoints, + serverless bool, + flushWg *sync.WaitGroup, + pipelineMonitor metrics.PipelineMonitor, + compressor logscompression.Component, +) sender.Strategy { if endpoints.UseHTTP || serverless { - encoder := sender.IdentityContentType + var encoder compressioncommon.Compressor + encoder = compressor.NewCompressor(compressioncommon.NoneKind, 0) if endpoints.Main.UseCompression { - encoder = sender.NewGzipContentEncoding(endpoints.Main.CompressionLevel) + encoder = compressor.NewCompressor(endpoints.Main.CompressionKind, endpoints.Main.CompressionLevel) } + return sender.NewBatchStrategy(inputChan, outputChan, flushChan, serverless, flushWg, sender.ArraySerializer, endpoints.BatchWait, endpoints.BatchMaxSize, endpoints.BatchMaxContentSize, "logs", encoder, pipelineMonitor) } - return sender.NewStreamStrategy(inputChan, outputChan, sender.IdentityContentType) + return sender.NewStreamStrategy(inputChan, outputChan, compressor.NewCompressor(compressioncommon.NoneKind, 0)) } diff --git a/pkg/logs/pipeline/provider.go b/pkg/logs/pipeline/provider.go index 15561004e8379a..c7979c06623add 100644 --- a/pkg/logs/pipeline/provider.go +++ b/pkg/logs/pipeline/provider.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/logs/auditor" "github.com/DataDog/datadog-agent/pkg/logs/client" @@ -54,19 +55,41 @@ type provider struct { serverless bool - status statusinterface.Status - hostname hostnameinterface.Component - cfg pkgconfigmodel.Reader + status statusinterface.Status + hostname hostnameinterface.Component + cfg pkgconfigmodel.Reader + compression logscompression.Component } // NewProvider returns a new Provider -func NewProvider(numberOfPipelines int, auditor auditor.Auditor, diagnosticMessageReceiver diagnostic.MessageReceiver, processingRules []*config.ProcessingRule, endpoints *config.Endpoints, destinationsContext *client.DestinationsContext, status statusinterface.Status, hostname hostnameinterface.Component, cfg pkgconfigmodel.Reader) Provider { - return newProvider(numberOfPipelines, auditor, diagnosticMessageReceiver, processingRules, endpoints, destinationsContext, false, status, hostname, cfg) +func NewProvider(numberOfPipelines int, + auditor auditor.Auditor, + diagnosticMessageReceiver diagnostic.MessageReceiver, + processingRules []*config.ProcessingRule, + endpoints *config.Endpoints, + destinationsContext *client.DestinationsContext, + status statusinterface.Status, + hostname hostnameinterface.Component, + cfg pkgconfigmodel.Reader, + compression logscompression.Component, +) Provider { + return newProvider(numberOfPipelines, auditor, diagnosticMessageReceiver, processingRules, endpoints, destinationsContext, false, status, hostname, cfg, compression) } // NewServerlessProvider returns a new Provider in serverless mode -func NewServerlessProvider(numberOfPipelines int, auditor auditor.Auditor, diagnosticMessageReceiver diagnostic.MessageReceiver, processingRules []*config.ProcessingRule, endpoints *config.Endpoints, destinationsContext *client.DestinationsContext, status statusinterface.Status, hostname hostnameinterface.Component, cfg pkgconfigmodel.Reader) Provider { - return newProvider(numberOfPipelines, auditor, diagnosticMessageReceiver, processingRules, endpoints, destinationsContext, true, status, hostname, cfg) +func NewServerlessProvider(numberOfPipelines int, + auditor auditor.Auditor, + diagnosticMessageReceiver diagnostic.MessageReceiver, + processingRules []*config.ProcessingRule, + endpoints *config.Endpoints, + destinationsContext *client.DestinationsContext, + status statusinterface.Status, + hostname hostnameinterface.Component, + cfg pkgconfigmodel.Reader, + compression logscompression.Component, +) Provider { + + return newProvider(numberOfPipelines, auditor, diagnosticMessageReceiver, processingRules, endpoints, destinationsContext, true, status, hostname, cfg, compression) } // NewMockProvider creates a new provider that will not provide any pipelines. @@ -74,7 +97,18 @@ func NewMockProvider() Provider { return &provider{} } -func newProvider(numberOfPipelines int, auditor auditor.Auditor, diagnosticMessageReceiver diagnostic.MessageReceiver, processingRules []*config.ProcessingRule, endpoints *config.Endpoints, destinationsContext *client.DestinationsContext, serverless bool, status statusinterface.Status, hostname hostnameinterface.Component, cfg pkgconfigmodel.Reader) Provider { +func newProvider(numberOfPipelines int, + auditor auditor.Auditor, + diagnosticMessageReceiver diagnostic.MessageReceiver, + processingRules []*config.ProcessingRule, + endpoints *config.Endpoints, + destinationsContext *client.DestinationsContext, + serverless bool, + status statusinterface.Status, + hostname hostnameinterface.Component, + cfg pkgconfigmodel.Reader, + compression logscompression.Component, +) Provider { return &provider{ numberOfPipelines: numberOfPipelines, auditor: auditor, @@ -88,6 +122,7 @@ func newProvider(numberOfPipelines int, auditor auditor.Auditor, diagnosticMessa status: status, hostname: hostname, cfg: cfg, + compression: compression, } } @@ -97,7 +132,7 @@ func (p *provider) Start() { p.outputChan = p.auditor.Channel() for i := 0; i < p.numberOfPipelines; i++ { - pipeline := NewPipeline(p.outputChan, p.processingRules, p.endpoints, p.destinationsContext, p.diagnosticMessageReceiver, p.serverless, i, p.status, p.hostname, p.cfg) + pipeline := NewPipeline(p.outputChan, p.processingRules, p.endpoints, p.destinationsContext, p.diagnosticMessageReceiver, p.serverless, i, p.status, p.hostname, p.cfg, p.compression) pipeline.Start() p.pipelines = append(p.pipelines, pipeline) } diff --git a/pkg/logs/pipeline/provider_test.go b/pkg/logs/pipeline/provider_test.go index 8f950c43dd6f39..c9aadc09139f56 100644 --- a/pkg/logs/pipeline/provider_test.go +++ b/pkg/logs/pipeline/provider_test.go @@ -13,6 +13,7 @@ import ( "go.uber.org/atomic" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + compressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/logs/auditor" "github.com/DataDog/datadog-agent/pkg/status/health" ) @@ -31,6 +32,7 @@ func (suite *ProviderTestSuite) SetupTest() { pipelines: []*Pipeline{}, endpoints: config.NewEndpoints(config.Endpoint{}, nil, true, false), currentPipelineIndex: atomic.NewUint32(0), + compression: compressionfx.NewMockCompressor(), } } diff --git a/pkg/logs/sender/batch_strategy.go b/pkg/logs/sender/batch_strategy.go index 47ccbaf86009be..9d1c28eafff775 100644 --- a/pkg/logs/sender/batch_strategy.go +++ b/pkg/logs/sender/batch_strategy.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/logs/metrics" "github.com/DataDog/datadog-agent/pkg/telemetry" + "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -31,12 +32,12 @@ type batchStrategy struct { flushWg *sync.WaitGroup buffer *MessageBuffer // pipelineName provides a name for the strategy to differentiate it from other instances in other internal pipelines - pipelineName string - serializer Serializer - batchWait time.Duration - contentEncoding ContentEncoding - stopChan chan struct{} // closed when the goroutine has finished - clock clock.Clock + pipelineName string + serializer Serializer + batchWait time.Duration + compression compression.Compressor + stopChan chan struct{} // closed when the goroutine has finished + clock clock.Clock // Telemtry pipelineMonitor metrics.PipelineMonitor @@ -54,9 +55,9 @@ func NewBatchStrategy(inputChan chan *message.Message, maxBatchSize int, maxContentSize int, pipelineName string, - contentEncoding ContentEncoding, + compression compression.Compressor, pipelineMonitor metrics.PipelineMonitor) Strategy { - return newBatchStrategyWithClock(inputChan, outputChan, flushChan, serverless, flushWg, serializer, batchWait, maxBatchSize, maxContentSize, pipelineName, clock.New(), contentEncoding, pipelineMonitor) + return newBatchStrategyWithClock(inputChan, outputChan, flushChan, serverless, flushWg, serializer, batchWait, maxBatchSize, maxContentSize, pipelineName, clock.New(), compression, pipelineMonitor) } func newBatchStrategyWithClock(inputChan chan *message.Message, @@ -70,7 +71,7 @@ func newBatchStrategyWithClock(inputChan chan *message.Message, maxContentSize int, pipelineName string, clock clock.Clock, - contentEncoding ContentEncoding, + compression compression.Compressor, pipelineMonitor metrics.PipelineMonitor) Strategy { return &batchStrategy{ @@ -82,7 +83,7 @@ func newBatchStrategyWithClock(inputChan chan *message.Message, buffer: NewMessageBuffer(maxBatchSize, maxContentSize), serializer: serializer, batchWait: batchWait, - contentEncoding: contentEncoding, + compression: compression, stopChan: make(chan struct{}), pipelineName: pipelineName, clock: clock, @@ -168,7 +169,7 @@ func (s *batchStrategy) sendMessages(messages []*message.Message, outputChan cha serializedMessage := s.serializer.Serialize(messages) log.Debugf("Send messages for pipeline %s (msg_count:%d, content_size=%d, avg_msg_size=%.2f)", s.pipelineName, len(messages), len(serializedMessage), float64(len(serializedMessage))/float64(len(messages))) - encodedPayload, err := s.contentEncoding.encode(serializedMessage) + encodedPayload, err := s.compression.Compress(serializedMessage) if err != nil { log.Warn("Encoding failed - dropping payload", err) s.utilization.Stop() @@ -183,7 +184,7 @@ func (s *batchStrategy) sendMessages(messages []*message.Message, outputChan cha p := &message.Payload{ Messages: messages, Encoded: encodedPayload, - Encoding: s.contentEncoding.name(), + Encoding: s.compression.ContentEncoding(), UnencodedSize: len(serializedMessage), } s.utilization.Stop() diff --git a/pkg/logs/sender/batch_strategy_test.go b/pkg/logs/sender/batch_strategy_test.go index 34cb6be7aa4e9e..f5d3c1b27c4647 100644 --- a/pkg/logs/sender/batch_strategy_test.go +++ b/pkg/logs/sender/batch_strategy_test.go @@ -12,8 +12,10 @@ import ( "github.com/benbjohnson/clock" "github.com/stretchr/testify/assert" + compressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/logs/metrics" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) func TestBatchStrategySendsPayloadWhenBufferIsFull(t *testing.T) { @@ -21,7 +23,7 @@ func TestBatchStrategySendsPayloadWhenBufferIsFull(t *testing.T) { output := make(chan *message.Payload) flushChan := make(chan struct{}) - s := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, 100*time.Millisecond, 2, 2, "test", &identityContentType{}, metrics.NewNoopPipelineMonitor("")) + s := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, 100*time.Millisecond, 2, 2, "test", compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1), metrics.NewNoopPipelineMonitor("")) s.Start() message1 := message.NewMessage([]byte("a"), nil, "", 0) @@ -53,7 +55,7 @@ func TestBatchStrategySendsPayloadWhenBufferIsOutdated(t *testing.T) { timerInterval := 100 * time.Millisecond clk := clock.NewMock() - s := newBatchStrategyWithClock(input, output, flushChan, false, nil, LineSerializer, timerInterval, 100, 100, "test", clk, &identityContentType{}, metrics.NewNoopPipelineMonitor("")) + s := newBatchStrategyWithClock(input, output, flushChan, false, nil, LineSerializer, timerInterval, 100, 100, "test", clk, compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1), metrics.NewNoopPipelineMonitor("")) s.Start() for round := 0; round < 3; round++ { @@ -78,7 +80,7 @@ func TestBatchStrategySendsPayloadWhenClosingInput(t *testing.T) { flushChan := make(chan struct{}) clk := clock.NewMock() - s := newBatchStrategyWithClock(input, output, flushChan, false, nil, LineSerializer, 100*time.Millisecond, 2, 2, "test", clk, &identityContentType{}, metrics.NewNoopPipelineMonitor("")) + s := newBatchStrategyWithClock(input, output, flushChan, false, nil, LineSerializer, 100*time.Millisecond, 2, 2, "test", clk, compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1), metrics.NewNoopPipelineMonitor("")) s.Start() message := message.NewMessage([]byte("a"), nil, "", 0) @@ -103,7 +105,7 @@ func TestBatchStrategyShouldNotBlockWhenStoppingGracefully(t *testing.T) { output := make(chan *message.Payload) flushChan := make(chan struct{}) - s := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, 100*time.Millisecond, 2, 2, "test", &identityContentType{}, metrics.NewNoopPipelineMonitor("")) + s := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, 100*time.Millisecond, 2, 2, "test", compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1), metrics.NewNoopPipelineMonitor("")) s.Start() message := message.NewMessage([]byte{}, nil, "", 0) @@ -127,7 +129,7 @@ func TestBatchStrategySynchronousFlush(t *testing.T) { // batch size is large so it will not flush until we trigger it manually // flush time is large so it won't automatically trigger during this test - strategy := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, time.Hour, 100, 100, "test", &identityContentType{}, metrics.NewNoopPipelineMonitor("")) + strategy := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, time.Hour, 100, 100, "test", compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1), metrics.NewNoopPipelineMonitor("")) strategy.Start() // all of these messages will get buffered @@ -172,7 +174,7 @@ func TestBatchStrategyFlushChannel(t *testing.T) { // batch size is large so it will not flush until we trigger it manually // flush time is large so it won't automatically trigger during this test - strategy := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, time.Hour, 100, 100, "test", &identityContentType{}, metrics.NewNoopPipelineMonitor("")) + strategy := NewBatchStrategy(input, output, flushChan, false, nil, LineSerializer, time.Hour, 100, 100, "test", compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1), metrics.NewNoopPipelineMonitor("")) strategy.Start() // all of these messages will get buffered diff --git a/pkg/logs/sender/content_encoding.go b/pkg/logs/sender/content_encoding.go deleted file mode 100644 index 75f59587ff5129..00000000000000 --- a/pkg/logs/sender/content_encoding.go +++ /dev/null @@ -1,73 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package sender - -import ( - "bytes" - "compress/gzip" -) - -// ContentEncoding encodes the payload -type ContentEncoding interface { - name() string - encode(payload []byte) ([]byte, error) -} - -// IdentityContentType encodes the payload using the identity function -var IdentityContentType ContentEncoding = &identityContentType{} - -type identityContentType struct{} - -func (c *identityContentType) name() string { - return "identity" -} - -func (c *identityContentType) encode(payload []byte) ([]byte, error) { - return payload, nil -} - -// GzipContentEncoding encodes the payload using gzip algorithm -type GzipContentEncoding struct { - level int -} - -// NewGzipContentEncoding creates a new Gzip content type -func NewGzipContentEncoding(level int) *GzipContentEncoding { - if level < gzip.NoCompression { - level = gzip.NoCompression - } else if level > gzip.BestCompression { - level = gzip.BestCompression - } - - return &GzipContentEncoding{ - level, - } -} - -func (c *GzipContentEncoding) name() string { - return "gzip" -} - -func (c *GzipContentEncoding) encode(payload []byte) ([]byte, error) { - var compressedPayload bytes.Buffer - gzipWriter, err := gzip.NewWriterLevel(&compressedPayload, c.level) - if err != nil { - return nil, err - } - _, err = gzipWriter.Write(payload) - if err != nil { - return nil, err - } - err = gzipWriter.Flush() - if err != nil { - return nil, err - } - err = gzipWriter.Close() - if err != nil { - return nil, err - } - return compressedPayload.Bytes(), nil -} diff --git a/pkg/logs/sender/content_encoding_test.go b/pkg/logs/sender/content_encoding_test.go deleted file mode 100644 index ef91096bfd7f27..00000000000000 --- a/pkg/logs/sender/content_encoding_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package sender - -import ( - "bytes" - "compress/gzip" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestIdentityContentType(t *testing.T) { - payload := []byte("my payload") - - encodedPayload, err := IdentityContentType.encode(payload) - assert.Nil(t, err) - - assert.Equal(t, payload, encodedPayload) -} - -func TestIdentityContentTypeName(t *testing.T) { - assert.Equal(t, IdentityContentType.name(), "identity") -} - -func TestGzipContentEncoding(t *testing.T) { - payload := []byte("my payload") - - encodedPayload, err := NewGzipContentEncoding(gzip.BestCompression).encode(payload) - assert.Nil(t, err) - - decompressedPayload, err := decompress(encodedPayload) - assert.Nil(t, err) - - assert.Equal(t, payload, decompressedPayload) -} - -func TestGzipContentEncodingName(t *testing.T) { - assert.Equal(t, NewGzipContentEncoding(gzip.BestCompression).name(), "gzip") -} - -func decompress(payload []byte) ([]byte, error) { - reader, err := gzip.NewReader(bytes.NewReader(payload)) - if err != nil { - return nil, err - } - - var buffer bytes.Buffer - _, err = buffer.ReadFrom(reader) - if err != nil { - return nil, err - } - - return buffer.Bytes(), nil -} diff --git a/pkg/logs/sender/go.mod b/pkg/logs/sender/go.mod index babe1a87f98dc2..8d40fbcfe7b95c 100644 --- a/pkg/logs/sender/go.mod +++ b/pkg/logs/sender/go.mod @@ -3,6 +3,7 @@ module github.com/DataDog/datadog-agent/pkg/logs/sender go 1.22.0 replace ( + github.com/DataDog/datadog-agent/cmd/agent/common/path => ../../../cmd/agent/common/path github.com/DataDog/datadog-agent/comp/api/api/def => ../../../comp/api/api/def github.com/DataDog/datadog-agent/comp/core/config => ../../../comp/core/config github.com/DataDog/datadog-agent/comp/core/flare/builder => ../../../comp/core/flare/builder @@ -13,6 +14,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../comp/def github.com/DataDog/datadog-agent/comp/logs/agent/config => ../../../comp/logs/agent/config + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ../../../comp/serializer/logscompression github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../collector/check/defaults github.com/DataDog/datadog-agent/pkg/config/env => ../../config/env github.com/DataDog/datadog-agent/pkg/config/mock => ../../config/mock @@ -31,6 +33,8 @@ replace ( github.com/DataDog/datadog-agent/pkg/logs/util/testutils => ../util/testutils github.com/DataDog/datadog-agent/pkg/telemetry => ../../telemetry github.com/DataDog/datadog-agent/pkg/util/backoff => ../../util/backoff + github.com/DataDog/datadog-agent/pkg/util/compression => ../../util/compression + github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../../util/executable github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../util/filesystem github.com/DataDog/datadog-agent/pkg/util/fxutil => ../../util/fxutil @@ -51,6 +55,7 @@ replace ( require ( github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/logs/client v0.56.0-rc.3 @@ -59,15 +64,19 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 github.com/benbjohnson/clock v1.3.5 github.com/stretchr/testify v1.10.0 ) require ( + github.com/DataDog/datadog-agent/comp/core/config v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.59.0 // indirect @@ -79,7 +88,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/backoff v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 // indirect @@ -91,6 +100,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.60.1 // indirect github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/DataDog/viper v1.14.0 // indirect + github.com/DataDog/zstd v1.5.6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/logs/sender/go.sum b/pkg/logs/sender/go.sum index 7f8605c770c275..17f4714db27899 100644 --- a/pkg/logs/sender/go.sum +++ b/pkg/logs/sender/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/viper v1.14.0 h1:dIjTe/uJiah+QFqFZ+MXeqgmUvWhg37l37ZxFWxr3is= github.com/DataDog/viper v1.14.0/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/sender/stream_strategy.go b/pkg/logs/sender/stream_strategy.go index e31d455e5e12a3..32cee91a95039d 100644 --- a/pkg/logs/sender/stream_strategy.go +++ b/pkg/logs/sender/stream_strategy.go @@ -7,6 +7,7 @@ package sender import ( "github.com/DataDog/datadog-agent/pkg/logs/message" + "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -14,19 +15,19 @@ import ( // that Message's Content. This is used for TCP destinations, which stream the output // without batching multiple messages together. type streamStrategy struct { - inputChan chan *message.Message - outputChan chan *message.Payload - contentEncoding ContentEncoding - done chan struct{} + inputChan chan *message.Message + outputChan chan *message.Payload + compression compression.Compressor + done chan struct{} } // NewStreamStrategy creates a new stream strategy -func NewStreamStrategy(inputChan chan *message.Message, outputChan chan *message.Payload, contentEncoding ContentEncoding) Strategy { +func NewStreamStrategy(inputChan chan *message.Message, outputChan chan *message.Payload, compression compression.Compressor) Strategy { return &streamStrategy{ - inputChan: inputChan, - outputChan: outputChan, - contentEncoding: contentEncoding, - done: make(chan struct{}), + inputChan: inputChan, + outputChan: outputChan, + compression: compression, + done: make(chan struct{}), } } @@ -38,7 +39,7 @@ func (s *streamStrategy) Start() { msg.Origin.LogSource.LatencyStats.Add(msg.GetLatency()) } - encodedPayload, err := s.contentEncoding.encode(msg.GetContent()) + encodedPayload, err := s.compression.Compress(msg.GetContent()) if err != nil { log.Warn("Encoding failed - dropping payload", err) return @@ -47,7 +48,7 @@ func (s *streamStrategy) Start() { s.outputChan <- &message.Payload{ Messages: []*message.Message{msg}, Encoded: encodedPayload, - Encoding: s.contentEncoding.name(), + Encoding: s.compression.ContentEncoding(), UnencodedSize: len(msg.GetContent()), } } diff --git a/pkg/logs/sender/stream_strategy_test.go b/pkg/logs/sender/stream_strategy_test.go index d535dafb169d1c..f46f078f2f4523 100644 --- a/pkg/logs/sender/stream_strategy_test.go +++ b/pkg/logs/sender/stream_strategy_test.go @@ -10,14 +10,16 @@ import ( "github.com/stretchr/testify/assert" + compressionfx "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/logs/message" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) func TestStreamStrategy(t *testing.T) { input := make(chan *message.Message) output := make(chan *message.Payload) - s := NewStreamStrategy(input, output, IdentityContentType) + s := NewStreamStrategy(input, output, compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1)) s.Start() content := []byte("a") @@ -45,7 +47,7 @@ func TestStreamStrategyShouldNotBlockWhenForceStopping(_ *testing.T) { input := make(chan *message.Message) output := make(chan *message.Payload) - s := NewStreamStrategy(input, output, IdentityContentType) + s := NewStreamStrategy(input, output, compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1)) message := message.NewMessage([]byte{}, nil, "", 0) go func() { @@ -60,7 +62,7 @@ func TestStreamStrategyShouldNotBlockWhenStoppingGracefully(t *testing.T) { input := make(chan *message.Message) output := make(chan *message.Payload) - s := NewStreamStrategy(input, output, IdentityContentType) + s := NewStreamStrategy(input, output, compressionfx.NewMockCompressor().NewCompressor(compression.NoneKind, 1)) message := message.NewMessage([]byte{}, nil, "", 0) go func() { diff --git a/pkg/security/module/cws.go b/pkg/security/module/cws.go index 3c1baa99828aa2..227ccfa1dea706 100644 --- a/pkg/security/module/cws.go +++ b/pkg/security/module/cws.go @@ -18,6 +18,7 @@ import ( "go.uber.org/atomic" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/eventmonitor" "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/events" @@ -64,7 +65,7 @@ type CWSConsumer struct { } // NewCWSConsumer initializes the module with options -func NewCWSConsumer(evm *eventmonitor.EventMonitor, cfg *config.RuntimeSecurityConfig, wmeta workloadmeta.Component, opts Opts) (*CWSConsumer, error) { +func NewCWSConsumer(evm *eventmonitor.EventMonitor, cfg *config.RuntimeSecurityConfig, wmeta workloadmeta.Component, opts Opts, compression compression.Component) (*CWSConsumer, error) { crtelemcfg := telemetry.ContainersRunningTelemetryConfig{ RuntimeEnabled: cfg.RuntimeEnabled, FIMEnabled: cfg.FIMEnabled, @@ -84,7 +85,7 @@ func NewCWSConsumer(evm *eventmonitor.EventMonitor, cfg *config.RuntimeSecurityC family, address := config.GetFamilyAddress(cfg.SocketPath) - apiServer, err := NewAPIServer(cfg, evm.Probe, opts.MsgSender, evm.StatsdClient, selfTester) + apiServer, err := NewAPIServer(cfg, evm.Probe, opts.MsgSender, evm.StatsdClient, selfTester, compression) if err != nil { return nil, err } diff --git a/pkg/security/module/msg_sender.go b/pkg/security/module/msg_sender.go index eddd3ebf67f0d2..a2a4292b9849f2 100644 --- a/pkg/security/module/msg_sender.go +++ b/pkg/security/module/msg_sender.go @@ -9,6 +9,7 @@ package module import ( "fmt" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/security/common" "github.com/DataDog/datadog-agent/pkg/security/proto/api" @@ -72,7 +73,7 @@ func (ds *DirectMsgSender) Send(msg *api.SecurityEventMessage, _ func(*api.Secur } // NewDirectMsgSender returns a new direct sender -func NewDirectMsgSender(stopper startstop.Stopper) (*DirectMsgSender, error) { +func NewDirectMsgSender(stopper startstop.Stopper, compression compression.Component) (*DirectMsgSender, error) { useSecRuntimeTrack := pkgconfigsetup.SystemProbe().GetBool("runtime_security_config.use_secruntime_track") endpoints, destinationsCtx, err := common.NewLogContextRuntime(useSecRuntimeTrack) @@ -86,7 +87,7 @@ func NewDirectMsgSender(stopper startstop.Stopper) (*DirectMsgSender, error) { // we set the hostname to the empty string to take advantage of the out of the box message hostname // resolution - reporter, err := reporter.NewCWSReporter("", stopper, endpoints, destinationsCtx) + reporter, err := reporter.NewCWSReporter("", stopper, endpoints, destinationsCtx, compression) if err != nil { return nil, fmt.Errorf("failed to create direct reporter: %w", err) } diff --git a/pkg/security/module/server.go b/pkg/security/module/server.go index 2bb7dffdf4249b..6faa5fe5d237c0 100644 --- a/pkg/security/module/server.go +++ b/pkg/security/module/server.go @@ -22,6 +22,7 @@ import ( "go.uber.org/atomic" "github.com/DataDog/datadog-agent/comp/core/tagger/types" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/security/common" "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/events" @@ -570,7 +571,7 @@ func (a *APIServer) getGlobalTags() []string { } // NewAPIServer returns a new gRPC event server -func NewAPIServer(cfg *config.RuntimeSecurityConfig, probe *sprobe.Probe, msgSender MsgSender, client statsd.ClientInterface, selfTester *selftests.SelfTester) (*APIServer, error) { +func NewAPIServer(cfg *config.RuntimeSecurityConfig, probe *sprobe.Probe, msgSender MsgSender, client statsd.ClientInterface, selfTester *selftests.SelfTester, compression compression.Component) (*APIServer, error) { stopper := startstop.NewSerialStopper() as := &APIServer{ @@ -593,7 +594,7 @@ func NewAPIServer(cfg *config.RuntimeSecurityConfig, probe *sprobe.Probe, msgSen if as.msgSender == nil { if cfg.SendEventFromSystemProbe { - msgSender, err := NewDirectMsgSender(stopper) + msgSender, err := NewDirectMsgSender(stopper, compression) if err != nil { log.Errorf("failed to setup direct reporter: %v", err) } else { diff --git a/pkg/security/reporter/reporter.go b/pkg/security/reporter/reporter.go index 227f94ac456edd..49ebfbf900aa0c 100644 --- a/pkg/security/reporter/reporter.go +++ b/pkg/security/reporter/reporter.go @@ -12,6 +12,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" "github.com/DataDog/datadog-agent/comp/logs/agent/agentimpl" logsconfig "github.com/DataDog/datadog-agent/comp/logs/agent/config" + compression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/logs/auditor" "github.com/DataDog/datadog-agent/pkg/logs/client" @@ -41,18 +42,18 @@ func (r *RuntimeReporter) ReportRaw(content []byte, service string, tags ...stri } // NewCWSReporter returns a new CWS reported based on the fields necessary to communicate with the intake -func NewCWSReporter(hostname string, stopper startstop.Stopper, endpoints *logsconfig.Endpoints, context *client.DestinationsContext) (seccommon.RawReporter, error) { - return newReporter(hostname, stopper, "runtime-security-agent", "runtime-security", endpoints, context) +func NewCWSReporter(hostname string, stopper startstop.Stopper, endpoints *logsconfig.Endpoints, context *client.DestinationsContext, compression compression.Component) (seccommon.RawReporter, error) { + return newReporter(hostname, stopper, "runtime-security-agent", "runtime-security", endpoints, context, compression) } -func newReporter(hostname string, stopper startstop.Stopper, sourceName, sourceType string, endpoints *logsconfig.Endpoints, context *client.DestinationsContext) (seccommon.RawReporter, error) { +func newReporter(hostname string, stopper startstop.Stopper, sourceName, sourceType string, endpoints *logsconfig.Endpoints, context *client.DestinationsContext, compression compression.Component) (seccommon.RawReporter, error) { // setup the auditor auditor := auditor.NewNullAuditor() auditor.Start() stopper.Add(auditor) // setup the pipeline provider that provides pairs of processor and sender - pipelineProvider := pipeline.NewProvider(4, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, context, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog()) + pipelineProvider := pipeline.NewProvider(4, auditor, &diagnostic.NoopMessageReceiver{}, nil, endpoints, context, agentimpl.NewStatusProvider(), hostnameimpl.NewHostnameService(), pkgconfigsetup.Datadog(), compression) pipelineProvider.Start() stopper.Add(pipelineProvider) diff --git a/pkg/security/tests/module_tester_linux.go b/pkg/security/tests/module_tester_linux.go index e58a1aaf73ea18..ccf8481a2ccaf1 100644 --- a/pkg/security/tests/module_tester_linux.go +++ b/pkg/security/tests/module_tester_linux.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/assert" "golang.org/x/sys/unix" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/impl" ebpftelemetry "github.com/DataDog/datadog-agent/pkg/ebpf/telemetry" "github.com/DataDog/datadog-agent/pkg/eventmonitor" secconfig "github.com/DataDog/datadog-agent/pkg/security/config" @@ -751,7 +752,8 @@ func newTestModuleWithOnDemandProbes(t testing.TB, onDemandHooks []rules.OnDeman if !opts.staticOpts.disableRuntimeSecurity { msgSender := newFakeMsgSender(testMod) - cws, err := module.NewCWSConsumer(testMod.eventMonitor, secconfig.RuntimeSecurity, nil, module.Opts{EventSender: testMod, MsgSender: msgSender}) + compression := logscompression.NewComponent() + cws, err := module.NewCWSConsumer(testMod.eventMonitor, secconfig.RuntimeSecurity, nil, module.Opts{EventSender: testMod, MsgSender: msgSender}, compression) if err != nil { return nil, fmt.Errorf("failed to create module: %w", err) } diff --git a/pkg/security/tests/module_tester_windows.go b/pkg/security/tests/module_tester_windows.go index 25876dc8136920..8a8cb70e123b08 100644 --- a/pkg/security/tests/module_tester_windows.go +++ b/pkg/security/tests/module_tester_windows.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/go-multierror" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/impl" "github.com/DataDog/datadog-agent/pkg/eventmonitor" secconfig "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/module" @@ -171,7 +172,8 @@ func newTestModule(t testing.TB, macroDefs []*rules.MacroDefinition, ruleDefs [] var ruleSetloadedErr *multierror.Error if !opts.staticOpts.disableRuntimeSecurity { - cws, err := module.NewCWSConsumer(testMod.eventMonitor, secconfig.RuntimeSecurity, nil, module.Opts{EventSender: testMod}) + compression := logscompression.NewComponent() + cws, err := module.NewCWSConsumer(testMod.eventMonitor, secconfig.RuntimeSecurity, nil, module.Opts{EventSender: testMod}, compression) if err != nil { return nil, fmt.Errorf("failed to create module: %w", err) } diff --git a/pkg/serializer/go.mod b/pkg/serializer/go.mod index d0cafe003cfd03..8cf27f84c67209 100644 --- a/pkg/serializer/go.mod +++ b/pkg/serializer/go.mod @@ -16,7 +16,7 @@ replace ( github.com/DataDog/datadog-agent/comp/def => ../../comp/def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../comp/forwarder/defaultforwarder github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface => ../../comp/forwarder/orchestrator/orchestratorinterface - github.com/DataDog/datadog-agent/comp/serializer/compression => ../../comp/serializer/compression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ../../comp/serializer/metricscompression github.com/DataDog/datadog-agent/pkg/aggregator/ckey => ../aggregator/ckey github.com/DataDog/datadog-agent/pkg/api => ../api github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../collector/check/defaults @@ -43,6 +43,7 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/backoff => ../util/backoff/ github.com/DataDog/datadog-agent/pkg/util/buf => ../util/buf/ github.com/DataDog/datadog-agent/pkg/util/common => ../util/common + github.com/DataDog/datadog-agent/pkg/util/compression => ../util/compression github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ../util/executable/ github.com/DataDog/datadog-agent/pkg/util/filesystem => ../util/filesystem/ @@ -68,7 +69,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/config v0.57.1 github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface v0.56.0-rc.3 - github.com/DataDog/datadog-agent/comp/serializer/compression v0.56.0-rc.3 + github.com/DataDog/datadog-agent/comp/serializer/metricscompression v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/aggregator/ckey v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 @@ -77,6 +78,7 @@ require ( github.com/DataDog/datadog-agent/pkg/tagger/types v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/tagset v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/json v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 @@ -90,14 +92,14 @@ require ( ) require ( - github.com/DataDog/datadog-agent/comp/core/flare/builder v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/types v0.57.1 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.62.0-rc.1 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect - github.com/DataDog/datadog-agent/comp/def v0.57.1 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/api v0.57.1 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect @@ -113,7 +115,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/common v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect - github.com/DataDog/datadog-agent/pkg/util/fxutil v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect github.com/DataDog/datadog-agent/pkg/util/http v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 // indirect diff --git a/pkg/serializer/internal/metrics/events_test.go b/pkg/serializer/internal/metrics/events_test.go index 9e029147072f51..2ce3cb740bd6a6 100644 --- a/pkg/serializer/internal/metrics/events_test.go +++ b/pkg/serializer/internal/metrics/events_test.go @@ -21,12 +21,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" configmock "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics/event" "github.com/DataDog/datadog-agent/pkg/serializer/internal/stream" taggertypes "github.com/DataDog/datadog-agent/pkg/tagger/types" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) func TestMarshal(t *testing.T) { @@ -173,8 +173,8 @@ func TestEventsSeveralPayloadsCreateSingleMarshaler(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -197,8 +197,8 @@ func TestEventsSeveralPayloadsCreateMarshalersBySourceType(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -259,8 +259,8 @@ func assertEqualEventsToMarshalJSON(t *testing.T, events Events) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -371,7 +371,8 @@ func BenchmarkCreateSingleMarshalerOneEventBySource(b *testing.B) { func benchmarkCreateSingleMarshaler(b *testing.B, createEvents func(numberOfItem int) Events) { runBenchmark(b, func(b *testing.B, numberOfItem int) { cfg := configmock.New(b) - payloadBuilder := stream.NewJSONPayloadBuilder(true, cfg, selector.NewCompressor(cfg)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: cfg}).Comp + payloadBuilder := stream.NewJSONPayloadBuilder(true, cfg, compressor) events := createEvents(numberOfItem) b.ResetTimer() @@ -385,7 +386,8 @@ func benchmarkCreateSingleMarshaler(b *testing.B, createEvents func(numberOfItem func BenchmarkCreateMarshalersBySourceType(b *testing.B) { runBenchmark(b, func(b *testing.B, numberOfItem int) { cfg := configmock.New(b) - payloadBuilder := stream.NewJSONPayloadBuilder(true, cfg, selector.NewCompressor(cfg)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: cfg}).Comp + payloadBuilder := stream.NewJSONPayloadBuilder(true, cfg, compressor) events := createBenchmarkEvents(numberOfItem) b.ResetTimer() @@ -401,7 +403,9 @@ func BenchmarkCreateMarshalersBySourceType(b *testing.B) { func BenchmarkCreateMarshalersSeveralSourceTypes(b *testing.B) { runBenchmark(b, func(b *testing.B, numberOfItem int) { cfg := configmock.New(b) - payloadBuilder := stream.NewJSONPayloadBuilder(true, cfg, selector.NewCompressor(cfg)) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: cfg}).Comp + payloadBuilder := stream.NewJSONPayloadBuilder(true, cfg, compressor) events := Events{} // Half of events have the same source type diff --git a/pkg/serializer/internal/metrics/iterable_series.go b/pkg/serializer/internal/metrics/iterable_series.go index 2a72c60157478f..e545789bb3cb95 100644 --- a/pkg/serializer/internal/metrics/iterable_series.go +++ b/pkg/serializer/internal/metrics/iterable_series.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serializer/internal/stream" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" diff --git a/pkg/serializer/internal/metrics/series_test.go b/pkg/serializer/internal/metrics/series_test.go index c584f02875c4be..a6eb7a0f236387 100644 --- a/pkg/serializer/internal/metrics/series_test.go +++ b/pkg/serializer/internal/metrics/series_test.go @@ -20,13 +20,13 @@ import ( "github.com/DataDog/agent-payload/v5/gogen" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serializer/internal/stream" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) func TestPopulateDeviceField(t *testing.T) { @@ -376,22 +376,22 @@ func TestMarshalSplitCompress(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { series := makeSeries(10000, 50) mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - strategy := selector.NewCompressor(mockConfig) - payloads, err := series.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, strategy) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := series.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) require.NoError(t, err) // check that we got multiple payloads, so splitting occurred require.Greater(t, len(payloads), 1) for _, compressedPayload := range payloads { - payload, err := strategy.Decompress(compressedPayload.GetContent()) + payload, err := compressor.Decompress(compressedPayload.GetContent()) require.NoError(t, err) pl := new(gogen.MetricPayload) @@ -410,8 +410,8 @@ func TestMarshalSplitCompressPointsLimit(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -422,7 +422,8 @@ func TestMarshalSplitCompressPointsLimit(t *testing.T) { // ten series, each with 50 points, so two should fit in each payload series := makeSeries(10, 50) - payloads, err := series.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := series.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) require.NoError(t, err) require.Equal(t, 5, len(payloads)) }) @@ -433,8 +434,8 @@ func TestMarshalSplitCompressMultiplePointsLimit(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -463,7 +464,8 @@ func TestMarshalSplitCompressMultiplePointsLimit(t *testing.T) { } series := CreateIterableSeries(CreateSerieSource(rawSeries)) - payloads, filteredPayloads, autoscalingFailoverPayloads, err := series.MarshalSplitCompressMultiple(mockConfig, selector.NewCompressor(mockConfig), + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, filteredPayloads, autoscalingFailoverPayloads, err := series.MarshalSplitCompressMultiple(mockConfig, compressor, func(s *metrics.Serie) bool { return s.Name == "test.metrics42" }, @@ -484,8 +486,8 @@ func TestMarshalSplitCompressPointsLimitTooBig(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -494,7 +496,9 @@ func TestMarshalSplitCompressPointsLimitTooBig(t *testing.T) { mockConfig.SetWithoutSource("serializer_max_series_points_per_payload", 1) series := makeSeries(1, 2) - payloads, err := series.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, selector.NewCompressor(mockConfig)) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := series.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) require.NoError(t, err) require.Len(t, payloads, 0) }) @@ -508,8 +512,8 @@ func TestPayloadsSeries(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -541,15 +545,16 @@ func TestPayloadsSeries(t *testing.T) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) originalLength := len(testSeries) - strategy := selector.NewCompressor(mockConfig) - builder := stream.NewJSONPayloadBuilder(true, mockConfig, strategy) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builder := stream.NewJSONPayloadBuilder(true, mockConfig, compressor) iterableSeries := CreateIterableSeries(CreateSerieSource(testSeries)) payloads, err := builder.BuildWithOnErrItemTooBigPolicy(iterableSeries, stream.DropItemOnErrItemTooBig) require.Nil(t, err) var splitSeries = []Series{} for _, compressedPayload := range payloads { - payload, err := strategy.Decompress(compressedPayload.GetContent()) + payload, err := compressor.Decompress(compressedPayload.GetContent()) require.NoError(t, err) var s = map[string]Series{} @@ -589,7 +594,8 @@ func BenchmarkPayloadsSeries(b *testing.B) { var r transaction.BytesPayloads mockConfig := mock.New(b) - builder := stream.NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builder := stream.NewJSONPayloadBuilder(true, mockConfig, compressor) for n := 0; n < b.N; n++ { // always record the result of Payloads to prevent // the compiler eliminating the function call. diff --git a/pkg/serializer/internal/metrics/service_checks_test.go b/pkg/serializer/internal/metrics/service_checks_test.go index 2559d7c9ed2cc1..94dd8e666cde2d 100644 --- a/pkg/serializer/internal/metrics/service_checks_test.go +++ b/pkg/serializer/internal/metrics/service_checks_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/metrics/servicecheck" @@ -80,14 +80,14 @@ func createServiceCheck(checkName string) *servicecheck.ServiceCheck { } func buildPayload(t *testing.T, m marshaler.StreamJSONMarshaler, cfg pkgconfigmodel.Config) [][]byte { - strategy := selector.NewCompressor(cfg) - builder := stream.NewJSONPayloadBuilder(true, cfg, strategy) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: cfg}).Comp + builder := stream.NewJSONPayloadBuilder(true, cfg, compressor) payloads, err := stream.BuildJSONPayload(builder, m) assert.NoError(t, err) var uncompressedPayloads [][]byte for _, compressedPayload := range payloads { - payload, err := strategy.Decompress(compressedPayload.GetContent()) + payload, err := compressor.Decompress(compressedPayload.GetContent()) assert.NoError(t, err) uncompressedPayloads = append(uncompressedPayloads, payload) @@ -159,7 +159,8 @@ func createServiceChecks(numberOfItem int) ServiceChecks { func benchmarkJSONPayloadBuilderServiceCheck(b *testing.B, numberOfItem int) { mockConfig := mock.New(b) - payloadBuilder := stream.NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloadBuilder := stream.NewJSONPayloadBuilder(true, mockConfig, compressor) serviceChecks := createServiceChecks(numberOfItem) b.ResetTimer() @@ -200,9 +201,9 @@ func benchmarkPayloadsServiceCheck(b *testing.B, numberOfItem int) { b.ResetTimer() mockConfig := mock.New(b) - strategy := selector.NewCompressor(mockConfig) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp for n := 0; n < b.N; n++ { - split.Payloads(serviceChecks, true, split.JSONMarshalFct, strategy) + split.Payloads(serviceChecks, true, split.JSONMarshalFct, compressor) } } diff --git a/pkg/serializer/internal/metrics/sketch_benchmark_test.go b/pkg/serializer/internal/metrics/sketch_benchmark_test.go index 0d1b6a0666c116..3cf90c370406b4 100644 --- a/pkg/serializer/internal/metrics/sketch_benchmark_test.go +++ b/pkg/serializer/internal/metrics/sketch_benchmark_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" @@ -30,9 +30,9 @@ func benchmarkSplitPayloadsSketchesSplit(b *testing.B, numPoints int) { b.ResetTimer() mockConfig := mock.New(b) - strategy := selector.NewCompressor(mockConfig) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp for n := 0; n < b.N; n++ { - split.Payloads(serializer, true, split.ProtoMarshalFct, strategy) + split.Payloads(serializer, true, split.ProtoMarshalFct, compressor) } } @@ -45,10 +45,10 @@ func benchmarkSplitPayloadsSketchesNew(b *testing.B, numPoints int) { b.ReportAllocs() b.ResetTimer() mockConfig := mock.New(b) - strategy := selector.NewCompressor(mockConfig) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp for n := 0; n < b.N; n++ { - payloads, err := serializer.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, strategy) + payloads, err := serializer.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) require.NoError(b, err) var pb int for _, p := range payloads { diff --git a/pkg/serializer/internal/metrics/sketch_series_list.go b/pkg/serializer/internal/metrics/sketch_series_list.go index 2dfb2173af619f..d73ffe9ebab53a 100644 --- a/pkg/serializer/internal/metrics/sketch_series_list.go +++ b/pkg/serializer/internal/metrics/sketch_series_list.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serializer/internal/stream" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" diff --git a/pkg/serializer/internal/metrics/sketch_series_test.go b/pkg/serializer/internal/metrics/sketch_series_test.go index 84ecf06c676ec7..c82901fca60851 100644 --- a/pkg/serializer/internal/metrics/sketch_series_test.go +++ b/pkg/serializer/internal/metrics/sketch_series_test.go @@ -12,12 +12,12 @@ import ( "github.com/DataDog/agent-payload/v5/gogen" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -93,8 +93,8 @@ func TestSketchSeriesMarshalSplitCompressEmpty(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -102,15 +102,16 @@ func TestSketchSeriesMarshalSplitCompressEmpty(t *testing.T) { mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) sl := SketchSeriesList{SketchesSource: metrics.NewSketchesSourceTest()} payload, _ := sl.Marshal() - strategy := selector.NewCompressor(mockConfig) - payloads, err := sl.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, strategy) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := sl.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) assert.Nil(t, err) firstPayload := payloads[0] assert.Equal(t, 0, firstPayload.GetPointCount()) - decompressed, _ := strategy.Decompress(firstPayload.GetContent()) + decompressed, _ := compressor.Decompress(firstPayload.GetContent()) // Check that we encoded the protobuf correctly assert.Equal(t, decompressed, payload) }) @@ -122,8 +123,8 @@ func TestSketchSeriesMarshalSplitCompressItemTooBigIsDropped(t *testing.T) { kind string maxUncompressedSize int }{ - "zlib": {kind: common.ZlibKind, maxUncompressedSize: 100}, - "zstd": {kind: common.ZstdKind, maxUncompressedSize: 200}, + "zlib": {kind: compression.ZlibKind, maxUncompressedSize: 100}, + "zstd": {kind: compression.ZstdKind, maxUncompressedSize: 200}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -144,15 +145,16 @@ func TestSketchSeriesMarshalSplitCompressItemTooBigIsDropped(t *testing.T) { }) serializer := SketchSeriesList{SketchesSource: sl} - strategy := selector.NewCompressor(mockConfig) - payloads, err := serializer.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, strategy) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := serializer.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) assert.Nil(t, err) firstPayload := payloads[0] require.Equal(t, 0, firstPayload.GetPointCount()) - decompressed, _ := strategy.Decompress(firstPayload.GetContent()) + decompressed, _ := compressor.Decompress(firstPayload.GetContent()) pl := new(gogen.SketchPayload) if err := pl.Unmarshal(decompressed); err != nil { @@ -170,8 +172,8 @@ func TestSketchSeriesMarshalSplitCompress(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -185,14 +187,15 @@ func TestSketchSeriesMarshalSplitCompress(t *testing.T) { sl.Reset() serializer2 := SketchSeriesList{SketchesSource: sl} - strategy := selector.NewCompressor(mockConfig) - payloads, err := serializer2.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, strategy) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := serializer2.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) require.NoError(t, err) firstPayload := payloads[0] assert.Equal(t, 11, firstPayload.GetPointCount()) - decompressed, _ := strategy.Decompress(firstPayload.GetContent()) + decompressed, _ := compressor.Decompress(firstPayload.GetContent()) pl := new(gogen.SketchPayload) err = pl.Unmarshal(decompressed) @@ -226,8 +229,8 @@ func TestSketchSeriesMarshalSplitCompressSplit(t *testing.T) { kind string maxUncompressedSize int }{ - "zlib": {kind: common.ZlibKind, maxUncompressedSize: 2000}, - "zstd": {kind: common.ZstdKind, maxUncompressedSize: 2000}, + "zlib": {kind: compression.ZlibKind, maxUncompressedSize: 2000}, + "zstd": {kind: compression.ZstdKind, maxUncompressedSize: 2000}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -244,15 +247,16 @@ func TestSketchSeriesMarshalSplitCompressSplit(t *testing.T) { } serializer := SketchSeriesList{SketchesSource: sl} - strategy := selector.NewCompressor(mockConfig) - payloads, err := serializer.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, strategy) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := serializer.MarshalSplitCompress(marshaler.NewBufferContext(), mockConfig, compressor) assert.Nil(t, err) recoveredSketches := []gogen.SketchPayload{} recoveredCount := 0 pointCount := 0 for _, pld := range payloads { - decompressed, _ := strategy.Decompress(pld.GetContent()) + decompressed, _ := compressor.Decompress(pld.GetContent()) pl := new(gogen.SketchPayload) if err := pl.Unmarshal(decompressed); err != nil { @@ -293,8 +297,8 @@ func TestSketchSeriesMarshalSplitCompressMultiple(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -308,8 +312,8 @@ func TestSketchSeriesMarshalSplitCompressMultiple(t *testing.T) { sl.Reset() serializer2 := SketchSeriesList{SketchesSource: sl} - strategy := selector.NewCompressor(mockConfig) - payloads, filteredPayloads, err := serializer2.MarshalSplitCompressMultiple(mockConfig, strategy, func(ss *metrics.SketchSeries) bool { + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, filteredPayloads, err := serializer2.MarshalSplitCompressMultiple(mockConfig, compressor, func(ss *metrics.SketchSeries) bool { return ss.Name == "name.0" }) require.NoError(t, err) diff --git a/pkg/serializer/internal/stream/compressor.go b/pkg/serializer/internal/stream/compressor.go index dea456eef7dba5..6d97f16181d7b2 100644 --- a/pkg/serializer/internal/stream/compressor.go +++ b/pkg/serializer/internal/stream/compressor.go @@ -11,9 +11,9 @@ import ( "errors" "expvar" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/telemetry" - - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) var ( @@ -56,7 +56,7 @@ func init() { type Compressor struct { input *bytes.Buffer // temporary buffer for data that has not been compressed yet compressed *bytes.Buffer // output buffer containing the compressed payload - strategy compression.Component + strategy metricscompression.Component zipper compression.StreamCompressor header []byte // json header to print at the beginning of the payload footer []byte // json footer to append at the end of the payload @@ -71,7 +71,7 @@ type Compressor struct { } // NewCompressor returns a new instance of a Compressor -func NewCompressor(input, output *bytes.Buffer, maxPayloadSize, maxUncompressedSize int, header, footer []byte, separator []byte, compressor compression.Component) (*Compressor, error) { +func NewCompressor(input, output *bytes.Buffer, maxPayloadSize, maxUncompressedSize int, header, footer []byte, separator []byte, compressor compression.Compressor) (*Compressor, error) { c := &Compressor{ header: header, footer: footer, diff --git a/pkg/serializer/internal/stream/compressor_test.go b/pkg/serializer/internal/stream/compressor_test.go index f44b8cc2f617d2..e074854273a389 100644 --- a/pkg/serializer/internal/stream/compressor_test.go +++ b/pkg/serializer/internal/stream/compressor_test.go @@ -17,15 +17,15 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/comp/core/config" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) func payloadToString(payload []byte, cfg config.Component) string { - strategy := selector.NewCompressor(cfg) - p, err := strategy.Decompress(payload) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: cfg}).Comp + p, err := compressor.Decompress(payload) if err != nil { return err.Error() } @@ -36,8 +36,8 @@ func TestCompressorSimple(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -45,10 +45,11 @@ func TestCompressorSimple(t *testing.T) { mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) maxPayloadSize := mockConfig.GetInt("serializer_max_payload_size") maxUncompressedSize := mockConfig.GetInt("serializer_max_uncompressed_payload_size") + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp c, err := NewCompressor( &bytes.Buffer{}, &bytes.Buffer{}, maxPayloadSize, maxUncompressedSize, - []byte("{["), []byte("]}"), []byte(","), selector.NewCompressor(mockConfig)) + []byte("{["), []byte("]}"), []byte(","), compressor) require.NoError(t, err) for i := 0; i < 5; i++ { @@ -70,18 +71,20 @@ func TestCompressorAddItemErrCodeWithEmptyCompressor(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp checkAddItemErrCode := func(maxPayloadSize, maxUncompressedSize, dataLen int) { c, err := NewCompressor( &bytes.Buffer{}, &bytes.Buffer{}, maxPayloadSize, maxUncompressedSize, - []byte("{["), []byte("]}"), []byte(","), selector.NewCompressor(mockConfig)) + []byte("{["), []byte("]}"), []byte(","), compressor) require.NoError(t, err) payload := strings.Repeat("A", dataLen) @@ -115,8 +118,8 @@ func TestOnePayloadSimple(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -128,7 +131,9 @@ func TestOnePayloadSimple(t *testing.T) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - builder := NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builder := NewJSONPayloadBuilder(true, mockConfig, compressor) payloads, err := BuildJSONPayload(builder, m) require.NoError(t, err) require.Len(t, payloads, 1) @@ -143,8 +148,8 @@ func TestMaxCompressedSizePayload(t *testing.T) { kind string maxPayloadSize int }{ - "zlib": {kind: common.ZlibKind, maxPayloadSize: 22}, - "zstd": {kind: common.ZstdKind, maxPayloadSize: 90}, + "zlib": {kind: compression.ZlibKind, maxPayloadSize: 22}, + "zstd": {kind: compression.ZstdKind, maxPayloadSize: 90}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -156,8 +161,8 @@ func TestMaxCompressedSizePayload(t *testing.T) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) mockConfig.SetDefault("serializer_max_payload_size", tc.maxPayloadSize) - - builder := NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builder := NewJSONPayloadBuilder(true, mockConfig, compressor) payloads, err := BuildJSONPayload(builder, m) require.NoError(t, err) require.Len(t, payloads, 1) @@ -181,7 +186,8 @@ func TestZstdCompressionLevel(t *testing.T) { mockConfig.SetWithoutSource("serializer_compressor_kind", "zstd") mockConfig.SetDefault("serializer_zstd_compressor_level", level) - builder := NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builder := NewJSONPayloadBuilder(true, mockConfig, compressor) payloads, err := BuildJSONPayload(builder, m) require.NoError(t, err) require.Len(t, payloads, 1) @@ -196,8 +202,8 @@ func TestTwoPayload(t *testing.T) { kind string maxPayloadSize int }{ - "zlib": {kind: common.ZlibKind, maxPayloadSize: 22}, - "zstd": {kind: common.ZstdKind, maxPayloadSize: 70}, + "zlib": {kind: compression.ZlibKind, maxPayloadSize: 22}, + "zstd": {kind: compression.ZstdKind, maxPayloadSize: 70}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -210,7 +216,8 @@ func TestTwoPayload(t *testing.T) { mockConfig.SetDefault("serializer_max_payload_size", tc.maxPayloadSize) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - builder := NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builder := NewJSONPayloadBuilder(true, mockConfig, compressor) payloads, err := BuildJSONPayload(builder, m) require.NoError(t, err) require.Len(t, payloads, 2) @@ -225,8 +232,8 @@ func TestLockedCompressorProducesSamePayloads(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -238,8 +245,9 @@ func TestLockedCompressorProducesSamePayloads(t *testing.T) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - builderLocked := NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) - builderUnLocked := NewJSONPayloadBuilder(false, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + builderLocked := NewJSONPayloadBuilder(true, mockConfig, compressor) + builderUnLocked := NewJSONPayloadBuilder(false, mockConfig, compressor) payloads1, err := BuildJSONPayload(builderLocked, m) require.NoError(t, err) payloads2, err := BuildJSONPayload(builderUnLocked, m) @@ -255,16 +263,18 @@ func TestBuildWithOnErrItemTooBigPolicyMetadata(t *testing.T) { kind string maxUncompressedPayloadSize int }{ - "zlib": {kind: common.ZlibKind, maxUncompressedPayloadSize: 40}, - "zstd": {kind: common.ZstdKind, maxUncompressedPayloadSize: 170}, + "zlib": {kind: compression.ZlibKind, maxUncompressedPayloadSize: 40}, + "zstd": {kind: compression.ZstdKind, maxUncompressedPayloadSize: 170}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) mockConfig.SetWithoutSource("serializer_max_uncompressed_payload_size", tc.maxUncompressedPayloadSize) + + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp marshaler := &IterableStreamJSONMarshalerMock{index: 0, maxIndex: 100} - builder := NewJSONPayloadBuilder(false, mockConfig, selector.NewCompressor(mockConfig)) + builder := NewJSONPayloadBuilder(false, mockConfig, compressor) payloads, err := builder.BuildWithOnErrItemTooBigPolicy( marshaler, DropItemOnErrItemTooBig) diff --git a/pkg/serializer/internal/stream/json_payload_builder.go b/pkg/serializer/internal/stream/json_payload_builder.go index 0c12e7c2fb078c..4812077688524b 100644 --- a/pkg/serializer/internal/stream/json_payload_builder.go +++ b/pkg/serializer/internal/stream/json_payload_builder.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" "github.com/DataDog/datadog-agent/pkg/telemetry" "github.com/DataDog/datadog-agent/pkg/util/log" diff --git a/pkg/serializer/serializer.go b/pkg/serializer/serializer.go index a60e474cebd35c..54a6474b877205 100644 --- a/pkg/serializer/serializer.go +++ b/pkg/serializer/serializer.go @@ -21,7 +21,6 @@ import ( orchestratorForwarder "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorinterface" "github.com/DataDog/datadog-agent/comp/core/config" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/metrics/event" "github.com/DataDog/datadog-agent/pkg/metrics/servicecheck" @@ -31,6 +30,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" "github.com/DataDog/datadog-agent/pkg/serializer/split" "github.com/DataDog/datadog-agent/pkg/serializer/types" + "github.com/DataDog/datadog-agent/pkg/util/compression" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" @@ -107,7 +107,7 @@ type Serializer struct { orchestratorForwarder orchestratorForwarder.Component config config.Component - Strategy compression.Component + Strategy compression.Compressor seriesJSONPayloadBuilder *stream.JSONPayloadBuilder jsonExtraHeaders http.Header protobufExtraHeaders http.Header @@ -133,7 +133,7 @@ type Serializer struct { } // NewSerializer returns a new Serializer initialized -func NewSerializer(forwarder forwarder.Forwarder, orchestratorForwarder orchestratorForwarder.Component, compressor compression.Component, config config.Component, hostName string) *Serializer { +func NewSerializer(forwarder forwarder.Forwarder, orchestratorForwarder orchestratorForwarder.Component, compressor compression.Compressor, config config.Component, hostName string) *Serializer { streamAvailable := compressor.NewStreamCompressor(&bytes.Buffer{}) != nil diff --git a/pkg/serializer/serializer_benchmark_test.go b/pkg/serializer/serializer_benchmark_test.go index f02bc89a44f72c..ec0a0ac9cc9dc4 100644 --- a/pkg/serializer/serializer_benchmark_test.go +++ b/pkg/serializer/serializer_benchmark_test.go @@ -12,7 +12,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompressionimpl "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics/event" metricsserializer "github.com/DataDog/datadog-agent/pkg/serializer/internal/metrics" @@ -43,7 +43,8 @@ func benchmarkJSONStream(b *testing.B, passes int, sharedBuffers bool, numberOfE events := buildEvents(numberOfEvents) marshaler := events.CreateSingleMarshaler() mockConfig := mock.New(b) - payloadBuilder := stream.NewJSONPayloadBuilder(sharedBuffers, mockConfig, selector.NewCompressor(mockConfig)) + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + payloadBuilder := stream.NewJSONPayloadBuilder(sharedBuffers, mockConfig, compressor) b.ResetTimer() for n := 0; n < b.N; n++ { @@ -58,9 +59,9 @@ func benchmarkSplit(b *testing.B, numberOfEvents int) { b.ResetTimer() mockConfig := mock.New(b) - strategy := selector.NewCompressor(mockConfig) + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp for n := 0; n < b.N; n++ { - results, _ = split.Payloads(events, true, split.JSONMarshalFct, strategy) + results, _ = split.Payloads(events, true, split.JSONMarshalFct, compressor) } } diff --git a/pkg/serializer/serializer_test.go b/pkg/serializer/serializer_test.go index 7675390cc3233e..84e579bd9b0299 100644 --- a/pkg/serializer/serializer_test.go +++ b/pkg/serializer/serializer_test.go @@ -22,21 +22,23 @@ import ( forwarder "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" + metricscompressionimpl "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" configmock "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/metrics/event" "github.com/DataDog/datadog-agent/pkg/metrics/servicecheck" metricsserializer "github.com/DataDog/datadog-agent/pkg/serializer/internal/metrics" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) func TestInitExtraHeadersNoopCompression(t *testing.T) { mockConfig := configmock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", "blah") - s := NewSerializer(nil, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(nil, nil, compressor, mockConfig, "testhost") initExtraHeaders(s) expected := make(http.Header) @@ -51,10 +53,12 @@ func TestInitExtraHeadersNoopCompression(t *testing.T) { // No "Content-Encoding" header expected = make(http.Header) expected.Set("Content-Type", jsonContentType) + expected.Set("Content-Encoding", "identity") assert.Equal(t, expected, s.jsonExtraHeadersWithCompression) expected = make(http.Header) expected.Set("Content-Type", protobufContentType) + expected.Set("Content-Encoding", "identity") expected.Set(payloadVersionHTTPHeader, AgentPayloadVersion) assert.Equal(t, expected, s.protobufExtraHeadersWithCompression) } @@ -64,14 +68,15 @@ func TestInitExtraHeadersWithCompression(t *testing.T) { kind string expectedEncoding string }{ - "zlib": {kind: common.ZlibKind, expectedEncoding: compression.ZlibEncoding}, - "zstd": {kind: common.ZstdKind, expectedEncoding: compression.ZstdEncoding}, + "zlib": {kind: compression.ZlibKind, expectedEncoding: compression.ZlibEncoding}, + "zstd": {kind: compression.ZstdKind, expectedEncoding: compression.ZstdEncoding}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { mockConfig := configmock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - s := NewSerializer(nil, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(nil, nil, compressor, mockConfig, "testhost") initExtraHeaders(s) expected := make(http.Header) @@ -112,7 +117,7 @@ var ( ) type testPayload struct { - compressor compression.Component + compressor metricscompression.Component } //nolint:revive // TODO(AML) Fix revive linter @@ -250,8 +255,8 @@ func TestSendV1Events(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -260,7 +265,8 @@ func TestSendV1Events(t *testing.T) { mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) f := &forwarder.MockedForwarder{} - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") matcher := createJSONPayloadMatcher(`{"apiKey":"","events":{},"internalHostname"`, s) f.On("SubmitV1Intake", matcher, s.jsonExtraHeadersWithCompression).Return(nil).Times(1) @@ -276,8 +282,8 @@ func TestSendV1EventsCreateMarshalersBySourceType(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -287,7 +293,8 @@ func TestSendV1EventsCreateMarshalersBySourceType(t *testing.T) { mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) f := &forwarder.MockedForwarder{} - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") events := event.Events{&event.Event{SourceTypeName: "source1"}, &event.Event{SourceTypeName: "source2"}, &event.Event{SourceTypeName: "source3"}} payloadsCountMatcher := func(payloadCount int) interface{} { @@ -315,8 +322,8 @@ func TestSendV1ServiceChecks(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -325,7 +332,9 @@ func TestSendV1ServiceChecks(t *testing.T) { mockConfig := configmock.New(t) mockConfig.SetWithoutSource("enable_service_checks_stream_payload_serialization", false) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") matcher := createJSONPayloadMatcher(`[{"check":"","host_name":"","timestamp":0,"status":0,"message":"","tags":null}]`, s) f.On("SubmitV1CheckRuns", matcher, s.jsonExtraHeadersWithCompression).Return(nil).Times(1) @@ -340,8 +349,8 @@ func TestSendV1Series(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -351,7 +360,9 @@ func TestSendV1Series(t *testing.T) { mockConfig.SetWithoutSource("enable_stream_payload_serialization", false) mockConfig.SetWithoutSource("use_v2_api.series", false) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") matcher := createJSONPayloadMatcher(`{"series":[]}`, s) f.On("SubmitV1Series", matcher, s.jsonExtraHeadersWithCompression).Return(nil).Times(1) @@ -367,8 +378,8 @@ func TestSendSeries(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -377,7 +388,9 @@ func TestSendSeries(t *testing.T) { mockConfig := configmock.New(t) mockConfig.SetWithoutSource("use_v2_api.series", true) // default value, but just to be sure mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") matcher := createProtoscopeMatcher(`1: { 1: { 1: {"host"} } 5: 3 @@ -396,8 +409,8 @@ func TestSendSketch(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -406,7 +419,9 @@ func TestSendSketch(t *testing.T) { mockConfig := configmock.New(t) mockConfig.SetWithoutSource("use_v2_api.series", true) // default value, but just to be sure mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") matcher := createProtoscopeMatcher(` 1: { 1: {"fakename"} 2: {"fakehost"} 8: { 1: { 4: 10 }}} 2: {} @@ -426,8 +441,8 @@ func TestSendMetadata(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -435,7 +450,8 @@ func TestSendMetadata(t *testing.T) { f := &forwarder.MockedForwarder{} mockConfig := configmock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - compressor := selector.NewCompressor(mockConfig) + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp s := NewSerializer(f, nil, compressor, mockConfig, "testhost") jsonPayloads, _ := mkPayloads(jsonString, true, s) f.On("SubmitMetadata", jsonPayloads, s.jsonExtraHeadersWithCompression).Return(nil).Times(1) @@ -461,8 +477,8 @@ func TestSendProcessesMetadata(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -471,7 +487,9 @@ func TestSendProcessesMetadata(t *testing.T) { payload := []byte("\"test\"") mockConfig := configmock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") payloads, _ := mkPayloads(payload, true, s) f.On("SubmitV1Intake", payloads, s.jsonExtraHeadersWithCompression).Return(nil).Times(1) @@ -495,8 +513,8 @@ func TestSendWithDisabledKind(t *testing.T) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { @@ -511,7 +529,9 @@ func TestSendWithDisabledKind(t *testing.T) { mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) f := &forwarder.MockedForwarder{} - s := NewSerializer(f, nil, selector.NewCompressor(mockConfig), mockConfig, "testhost") + + compressor := metricscompressionimpl.NewCompressorReq(metricscompressionimpl.Requires{Cfg: mockConfig}).Comp + s := NewSerializer(f, nil, compressor, mockConfig, "testhost") jsonPayloads, _ := mkPayloads(jsonString, true, s) payload := &testPayload{} diff --git a/pkg/serializer/series_benchmark_test.go b/pkg/serializer/series_benchmark_test.go index 0ec06d08339911..8b96a7d4f88c87 100644 --- a/pkg/serializer/series_benchmark_test.go +++ b/pkg/serializer/series_benchmark_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" metricsserializer "github.com/DataDog/datadog-agent/pkg/serializer/internal/metrics" @@ -77,12 +77,13 @@ func BenchmarkSeries(b *testing.B) { } bufferContext := marshaler.NewBufferContext() mockConfig := mock.New(b) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp pb := func(series metrics.Series) (transaction.BytesPayloads, error) { iterableSeries := metricsserializer.CreateIterableSeries(metricsserializer.CreateSerieSource(series)) - return iterableSeries.MarshalSplitCompress(bufferContext, mockConfig, selector.NewCompressor(mockConfig)) + return iterableSeries.MarshalSplitCompress(bufferContext, mockConfig, compressor) } - payloadBuilder := stream.NewJSONPayloadBuilder(true, mockConfig, selector.NewCompressor(mockConfig)) + payloadBuilder := stream.NewJSONPayloadBuilder(true, mockConfig, compressor) json := func(series metrics.Series) (transaction.BytesPayloads, error) { iterableSeries := metricsserializer.CreateIterableSeries(metricsserializer.CreateSerieSource(series)) return payloadBuilder.BuildWithOnErrItemTooBigPolicy(iterableSeries, stream.DropItemOnErrItemTooBig) diff --git a/pkg/serializer/split/split.go b/pkg/serializer/split/split.go index 0d73e9b6ff0e62..00f65c90494cfc 100644 --- a/pkg/serializer/split/split.go +++ b/pkg/serializer/split/split.go @@ -10,7 +10,7 @@ import ( "expvar" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + compression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/def" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" "github.com/DataDog/datadog-agent/pkg/telemetry" diff --git a/pkg/serializer/split/split_test.go b/pkg/serializer/split/split_test.go index 22fa78e0ff9bd2..7c83aea822f6b6 100644 --- a/pkg/serializer/split/split_test.go +++ b/pkg/serializer/split/split_test.go @@ -15,13 +15,14 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/impl" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/metrics/event" "github.com/DataDog/datadog-agent/pkg/metrics/servicecheck" metricsserializer "github.com/DataDog/datadog-agent/pkg/serializer/internal/metrics" "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/compression" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" "github.com/DataDog/datadog-agent/pkg/config/mock" ) @@ -55,8 +56,8 @@ func testSplitPayloadsSeries(t *testing.T, numPoints int, compress bool) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -85,9 +86,7 @@ func testSplitPayloadsSeries(t *testing.T, numPoints int, compress bool) { testSeries = append(testSeries, &point) } - mockConfig := mock.New(t) - mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - strategy := selector.NewCompressor(mockConfig) + strategy := selector.NewCompressor(tc.kind, 1) payloads, err := Payloads(testSeries, compress, JSONMarshalFct, strategy) require.Nil(t, err) @@ -137,12 +136,12 @@ func BenchmarkSplitPayloadsSeries(b *testing.B) { } mockConfig := mock.New(b) - strategy := selector.NewCompressor(mockConfig) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp var r transaction.BytesPayloads for n := 0; n < b.N; n++ { // always record the result of Payloads to prevent // the compiler eliminating the function call. - r, _ = Payloads(testSeries, true, JSONMarshalFct, strategy) + r, _ = Payloads(testSeries, true, JSONMarshalFct, compressor) } // ensure we actually had to split @@ -194,8 +193,8 @@ func testSplitPayloadsEvents(t *testing.T, numPoints int, compress bool) { tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -216,8 +215,8 @@ func testSplitPayloadsEvents(t *testing.T, numPoints int, compress bool) { mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - strategy := selector.NewCompressor(mockConfig) - payloads, err := Payloads(testEvent, compress, JSONMarshalFct, strategy) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := Payloads(testEvent, compress, JSONMarshalFct, compressor) require.Nil(t, err) originalLength := len(testEvent.EventsArr) @@ -226,7 +225,7 @@ func testSplitPayloadsEvents(t *testing.T, numPoints int, compress bool) { var s map[string]interface{} localPayload := payload.GetContent() if compress { - localPayload, err = strategy.Decompress(localPayload) + localPayload, err = compressor.Decompress(localPayload) require.Nil(t, err) } @@ -275,8 +274,8 @@ func testSplitPayloadsServiceChecks(t *testing.T, numPoints int, compress bool) tests := map[string]struct { kind string }{ - "zlib": {kind: common.ZlibKind}, - "zstd": {kind: common.ZstdKind}, + "zlib": {kind: compression.ZlibKind}, + "zstd": {kind: compression.ZstdKind}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -295,8 +294,8 @@ func testSplitPayloadsServiceChecks(t *testing.T, numPoints int, compress bool) mockConfig := mock.New(t) mockConfig.SetWithoutSource("serializer_compressor_kind", tc.kind) - strategy := selector.NewCompressor(mockConfig) - payloads, err := Payloads(testServiceChecks, compress, JSONMarshalFct, strategy) + compressor := metricscompression.NewCompressorReq(metricscompression.Requires{Cfg: mockConfig}).Comp + payloads, err := Payloads(testServiceChecks, compress, JSONMarshalFct, compressor) require.Nil(t, err) originalLength := len(testServiceChecks) @@ -305,7 +304,7 @@ func testSplitPayloadsServiceChecks(t *testing.T, numPoints int, compress bool) var s []interface{} localPayload := payload.GetContent() if compress { - localPayload, err = strategy.Decompress(localPayload) + localPayload, err = compressor.Decompress(localPayload) require.Nil(t, err) } diff --git a/pkg/serverless/invocationlifecycle/lifecycle_test.go b/pkg/serverless/invocationlifecycle/lifecycle_test.go index 3eefd8dc51bbd0..2f2f36366475d4 100644 --- a/pkg/serverless/invocationlifecycle/lifecycle_test.go +++ b/pkg/serverless/invocationlifecycle/lifecycle_test.go @@ -18,7 +18,8 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/metrics" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/serverless/logs" @@ -1383,5 +1384,5 @@ func getEventFromFile(filename string) []byte { } func createDemultiplexer(t *testing.T) demultiplexer.FakeSamplerMock { - return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), compressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) + return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), logscompression.MockModule(), metricscompression.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) } diff --git a/pkg/serverless/logs/logs_test.go b/pkg/serverless/logs/logs_test.go index 132efb75bbcaaa..9fd50a27f71281 100644 --- a/pkg/serverless/logs/logs_test.go +++ b/pkg/serverless/logs/logs_test.go @@ -26,7 +26,8 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" "github.com/DataDog/datadog-agent/comp/logs/agent/config" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompressionmock "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/serverless/executioncontext" serverlessMetrics "github.com/DataDog/datadog-agent/pkg/serverless/metrics" @@ -1475,5 +1476,5 @@ func TestMultipleStartLogCollection(t *testing.T) { } func createDemultiplexer(t *testing.T) demultiplexer.FakeSamplerMock { - return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), compressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) + return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), logscompressionmock.MockModule(), metricscompressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) } diff --git a/pkg/serverless/logs/scheduler.go b/pkg/serverless/logs/scheduler.go index b5be31485d14f9..962652952aac64 100644 --- a/pkg/serverless/logs/scheduler.go +++ b/pkg/serverless/logs/scheduler.go @@ -10,6 +10,7 @@ import ( logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent" "github.com/DataDog/datadog-agent/comp/logs/agent/agentimpl" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/def" "github.com/DataDog/datadog-agent/pkg/logs/schedulers/channel" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -19,8 +20,8 @@ import ( var logsScheduler *channel.Scheduler // SetupLogAgent sets up the logs agent to handle messages on the given channel. -func SetupLogAgent(logChannel chan *config.ChannelMessage, sourceName string, source string, tagger tagger.Component) (logsAgent.ServerlessLogsAgent, error) { - agent := agentimpl.NewServerlessLogsAgent(tagger) +func SetupLogAgent(logChannel chan *config.ChannelMessage, sourceName string, source string, tagger tagger.Component, compression logscompression.Component) (logsAgent.ServerlessLogsAgent, error) { + agent := agentimpl.NewServerlessLogsAgent(tagger, compression) err := agent.Start() if err != nil { log.Error("Could not start an instance of the Logs Agent:", err) diff --git a/pkg/serverless/metrics/enhanced_metrics_test.go b/pkg/serverless/metrics/enhanced_metrics_test.go index 9c6f9472ee7566..106a00af5896f8 100644 --- a/pkg/serverless/metrics/enhanced_metrics_test.go +++ b/pkg/serverless/metrics/enhanced_metrics_test.go @@ -12,7 +12,8 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/fx" - compressionmock "github.com/DataDog/datadog-agent/comp/serializer/compression/fx-mock" + logscompression "github.com/DataDog/datadog-agent/comp/serializer/logscompression/fx-mock" + metricscompression "github.com/DataDog/datadog-agent/comp/serializer/metricscompression/fx-mock" "github.com/DataDog/datadog-agent/pkg/serverless/proc" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" @@ -834,5 +835,5 @@ func TestSendFailoverReasonMetric(t *testing.T) { } func createDemultiplexer(t *testing.T) demultiplexer.FakeSamplerMock { - return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), compressionmock.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) + return fxutil.Test[demultiplexer.FakeSamplerMock](t, fx.Provide(func() log.Component { return logmock.New(t) }), metricscompression.MockModule(), logscompression.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) } diff --git a/comp/serializer/compression/def/component.go b/pkg/util/compression/compression.go similarity index 52% rename from comp/serializer/compression/def/component.go rename to pkg/util/compression/compression.go index 461c5a92694975..81cd2d4c5784a6 100644 --- a/comp/serializer/compression/def/component.go +++ b/pkg/util/compression/compression.go @@ -1,9 +1,9 @@ // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024-present Datadog, Inc. +// Copyright 2016-present Datadog, Inc. -// Package compression provides a compression implementation based on the configuration or available build tags. +// Package compression provides a set of constants describing the compression options package compression import ( @@ -11,7 +11,17 @@ import ( "io" ) -// team: agent-metrics-logs +// ZlibKind defines a const value for the zlib compressor +const ZlibKind = "zlib" + +// ZstdKind defines a const value for the zstd compressor +const ZstdKind = "zstd" + +// GzipKind defines a const value for the gzip compressor +const GzipKind = "gzip" + +// NoneKind defines a const value for disabling compression +const NoneKind = "none" // ZlibEncoding is the content-encoding value for Zlib const ZlibEncoding = "deflate" @@ -19,8 +29,12 @@ const ZlibEncoding = "deflate" // ZstdEncoding is the content-encoding value for Zstd const ZstdEncoding = "zstd" -// Component is the component type. -type Component interface { +// GzipEncoding is the content-encoding value for Gzip +const GzipEncoding = "gzip" + +// Compressor is the interface that a given compression algorithm +// needs to implement +type Compressor interface { Compress(src []byte) ([]byte, error) Decompress(src []byte) ([]byte, error) CompressBound(sourceLen int) int @@ -28,7 +42,8 @@ type Component interface { NewStreamCompressor(output *bytes.Buffer) StreamCompressor } -// StreamCompressor is the interface that zlib and zstd should implement +// StreamCompressor is the interface that the compression algorithm +// should implement for streaming type StreamCompressor interface { io.WriteCloser Flush() error diff --git a/pkg/util/compression/go.mod b/pkg/util/compression/go.mod new file mode 100644 index 00000000000000..3c75e1f7e2045d --- /dev/null +++ b/pkg/util/compression/go.mod @@ -0,0 +1,130 @@ +module github.com/DataDog/datadog-agent/pkg/util/compression + +go 1.22.0 + +replace github.com/DataDog/datadog-agent/pkg/util/log => ../log + +require ( + github.com/DataDog/datadog-agent/comp/core/config v0.0.0-00010101000000-000000000000 + github.com/DataDog/datadog-agent/pkg/util/log v0.60.1 + github.com/DataDog/zstd v1.5.6 + github.com/klauspost/compress v1.17.11 +) + +require ( + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/env v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/mock v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/setup v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/structure v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/executable v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/filesystem v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/option v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/pointer v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/system v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/system/socket v0.59.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.60.1 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect + github.com/DataDog/viper v1.14.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/ebitengine/purego v0.8.1 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.24.12 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/dig v1.18.0 // indirect + go.uber.org/fx v1.23.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace ( + github.com/DataDog/datadog-agent/comp/core/config => ../../../comp/core/config + github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../defaultpaths + github.com/DataDog/datadog-agent/pkg/util/option => ../option + github.com/DataDog/datadog-agent/pkg/util/scrubber => ../scrubber + github.com/DataDog/datadog-agent/pkg/version => ../../version +) + +replace github.com/DataDog/datadog-agent/comp/api/api/def => ../../../comp/api/api/def + +replace github.com/DataDog/datadog-agent/comp/core/flare/builder => ../../../comp/core/flare/builder + +replace github.com/DataDog/datadog-agent/comp/core/flare/types => ../../../comp/core/flare/types + +replace github.com/DataDog/datadog-agent/comp/core/secrets => ../../../comp/core/secrets + +replace github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../comp/core/telemetry + +replace github.com/DataDog/datadog-agent/comp/def => ../../../comp/def + +replace github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../collector/check/defaults + +replace github.com/DataDog/datadog-agent/pkg/config/env => ../../config/env + +replace github.com/DataDog/datadog-agent/pkg/config/mock => ../../config/mock + +replace github.com/DataDog/datadog-agent/pkg/config/model => ../../config/model + +replace github.com/DataDog/datadog-agent/pkg/config/nodetreemodel => ../../config/nodetreemodel + +replace github.com/DataDog/datadog-agent/pkg/config/setup => ../../config/setup + +replace github.com/DataDog/datadog-agent/pkg/config/teeconfig => ../../config/teeconfig + +replace github.com/DataDog/datadog-agent/pkg/util/executable => ../executable + +replace github.com/DataDog/datadog-agent/pkg/util/filesystem => ../filesystem + +replace github.com/DataDog/datadog-agent/pkg/util/fxutil => ../fxutil + +replace github.com/DataDog/datadog-agent/pkg/util/hostname/validate => ../hostname/validate + +replace github.com/DataDog/datadog-agent/pkg/util/pointer => ../pointer + +replace github.com/DataDog/datadog-agent/pkg/util/system => ../system + +replace github.com/DataDog/datadog-agent/pkg/util/system/socket => ../system/socket + +replace github.com/DataDog/datadog-agent/pkg/util/testutil => ../testutil + +replace github.com/DataDog/datadog-agent/pkg/util/winutil => ../winutil + +replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../config/structure diff --git a/comp/serializer/compression/go.sum b/pkg/util/compression/go.sum similarity index 100% rename from comp/serializer/compression/go.sum rename to pkg/util/compression/go.sum diff --git a/pkg/util/compression/impl-gzip/gzip_strategy.go b/pkg/util/compression/impl-gzip/gzip_strategy.go new file mode 100644 index 00000000000000..c5f9a7b6c68b76 --- /dev/null +++ b/pkg/util/compression/impl-gzip/gzip_strategy.go @@ -0,0 +1,127 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package gzipimpl provides a set of functions for compressing with zlib / zstd / gzip +package gzipimpl + +import ( + "bytes" + "compress/gzip" + "io" + + "github.com/DataDog/datadog-agent/pkg/util/compression" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// Requires contains the compression level for gzip compression +type Requires struct { + Level int +} + +// GzipStrategy is the strategy for when serializer_compression_kind is gzip +type GzipStrategy struct { + level int +} + +// New returns a new GzipStrategy +func New(req Requires) compression.Compressor { + level := req.Level + if level < gzip.NoCompression { + log.Warnf("Gzip log level set to %d, minimum is %d.", level, gzip.NoCompression) + level = gzip.NoCompression + } else if level > gzip.BestCompression { + log.Warnf("Gzip log level set to %d, maximum is %d.", level, gzip.BestCompression) + level = gzip.BestCompression + } + + return &GzipStrategy{ + level: level, + } +} + +// Compress will compress the data with gzip +func (s *GzipStrategy) Compress(src []byte) (result []byte, err error) { + var compressedPayload bytes.Buffer + gzipWriter, err := gzip.NewWriterLevel(&compressedPayload, s.level) + + if err != nil { + return nil, err + } + _, err = gzipWriter.Write(src) + if err != nil { + return nil, err + } + err = gzipWriter.Flush() + if err != nil { + return nil, err + } + err = gzipWriter.Close() + if err != nil { + return nil, err + } + + return compressedPayload.Bytes(), nil +} + +// Decompress will decompress the data with gzip +func (s *GzipStrategy) Decompress(src []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(src)) + if err != nil { + return nil, err + } + defer reader.Close() + + // Read all decompressed data + var result bytes.Buffer + _, err = io.Copy(&result, reader) + if err != nil { + return nil, err + } + + return result.Bytes(), nil +} + +// CompressBound returns the worst case size needed for a destination buffer +// when using gzip +// +// The worst case expansion is a few bytes for the gzip file header, plus +// 5 bytes per 32 KiB block, or an expansion ratio of 0.015% for large files. +// The additional 18 bytes comes from the header (10 bytes) and trailer +// (8 bytes). There is no theoretical maximum to the header, +// but we don't set any extra header fields so it is safe to assume +// +// Source: https://www.gnu.org/software/gzip/manual/html_node/Overview.html +// More details are in the linked RFC: https://www.ietf.org/rfc/rfc1952.txt +func (s *GzipStrategy) CompressBound(sourceLen int) int { + return sourceLen + (sourceLen/32768)*5 + 18 +} + +// ContentEncoding returns the content encoding value for gzip +func (s *GzipStrategy) ContentEncoding() string { + return compression.GzipEncoding +} + +// NewStreamCompressor returns a new gzip Writer +func (s *GzipStrategy) NewStreamCompressor(output *bytes.Buffer) compression.StreamCompressor { + // Ensure level is within a range that doesn't cause NewWriterLevel to error. + level := s.level + if level < gzip.HuffmanOnly { + log.Warnf("Gzip streaming log level set to %d, minimum is %d. Setting to minimum.", level, gzip.HuffmanOnly) + level = gzip.HuffmanOnly + } + + if level > gzip.BestCompression { + log.Warnf("Gzip streaming log level set to %d, maximum is %d. Setting to maximum.", level, gzip.BestCompression) + level = gzip.BestCompression + } + + writer, err := gzip.NewWriterLevel(output, level) + if err != nil { + log.Warnf("Error creating gzip writer with level %d. Using default.", level) + writer = gzip.NewWriter(output) + } + + return writer +} diff --git a/comp/serializer/compression/impl-noop/no_strategy.go b/pkg/util/compression/impl-noop/no_strategy.go similarity index 78% rename from comp/serializer/compression/impl-noop/no_strategy.go rename to pkg/util/compression/impl-noop/no_strategy.go index 3c092a8a536d42..74acedca7c1cc3 100644 --- a/comp/serializer/compression/impl-noop/no_strategy.go +++ b/pkg/util/compression/impl-noop/no_strategy.go @@ -9,23 +9,15 @@ package noopimpl import ( "bytes" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) -// Provides contains the compression component -type Provides struct { - Comp compression.Component -} - // NoopStrategy is the strategy for when serializer_compressor_kind is neither zlib nor zstd -type NoopStrategy struct { -} +type NoopStrategy struct{} -// NewComponent returns a new NoopStrategy for when kind is neither zlib nor zstd -func NewComponent() Provides { - return Provides{ - Comp: &NoopStrategy{}, - } +// New returns a new NoopStrategy for when kind is neither zlib nor zstd +func New() compression.Compressor { + return &NoopStrategy{} } // Compress implements the Compress method for NoopStrategy to satisfy the Compressor interface @@ -45,7 +37,7 @@ func (s *NoopStrategy) CompressBound(sourceLen int) int { // ContentEncoding implements the ContentEncoding method for NoopStrategy to satisfy the Compressor interface func (s *NoopStrategy) ContentEncoding() string { - return "" + return "identity" } // NewStreamCompressor returns a nil when there is no compression implementation. diff --git a/comp/serializer/compression/impl-zlib/zlib_strategy.go b/pkg/util/compression/impl-zlib/zlib_strategy.go similarity index 84% rename from comp/serializer/compression/impl-zlib/zlib_strategy.go rename to pkg/util/compression/impl-zlib/zlib_strategy.go index 0bd47af4f91fc9..7bf8a20033429b 100644 --- a/comp/serializer/compression/impl-zlib/zlib_strategy.go +++ b/pkg/util/compression/impl-zlib/zlib_strategy.go @@ -11,23 +11,15 @@ import ( "compress/zlib" "io" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) -// Provides contains the compression component -type Provides struct { - Comp compression.Component -} - // ZlibStrategy is the strategy for when serializer_compressor_kind is zlib -type ZlibStrategy struct { -} +type ZlibStrategy struct{} -// NewComponent returns a new ZlibStrategy -func NewComponent() Provides { - return Provides{ - Comp: &ZlibStrategy{}, - } +// New returns a new ZlibStrategy +func New() compression.Compressor { + return &ZlibStrategy{} } // Compress will compress the data with zlib diff --git a/comp/serializer/compression/impl-zstd-nocgo/zstd_nocgo_strategy.go b/pkg/util/compression/impl-zstd-nocgo/zstd_nocgo_strategy.go similarity index 85% rename from comp/serializer/compression/impl-zstd-nocgo/zstd_nocgo_strategy.go rename to pkg/util/compression/impl-zstd-nocgo/zstd_nocgo_strategy.go index 5d4735ee4e528d..d79eaa2ba82539 100644 --- a/comp/serializer/compression/impl-zstd-nocgo/zstd_nocgo_strategy.go +++ b/pkg/util/compression/impl-zstd-nocgo/zstd_nocgo_strategy.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/klauspost/compress/zstd" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) // Requires contains the compression level for zstd compression @@ -22,19 +22,14 @@ type Requires struct { Level compression.ZstdCompressionLevel } -// Provides contains the compression component -type Provides struct { - Comp compression.Component -} - // ZstdNoCgoStrategy can be manually selected via component - it's not used by any selector / config option type ZstdNoCgoStrategy struct { level int encoder *zstd.Encoder } -// NewComponent returns a new ZstdNoCgoStrategy -func NewComponent(reqs Requires) Provides { +// New returns a new ZstdNoCgoStrategy +func New(reqs Requires) compression.Compressor { level := int(reqs.Level) log.Debugf("Compressing native zstd at level %d", level) @@ -56,16 +51,12 @@ func NewComponent(reqs Requires) Provides { zstd.WithWindowSize(window)) if err != nil { _ = log.Errorf("Error creating zstd encoder: %v", err) - return Provides{ - Comp: nil, - } + return nil } - return Provides{ - Comp: &ZstdNoCgoStrategy{ - level: level, - encoder: encoder, - }, + return &ZstdNoCgoStrategy{ + level: level, + encoder: encoder, } } diff --git a/comp/serializer/compression/impl-zstd/zstd_strategy.go b/pkg/util/compression/impl-zstd/zstd_strategy.go similarity index 81% rename from comp/serializer/compression/impl-zstd/zstd_strategy.go rename to pkg/util/compression/impl-zstd/zstd_strategy.go index 33b196c5febceb..8727c25d3ca355 100644 --- a/comp/serializer/compression/impl-zstd/zstd_strategy.go +++ b/pkg/util/compression/impl-zstd/zstd_strategy.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/zstd" - compression "github.com/DataDog/datadog-agent/comp/serializer/compression/def" + "github.com/DataDog/datadog-agent/pkg/util/compression" ) // Requires contains the compression level for zstd compression @@ -19,22 +19,15 @@ type Requires struct { Level compression.ZstdCompressionLevel } -// Provides contains the compression component -type Provides struct { - Comp compression.Component -} - // ZstdStrategy is the strategy for when serializer_compressor_kind is zstd type ZstdStrategy struct { level int } -// NewComponent returns a new ZstdStrategy -func NewComponent(reqs Requires) Provides { - return Provides{ - Comp: &ZstdStrategy{ - level: int(reqs.Level), - }, +// New returns a new ZstdStrategy +func New(reqs Requires) compression.Compressor { + return &ZstdStrategy{ + level: int(reqs.Level), } } diff --git a/pkg/util/compression/selector/fromconfig.go b/pkg/util/compression/selector/fromconfig.go new file mode 100644 index 00000000000000..6b3e5143e560a5 --- /dev/null +++ b/pkg/util/compression/selector/fromconfig.go @@ -0,0 +1,31 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package selector provides correct compression impl to fx +package selector + +import ( + "github.com/DataDog/datadog-agent/comp/core/config" + common "github.com/DataDog/datadog-agent/pkg/util/compression" +) + +// FromConfig will return the compression algorithm specified in the provided config +// under the `serializer_compressor_kind` key. +// If `zstd` the compression level is taken from the serializer_zstd_compressor_level +// key. +func FromConfig(cfg config.Reader) common.Compressor { + kind := cfg.GetString("serializer_compressor_kind") + var level int + + switch kind { + case common.ZstdKind: + level = cfg.GetInt("serializer_zstd_compressor_level") + case common.GzipKind: + // There is no configuration option for gzip compression level when set via this method. + level = 6 + } + + return NewCompressor(kind, level) +} diff --git a/pkg/util/compression/selector/no-zlib-no-zstd.go b/pkg/util/compression/selector/no-zlib-no-zstd.go new file mode 100644 index 00000000000000..282c3e369715be --- /dev/null +++ b/pkg/util/compression/selector/no-zlib-no-zstd.go @@ -0,0 +1,40 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !zlib && !zstd + +// Package selector provides correct compression impl to fx +package selector + +import ( + common "github.com/DataDog/datadog-agent/pkg/util/compression" + implgzip "github.com/DataDog/datadog-agent/pkg/util/compression/impl-gzip" + implnoop "github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// NewCompressor returns a new Compressor based on serializer_compressor_kind +// This function is called only when there is no zlib or zstd tag +func NewCompressor(kind string, level int) common.Compressor { + switch kind { + case common.GzipKind: + return implgzip.New(implgzip.Requires{ + Level: level, + }) + case common.NoneKind: + return implnoop.New() + default: + log.Error("invalid compression set") + return implnoop.New() + } +} + +// NewNoopCompressor returns a new Noop Compressor. It does not do any +// compression, but can be used to create a compressor that does at a later +// point. +// This function is called only when there is no zlib or zstd tag +func NewNoopCompressor() common.Compressor { + return implnoop.New() +} diff --git a/pkg/util/compression/selector/zlib-and-zstd.go b/pkg/util/compression/selector/zlib-and-zstd.go new file mode 100644 index 00000000000000..5f3aafb8a5a06c --- /dev/null +++ b/pkg/util/compression/selector/zlib-and-zstd.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build zlib && zstd + +// Package selector provides correct compression impl to fx +package selector + +import ( + common "github.com/DataDog/datadog-agent/pkg/util/compression" + implgzip "github.com/DataDog/datadog-agent/pkg/util/compression/impl-gzip" + implnoop "github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop" + implzlib "github.com/DataDog/datadog-agent/pkg/util/compression/impl-zlib" + implzstd "github.com/DataDog/datadog-agent/pkg/util/compression/impl-zstd" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// NewCompressor returns a new Compressor based on serializer_compressor_kind +// This function is called only when the zlib build tag is included +func NewCompressor(kind string, level int) common.Compressor { + switch kind { + case common.ZlibKind: + return implzlib.New() + case common.ZstdKind: + return implzstd.New(implzstd.Requires{ + Level: common.ZstdCompressionLevel(level), + }) + case common.GzipKind: + return implgzip.New(implgzip.Requires{ + Level: level, + }) + case common.NoneKind: + return implnoop.New() + default: + log.Error("invalid compression set") + return implnoop.New() + } +} + +// NewNoopCompressor returns a new Noop Compressor. It does not do any +// compression, but can be used to create a compressor that does at a later +// point. +// This function is called only when there is no zlib or zstd tag +func NewNoopCompressor() common.Compressor { + return implnoop.New() +} diff --git a/pkg/util/compression/selector/zlib-no-zstd.go b/pkg/util/compression/selector/zlib-no-zstd.go new file mode 100644 index 00000000000000..4244c59fbff6a0 --- /dev/null +++ b/pkg/util/compression/selector/zlib-no-zstd.go @@ -0,0 +1,46 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build zlib && !zstd + +// Package selector provides correct compression impl to fx +package selector + +import ( + common "github.com/DataDog/datadog-agent/pkg/util/compression" + implgzip "github.com/DataDog/datadog-agent/pkg/util/compression/impl-gzip" + implnoop "github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop" + implzlib "github.com/DataDog/datadog-agent/pkg/util/compression/impl-zlib" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// NewCompressorKind returns a new Compressor based on serializer_compressor_kind +// This function is called only when the zlib build tag is included +func NewCompressor(kind string, level int) common.Compressor { + switch kind { + case common.ZlibKind: + return implzlib.New() + case common.ZstdKind: + log.Warn("zstd build tag not included. using zlib") + return implzlib.New() + case common.GzipKind: + return implgzip.New(implgzip.Requires{ + Level: level, + }) + case common.NoneKind: + return implnoop.New() + default: + log.Error("invalid compression set") + return implnoop.New() + } +} + +// NewNoopCompressor returns a new Noop Compressor. It does not do any +// compression, but can be used to create a compressor that does at a later +// point. +// This function is called only when there is no zlib or zstd tag +func NewNoopCompressor() common.Compressor { + return implnoop.New() +} diff --git a/pkg/util/log/go.mod b/pkg/util/log/go.mod index a439a62283f769..26ceac70912901 100644 --- a/pkg/util/log/go.mod +++ b/pkg/util/log/go.mod @@ -2,7 +2,10 @@ module github.com/DataDog/datadog-agent/pkg/util/log go 1.22.0 -replace github.com/DataDog/datadog-agent/pkg/util/scrubber => ../scrubber +replace ( + github.com/DataDog/datadog-agent/pkg/util/compression => ../compression + github.com/DataDog/datadog-agent/pkg/util/scrubber => ../scrubber +) require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 diff --git a/test/integration/serverless/src/go.sum b/test/integration/serverless/src/go.sum index fdd2df9c60e87e..14914f8c240949 100644 --- a/test/integration/serverless/src/go.sum +++ b/test/integration/serverless/src/go.sum @@ -1,67 +1,125 @@ github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= github.com/DataDog/appsec-internal-go v1.0.0 h1:2u5IkF4DBj3KVeQn5Vg2vjPUtt513zxEYglcqnd500U= github.com/DataDog/appsec-internal-go v1.0.0/go.mod h1:+Y+4klVWKPOnZx6XESG7QHydOaUGEXyH2j/vSg9JiNM= +github.com/DataDog/appsec-internal-go v1.6.0 h1:QHvPOv/O0s2fSI/BraZJNpRDAtdlrRm5APJFZNBxjAw= +github.com/DataDog/appsec-internal-go v1.6.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= github.com/DataDog/datadog-agent/pkg/obfuscate v0.45.0-rc.1 h1:XyYvstMFpSyZtfJHWJm1Sf1meNyCdfhKJrjB6+rUNOk= github.com/DataDog/datadog-agent/pkg/obfuscate v0.45.0-rc.1/go.mod h1:e933RWa4kAWuHi5jpzEuOiULlv21HcCFEVIYegmaB5c= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.50.2 h1:y08IzbpFM/HBaKfgayFZe1FpcbZn6bVPXoZ++93vxv8= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.50.2/go.mod h1:A4nLJvxlg6BO/8/zg81til9yT0uRPuXDFMAzDMpmgn4= github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.0-devel.0.20230725154044-2549ba9058df h1:PbzrhHhs2+RRdKKti7JBSM8ATIeiji2T2cVt/d8GT8k= github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.0-devel.0.20230725154044-2549ba9058df/go.mod h1:5Q39ZOIOwZMnFyRadp+5gH1bFdjmb+Pgxe+j5XOwaTg= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.50.2 h1:7jn5EOu84uph4sd+pB3vF8LnsdTjhh+1/NnCvfNpG4A= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.50.2/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ= github.com/DataDog/datadog-go/v5 v5.1.1/go.mod h1:KhiYb2Badlv9/rofz+OznKoEF5XKTonWyhx5K83AP8E= github.com/DataDog/datadog-go/v5 v5.2.0 h1:kSptqUGSNK67DgA+By3rwtFnAh6pTBxJ7Hn8JCLZcKY= github.com/DataDog/datadog-go/v5 v5.2.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q= +github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= +github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/datadog-lambda-go v1.11.1-0.20231030204701-7ec92619787d h1:wKf99X1SvY0zuCK//Kb2P/EkHOx3cz6wWFOw0/zQiZI= github.com/DataDog/datadog-lambda-go v1.11.1-0.20231030204701-7ec92619787d/go.mod h1:yDYV1eC/P3tzLlumIMf+yzrvIjkuyRfm0KhV1GWLfAs= +github.com/DataDog/datadog-lambda-go v1.20.0 h1:Pw3xo7ANjKXGUjaQEL+Dn4zdnRO6Tk37cTViJm5AfP4= +github.com/DataDog/datadog-lambda-go v1.20.0/go.mod h1:V18Zh3T4QnzUGj89ajYigDqEykYjQjS9IkDzRp4GSu8= github.com/DataDog/go-libddwaf v1.4.2 h1:JgHc+ARmfIzVqEl31HLedVYiNCu3LAQiluvpeNnEx2o= github.com/DataDog/go-libddwaf v1.4.2/go.mod h1:l2+rV8UlnYANNNECQyBE/a1dgc0qP0vg0xcgBscg7Mw= +github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8= +github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY= +github.com/DataDog/go-sqllexer v0.0.10 h1:u07DuRfdlPPmOX/dclb1gcn/zaqWxUiURRRVenKILxc= +github.com/DataDog/go-sqllexer v0.0.10/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.0.1-0.5.2 h1:gld/e3MXfFVB/O8hc3mloP1ayFk75Mmdkmll/9lyd9I= github.com/DataDog/go-tuf v1.0.1-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= +github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I= +github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.5.0 h1:jb72P6GFHPHz2W0onsN51cS3FkaMDcjb0QzgxxA4gDk= github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8CwZNAY= github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= +github.com/DataDog/sketches-go v1.4.5 h1:ki7VfeNz7IcNafq7yI/j5U/YCkO3LJiMDtXz9OMQbyE= +github.com/DataDog/sketches-go v1.4.5/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/aws/aws-lambda-go v1.34.1 h1:M3a/uFYBjii+tDcOJ0wL/WyFi2550FHoECdPf27zvOs= github.com/aws/aws-lambda-go v1.34.1/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= +github.com/aws/aws-lambda-go v1.46.0 h1:UWVnvh2h2gecOlFhHQfIPQcD8pL/f7pVCutmFl+oXU8= +github.com/aws/aws-lambda-go v1.46.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/aws/aws-sdk-go v1.44.168 h1:/NNDLkjcgW8UrvAUk7QvQS9yzo/CFu9Zp4BCiPHoV+E= github.com/aws/aws-sdk-go v1.44.168/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.50.9 h1:yX66aKnEtRc/uNV/1EH8CudRT5aLwVwcSwTBphuVPt8= +github.com/aws/aws-sdk-go v1.50.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.8/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/config v1.18.21 h1:ENTXWKwE8b9YXgQCsruGLhvA9bhg+RqAsL9XEMEsa2c= github.com/aws/aws-sdk-go-v2/config v1.18.21/go.mod h1:+jPQiVPz1diRnjj6VGqWcLK6EzNmQ42l7J3OqGTLsSY= +github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= +github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= github.com/aws/aws-sdk-go-v2/credentials v1.13.20 h1:oZCEFcrMppP/CNiS8myzv9JgOzq2s0d3v3MXYil/mxQ= github.com/aws/aws-sdk-go-v2/credentials v1.13.20/go.mod h1:xtZnXErtbZ8YGXC3+8WfajpMBn5Ga/3ojZdxHq6iI8o= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 h1:jOzQAesnBFDmz93feqKnsTHsXrlwWORNZMFHMV+WLFU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2/go.mod h1:cDh1p6XkSGSwSRIArWRc6+UqAQ7x4alQ0QfpVR6f+co= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 h1:dpbVNUjczQ8Ae3QKHbpHBpfvaVkRdesxpTOe9pTouhU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32/go.mod h1:RudqOgadTWdcS3t/erPQo24pcVEoYyqj/kKW5Vya21I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 h1:QH2kOS3Ht7x+u0gHCh06CXL/h6G8LQJFpZfFBYBNboo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26/go.mod h1:vq86l7956VgFr0/FWQ2BWnK07QC3WYsepKzy33qqY5U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33/go.mod h1:zG2FcwjQarWaqXSCGpgcr3RSjZ6dHGguZSppUL0XR7Q= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 h1:uUt4XctZLhl9wBE1L8lobU3bVN8SNUP7T+olb0bWBO4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26/go.mod h1:Bd4C/4PkVGubtNe5iMXu5BNnaBi/9t/UsFspPt4ram8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= github.com/aws/aws-sdk-go-v2/service/kms v1.19.2 h1:pgOVfu7E6zBddKGks4TvL4YuFsL/oTpiWDIzs4WPLjY= github.com/aws/aws-sdk-go-v2/service/kms v1.19.2/go.mod h1:XH60PhgtbXDXFBzJ2auE6bpIELxAYTnoVFFwPtG8JwY= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 h1:W9PbZAZAEcelhhjb7KuwUtf+Lbc+i7ByYJRuWLlnxyQ= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.9/go.mod h1:2tFmR7fQnOdQlM2ZCEPpFnBIQD1U8wmXmduBgZbOag0= github.com/aws/aws-sdk-go-v2/service/sso v1.12.8 h1:5cb3D6xb006bPTqEfCNaEA6PPEfBXxxy4NNeX/44kGk= github.com/aws/aws-sdk-go-v2/service/sso v1.12.8/go.mod h1:GNIveDnP+aE3jujyUSH5aZ/rktsTM5EvtKnCqBZawdw= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8 h1:NZaj0ngZMzsubWZbrEFSB4rgSQRbFq38Sd6KBxHuOIU= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8/go.mod h1:44qFP1g7pfd+U+sQHLPalAPKnyfTZjJsYR4xIwsJy5o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 h1:Qf1aWwnsNkyAoqDqmdM3nHwN78XQjec27LjM6b9vyfI= github.com/aws/aws-sdk-go-v2/service/sts v1.18.9/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= +github.com/aws/aws-xray-sdk-go v1.8.3 h1:S8GdgVncBRhzbNnNUgTPwhEqhwt2alES/9rLASyhxjU= +github.com/aws/aws-xray-sdk-go v1.8.3/go.mod h1:tv8uLMOSCABolrIF8YCcp3ghyswArsan8dfLCA1ZATk= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -76,6 +134,13 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/ebitengine/purego v0.5.0-alpha h1:pNZNC8WofBTN3Nm196An50C5taL/87BhFR/RzKy2o4k= github.com/ebitengine/purego v0.5.0-alpha/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/ebitengine/purego v0.6.0-alpha.5 h1:EYID3JOAdmQ4SNZYJHu9V6IqOeRQDBYxqKAg9PyoHFY= +github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -89,6 +154,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20230509042627-b1315fad0c5a h1:PEOGDI1kkyW37YqPWHLHc+D20D9+87Wt12TCcfTUo5Q= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -96,11 +163,15 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= +github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/outcaste-io/ristretto v0.2.1 h1:KCItuNIGJZcursqHr3ghO7fc5ddZLEHspL9UR0cQM64= github.com/outcaste-io/ristretto v0.2.1/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= +github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -110,6 +181,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= +github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= +github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= @@ -118,6 +191,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -128,13 +202,23 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= +github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.45.0 h1:zPkkzpIn8tdHZUrVa6PzYd0i5verqiPSkgTd3bSUcpA= github.com/valyala/fasthttp v1.45.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -149,10 +233,13 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -163,6 +250,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -185,6 +274,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -196,31 +287,49 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/DataDog/dd-trace-go.v1 v1.54.1 h1:FJyqbLawXX5CgIkEAV/gUliRiYFa4iQDqvT0lfmobrI= gopkg.in/DataDog/dd-trace-go.v1 v1.54.1/go.mod h1:1JqaWiPl1+vHNYuVNmHOG4HDyHbF84z98BW/hwq8FeU= +gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 h1:Ne7kzWr/br/jwhUJR7CnqPl/mUpNxa6LfgZs0S4htZM= +gopkg.in/DataDog/dd-trace-go.v1 v1.65.1/go.mod h1:beNFIWd/H04d0k96cfltgiDH2+t0T5sDbyYLF3VTXqk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/test/otel/dependencies.go b/test/otel/dependencies.go index e3beed26a8adca..10144f7821c48b 100644 --- a/test/otel/dependencies.go +++ b/test/otel/dependencies.go @@ -17,11 +17,11 @@ import ( "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/logsagentexporter" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor" - strategy_none "github.com/DataDog/datadog-agent/comp/serializer/compression/impl-noop" "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/api" + strategy_none "github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop" ) const ( diff --git a/test/otel/go.mod b/test/otel/go.mod index d3fc25971f5cf6..ea0c3474ed6639 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -26,7 +26,8 @@ replace ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ./../../comp/otelcol/otlp/components/metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ./../../comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil => ./../../comp/otelcol/otlp/testutil - github.com/DataDog/datadog-agent/comp/serializer/compression => ./../../comp/serializer/compression + github.com/DataDog/datadog-agent/comp/serializer/logscompression => ./../../comp/serializer/logscompression + github.com/DataDog/datadog-agent/comp/serializer/metricscompression => ./../../comp/serializer/metricscompression github.com/DataDog/datadog-agent/comp/trace/agent/def => ./../../comp/trace/agent/def github.com/DataDog/datadog-agent/comp/trace/compression/def => ./../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ./../../comp/trace/compression/impl-gzip @@ -71,6 +72,8 @@ replace ( github.com/DataDog/datadog-agent/pkg/util/buf => ../../pkg/util/buf github.com/DataDog/datadog-agent/pkg/util/cgroups => ./../../pkg/util/cgroups github.com/DataDog/datadog-agent/pkg/util/common => ../../pkg/util/common + github.com/DataDog/datadog-agent/pkg/util/compression => ../../pkg/util/compression + github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop => ../../pkg/util/compression/impl-noop github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ./../../pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/executable => ./../../pkg/util/executable github.com/DataDog/datadog-agent/pkg/util/filesystem => ./../../pkg/util/filesystem @@ -103,11 +106,11 @@ require ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/logsagentexporter v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor v0.56.0-rc.1 - github.com/DataDog/datadog-agent/comp/serializer/compression v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/config/setup v0.59.0 github.com/DataDog/datadog-agent/pkg/proto v0.60.0 github.com/DataDog/datadog-agent/pkg/trace v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/util/compression v0.56.0-rc.3 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/datadog v0.117.0 ) @@ -129,13 +132,14 @@ require ( require ( github.com/DataDog/agent-payload/v5 v5.0.140 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/serializer/logscompression v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.59.0 // indirect diff --git a/test/system/dogstatsd/receive_and_forward_test.go b/test/system/dogstatsd/receive_and_forward_test.go index 7961d4b43e8ee6..229f16e72c5288 100644 --- a/test/system/dogstatsd/receive_and_forward_test.go +++ b/test/system/dogstatsd/receive_and_forward_test.go @@ -16,9 +16,9 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl" - "github.com/DataDog/datadog-agent/comp/serializer/compression/common" - "github.com/DataDog/datadog-agent/comp/serializer/compression/selector" "github.com/DataDog/datadog-agent/pkg/metrics/servicecheck" + common "github.com/DataDog/datadog-agent/pkg/util/compression" + "github.com/DataDog/datadog-agent/pkg/util/compression/selector" "github.com/DataDog/datadog-agent/pkg/config/mock" )