SQL normalization in the APM trace agent to improve APM/DBM correlation

comp/trace/config/setup.go

@@ -227,6 +227,9 @@ func applyDatadogConfig(c *config.AgentConfig, core corecompcfg.Component) error
 	if core.IsSet("apm_config.connection_limit") {
 		c.ConnectionLimit = core.GetInt("apm_config.connection_limit")
 	}
+	if core.IsSet("apm_config.sql_obfuscation_mode") {
+		c.SQLObfuscationMode = core.GetString("apm_config.sql_obfuscation_mode")
+	}
 
 	/**
 	 * NOTE: PeerTagsAggregation is on by default as of Q4 2024. To get the default experience,

pkg/config/config_template.yaml

@@ -1270,6 +1270,15 @@ api_key:
   ##        Datadog stores a minimum of 1000 queries (5000000 / 5000) by default.
   #         max_size: 5000000
 
+  ## @sql_obfuscation_mode - string - optional - default: ""
+  ## @env DD_APM_SQL_OBFUSCATION_MODE - string - optional - default: ""
+  ## Obfuscator mode for SQL queries.
+  ## Leave empty to use the default obfuscator.
+  ## Set to "obfuscate_only" to obfuscate the query with the new `sqllexer` obfuscator.
+  ## If you use DBM, set to "obfuscate_and_normalize" to obfuscate and normalize the query for better APM/DBM correlation.
+  #
+  # sql_obfuscation_mode: ""
+
   ## @param filter_tags - object - optional
   ## @env DD_APM_FILTER_TAGS_REQUIRE - object - optional
   ## @env DD_APM_FILTER_TAGS_REJECT - object - optional

pkg/config/setup/apm.go

@@ -155,6 +155,7 @@ func setupAPM(config pkgconfigmodel.Setup) {
 	config.BindEnvAndSetDefault("apm_config.obfuscation.credit_cards.enabled", true, "DD_APM_OBFUSCATION_CREDIT_CARDS_ENABLED")
 	config.BindEnvAndSetDefault("apm_config.obfuscation.credit_cards.luhn", false, "DD_APM_OBFUSCATION_CREDIT_CARDS_LUHN")
 	config.BindEnvAndSetDefault("apm_config.obfuscation.credit_cards.keep_values", []string{}, "DD_APM_OBFUSCATION_CREDIT_CARDS_KEEP_VALUES")
+	config.BindEnvAndSetDefault("apm_config.sql_obfuscation_mode", "", "DD_APM_SQL_OBFUSCATION_MODE")
 	config.BindEnvAndSetDefault("apm_config.debug.port", 5012, "DD_APM_DEBUG_PORT")
 	config.BindEnv("apm_config.features", "DD_APM_FEATURES")
 	config.ParseEnvAsStringSlice("apm_config.features", func(s string) []string {

pkg/flare/envvars.go

@@ -108,6 +108,7 @@ var allowedEnvvarNames = []string{
 	"DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_KEEP_VALUES",
 	"DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_OBFUSCATE_SQL_VALUES",
 	"DD_APM_OBFUSCATION_CACHE_ENABLED",
+	"DD_APM_SQL_OBFUSCATION_MODE",
 	"DD_APM_OBFUSCATION_CACHE_MAX_SIZE",
 	"DD_APM_DEBUG_PORT",
 	"DD_APM_INSTRUMENTATION_ENABLED",

pkg/trace/config/config.go

@@ -121,8 +121,15 @@ type ObfuscationConfig struct {
 	Cache obfuscate.CacheConfig `mapstructure:"cache"`
 }
 
-func obfuscationMode(enabled bool) obfuscate.ObfuscationMode {
-	if enabled {
+func obfuscationMode(conf *AgentConfig, sqllexerEnabled bool) obfuscate.ObfuscationMode {
+	if conf.SQLObfuscationMode != "" {
+		if conf.SQLObfuscationMode == string(obfuscate.ObfuscateOnly) || conf.SQLObfuscationMode == string(obfuscate.ObfuscateAndNormalize) {
+			return obfuscate.ObfuscationMode(conf.SQLObfuscationMode)
+		}
+		log.Warnf("Invalid SQL obfuscator mode %s, falling back to default", conf.SQLObfuscationMode)
+		return ""
+	}
+	if sqllexerEnabled {
 		return obfuscate.ObfuscateOnly
 	}
 	return ""
@@ -136,7 +143,7 @@ func (o *ObfuscationConfig) Export(conf *AgentConfig) obfuscate.Config {
 			ReplaceDigits:    conf.HasFeature("quantize_sql_tables") || conf.HasFeature("replace_sql_digits"),
 			KeepSQLAlias:     conf.HasFeature("keep_sql_alias"),
 			DollarQuotedFunc: conf.HasFeature("dollar_quoted_func"),
-			ObfuscationMode:  obfuscationMode(conf.HasFeature("sqllexer")),
+			ObfuscationMode:  obfuscationMode(conf, conf.HasFeature("sqllexer")),
 		},
 		ES:                   o.ES,
 		OpenSearch:           o.OpenSearch,
@@ -396,6 +403,9 @@ type AgentConfig struct {
 	// Obfuscation holds sensitive data obufscator's configuration.
 	Obfuscation *ObfuscationConfig
 
+	// SQLObfuscationMode holds obfuscator mode.
+	SQLObfuscationMode string
+
 	// MaxResourceLen the maximum length the resource can have
 	MaxResourceLen int
 
@@ -542,6 +552,7 @@ func New() *AgentConfig {
 		AnalyzedRateByServiceLegacy: make(map[string]float64),
 		AnalyzedSpansByService:      make(map[string]map[string]float64),
 		Obfuscation:                 &ObfuscationConfig{},
+		SQLObfuscationMode:          "",
 		MaxResourceLen:              5000,
 
 		GlobalTags: computeGlobalTags(),

releasenotes/notes/sql-normalization-6f499718a85da054.yaml

@@ -0,0 +1,11 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Introduce ``sql_obfuscation_mode`` parameter. The value ``obfuscate_and_normalize`` is recommended for DBM customers to enhance APM/DBM correlation.