Merge branch 'main' into munir/fix-ddwaf-circular-import

.github/CODEOWNERS

@@ -115,6 +115,8 @@ ddtrace/contrib/flask_login/        @DataDog/asm-python
 ddtrace/contrib/webbrowser          @DataDog/asm-python
 ddtrace/contrib/urllib              @DataDog/asm-python
 ddtrace/internal/_exceptions.py     @DataDog/asm-python
+ddtrace/internal/appsec/            @DataDog/asm-python
+ddtrace/internal/iast/              @DataDog/asm-python
 tests/appsec/                       @DataDog/asm-python
 tests/contrib/dbapi/test_dbapi_appsec.py    @DataDog/asm-python
 tests/contrib/subprocess            @DataDog/asm-python

.github/workflows/build-and-publish-image.yml

@@ -26,6 +26,9 @@ on:
 jobs:
   build_push:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/pypa_musllinux_1_2_i686.yml

@@ -12,6 +12,9 @@ on:
 jobs:
   build-and-publish:
     uses: ./.github/workflows/build-and-publish-image.yml
+    permissions:
+      contents: read
+      packages: write
     with:
       tags: 'ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:${{ github.sha }},ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest'
       platforms: 'linux/386'

.github/workflows/rust-ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        extension: ["src/core"]
+        extension: ["src/native"]
     steps:
       - uses: actions/checkout@v4
         with:

.gitlab-ci.yml

@@ -59,18 +59,20 @@ onboarding_tests_installer:
     matrix:
       - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine]
 
-
 onboarding_tests_k8s_injection:
   parallel:
     matrix:
-      - WEBLOG_VARIANT:
-          - dd-lib-python-init-test-django
-          - dd-lib-python-init-test-django-gunicorn
-          - dd-lib-python-init-test-django-gunicorn-alpine
-          - dd-lib-python-init-test-django-preinstalled
-          - dd-lib-python-init-test-django-unsupported-package-force
-          - dd-lib-python-init-test-django-uvicorn
-          - dd-lib-python-init-test-protobuf-old
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django, ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS, K8S_LIB_INJECTION_PROFILING_DISABLED, K8S_LIB_INJECTION_PROFILING_ENABLED, K8S_LIB_INJECTION_PROFILING_OVERRIDE]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-gunicorn, dd-lib-python-init-test-django-gunicorn-alpine, dd-lib-python-init-test-django-unsupported-package-force, dd-lib-python-init-test-django-uvicorn, dd-lib-python-init-test-protobuf-old ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_PROFILING_ENABLED]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-preinstalled]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
 
 deploy_to_di_backend:manual:
   stage: shared-pipeline

.gitlab/benchmarks.yml

@@ -25,7 +25,6 @@ variables:
     paths:
       - reports/
     expire_in: 3 months
-  allow_failure: true  # Allow failure, so partial results are uploaded
   variables:
     UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
     UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-py"
@@ -78,6 +77,21 @@ benchmarks-pr-comment:
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
     KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-py
 
+check-big-regressions:
+  stage: benchmarks
+  needs: [ microbenchmarks, benchmark-serverless ]
+  when: always
+  tags: ["arch:amd64"]
+  image: $MICROBENCHMARKS_CI_IMAGE
+  script:
+    - export ARTIFACTS_DIR="$(pwd)/reports/"
+    - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
+    - git clone --branch dd-trace-py https://github.com/DataDog/benchmarking-platform /platform && cd /platform
+    - bp-runner bp-runner.fail-on-regression.yml --debug
+  variables:
+    # Gitlab and BP specific env vars. Do not modify.
+    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-py
+
 benchmark-serverless:
   stage: benchmarks
   image: $SLS_CI_IMAGE

benchmarks/http_propagation_extract/scenario.py

@@ -29,6 +29,8 @@ def generate_headers(self):
     def run(self):
         if self.styles:
             config._propagation_style_extract = self.styles.split(",") if ("," in self.styles) else [self.styles]
+            if "none" in config._propagation_style_extract:
+                config._propagation_style_extract.remove("none")
 
         headers = self.generate_headers()
 

ddtrace/_monkey.py

@@ -176,9 +176,13 @@ def _on_import_factory(module, prefix="ddtrace.contrib", raise_errors=True, patc
 
     def on_import(hook):
         # Import and patch module
-        path = "%s.%s" % (prefix, module)
         try:
-            imported_module = importlib.import_module(path)
+            try:
+                imported_module = importlib.import_module("%s.internal.%s.patch" % (prefix, module))
+            except ImportError:
+                # Some integrations do not have an internal patch module, so we use the public one
+                # FIXME: This is a temporary solution until we refactor the patching logic.
+                imported_module = importlib.import_module("%s.%s" % (prefix, module))
             imported_module.patch()
             if hasattr(imported_module, "patch_submodules"):
                 imported_module.patch_submodules(patch_indicator)
@@ -204,7 +208,8 @@ def on_import(hook):
                     telemetry.telemetry_writer.add_integration(
                         name, True, PATCH_MODULES.get(module) is True, "", version=v
                     )
-            else:
+            elif hasattr(imported_module, "get_version"):
+                # TODO: Ensure every integration defines either get_version or get_versions in their patch.py module
                 version = imported_module.get_version()
                 telemetry.telemetry_writer.add_integration(
                     module, True, PATCH_MODULES.get(module) is True, "", version=version

ddtrace/_trace/context.py

@@ -12,9 +12,9 @@
 from ddtrace._trace._span_link import SpanLink
 from ddtrace._trace.types import _MetaDictType
 from ddtrace._trace.types import _MetricDictType
-from ddtrace.constants import ORIGIN_KEY
-from ddtrace.constants import SAMPLING_PRIORITY_KEY
-from ddtrace.constants import USER_ID_KEY
+from ddtrace.constants import _ORIGIN_KEY
+from ddtrace.constants import _SAMPLING_PRIORITY_KEY
+from ddtrace.constants import _USER_ID_KEY
 from ddtrace.internal.compat import NumericType
 from ddtrace.internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS
 from ddtrace.internal.constants import W3C_TRACEPARENT_KEY
@@ -72,9 +72,9 @@ def __init__(
         self._is_remote: bool = is_remote
 
         if dd_origin is not None and _DD_ORIGIN_INVALID_CHARS_REGEX.search(dd_origin) is None:
-            self._meta[ORIGIN_KEY] = dd_origin
+            self._meta[_ORIGIN_KEY] = dd_origin
         if sampling_priority is not None:
-            self._metrics[SAMPLING_PRIORITY_KEY] = sampling_priority
+            self._metrics[_SAMPLING_PRIORITY_KEY] = sampling_priority
         if span_links is not None:
             self._span_links = span_links
         else:
@@ -127,16 +127,16 @@ def _update_tags(self, span: "Span") -> None:
     @property
     def sampling_priority(self) -> Optional[NumericType]:
         """Return the context sampling priority for the trace."""
-        return self._metrics.get(SAMPLING_PRIORITY_KEY)
+        return self._metrics.get(_SAMPLING_PRIORITY_KEY)
 
     @sampling_priority.setter
     def sampling_priority(self, value: Optional[NumericType]) -> None:
         with self._lock:
             if value is None:
-                if SAMPLING_PRIORITY_KEY in self._metrics:
-                    del self._metrics[SAMPLING_PRIORITY_KEY]
+                if _SAMPLING_PRIORITY_KEY in self._metrics:
+                    del self._metrics[_SAMPLING_PRIORITY_KEY]
                 return
-            self._metrics[SAMPLING_PRIORITY_KEY] = value
+            self._metrics[_SAMPLING_PRIORITY_KEY] = value
 
     @property
     def _traceparent(self) -> str:
@@ -180,22 +180,22 @@ def _tracestate(self) -> str:
     @property
     def dd_origin(self) -> Optional[Text]:
         """Get the origin of the trace."""
-        return self._meta.get(ORIGIN_KEY)
+        return self._meta.get(_ORIGIN_KEY)
 
     @dd_origin.setter
     def dd_origin(self, value: Optional[Text]) -> None:
         """Set the origin of the trace."""
         with self._lock:
             if value is None:
-                if ORIGIN_KEY in self._meta:
-                    del self._meta[ORIGIN_KEY]
+                if _ORIGIN_KEY in self._meta:
+                    del self._meta[_ORIGIN_KEY]
                 return
-            self._meta[ORIGIN_KEY] = value
+            self._meta[_ORIGIN_KEY] = value
 
     @property
     def dd_user_id(self) -> Optional[Text]:
         """Get the user ID of the trace."""
-        user_id = self._meta.get(USER_ID_KEY)
+        user_id = self._meta.get(_USER_ID_KEY)
         if user_id:
             return str(base64.b64decode(user_id), encoding="utf-8")
         return None
@@ -205,10 +205,10 @@ def dd_user_id(self, value: Optional[Text]) -> None:
         """Set the user ID of the trace."""
         with self._lock:
             if value is None:
-                if USER_ID_KEY in self._meta:
-                    del self._meta[USER_ID_KEY]
+                if _USER_ID_KEY in self._meta:
+                    del self._meta[_USER_ID_KEY]
                 return
-            self._meta[USER_ID_KEY] = str(base64.b64encode(bytes(value, encoding="utf-8")), encoding="utf-8")
+            self._meta[_USER_ID_KEY] = str(base64.b64encode(bytes(value, encoding="utf-8")), encoding="utf-8")
 
     @property
     def _trace_id_64bits(self):

ddtrace/_trace/processor/__init__.py

@@ -14,7 +14,7 @@
 from ddtrace._trace.span import _get_64_highest_order_bits_as_hex
 from ddtrace._trace.span import _is_top_level
 from ddtrace.constants import _APM_ENABLED_METRIC_KEY as MK_APM_ENABLED
-from ddtrace.constants import SAMPLING_PRIORITY_KEY
+from ddtrace.constants import _SAMPLING_PRIORITY_KEY
 from ddtrace.constants import USER_KEEP
 from ddtrace.internal import gitmetadata
 from ddtrace.internal import telemetry
@@ -165,7 +165,7 @@ def process_trace(self, trace: List[Span]) -> Optional[List[Span]]:
                                 # In order to ensure that the agent does not update priority sampling rates
                                 # due to single spans sampling, we set all of these spans to manual keep.
                                 if config._trace_compute_stats:
-                                    span.set_metric(SAMPLING_PRIORITY_KEY, USER_KEEP)
+                                    span.set_metric(_SAMPLING_PRIORITY_KEY, USER_KEEP)
                                 break
 
             return trace

ddtrace/_trace/sampler.py

@@ -12,7 +12,7 @@
 from typing import Tuple  # noqa:F401
 
 from ddtrace import config
-from ddtrace.constants import SAMPLING_LIMIT_DECISION
+from ddtrace.constants import _SAMPLING_LIMIT_DECISION
 
 from ..constants import ENV_KEY
 from ..internal.constants import _PRIORITY_CATEGORY
@@ -342,7 +342,7 @@ def sample(self, span):
             # uses DatadogSampler._rate_limit_always_on to override this functionality.
             if sampled:
                 sampled = self.limiter.is_allowed()
-                span.set_metric(SAMPLING_LIMIT_DECISION, self.limiter.effective_rate)
+                span.set_metric(_SAMPLING_LIMIT_DECISION, self.limiter.effective_rate)
         _set_sampling_tags(
             span,
             sampled,

ddtrace/_trace/span.py

@@ -24,17 +24,17 @@
 from ddtrace._trace.types import _MetricDictType
 from ddtrace._trace.types import _TagNameType
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SAMPLING_AGENT_DECISION
+from ddtrace.constants import _SAMPLING_LIMIT_DECISION
+from ddtrace.constants import _SAMPLING_RULE_DECISION
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import ERROR_MSG
 from ddtrace.constants import ERROR_STACK
 from ddtrace.constants import ERROR_TYPE
 from ddtrace.constants import MANUAL_DROP_KEY
 from ddtrace.constants import MANUAL_KEEP_KEY
-from ddtrace.constants import SAMPLING_AGENT_DECISION
-from ddtrace.constants import SAMPLING_LIMIT_DECISION
-from ddtrace.constants import SAMPLING_RULE_DECISION
 from ddtrace.constants import SERVICE_KEY
 from ddtrace.constants import SERVICE_VERSION_KEY
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.constants import USER_KEEP
 from ddtrace.constants import USER_REJECT
 from ddtrace.constants import VERSION_KEY
@@ -327,7 +327,7 @@ def _override_sampling_decision(self, decision: Optional[NumericType]):
         self.context.sampling_priority = decision
         set_sampling_decision_maker(self.context, SamplingMechanism.MANUAL)
         if self._local_root:
-            for key in (SAMPLING_RULE_DECISION, SAMPLING_AGENT_DECISION, SAMPLING_LIMIT_DECISION):
+            for key in (_SAMPLING_RULE_DECISION, _SAMPLING_AGENT_DECISION, _SAMPLING_LIMIT_DECISION):
                 if key in self._local_root._metrics:
                     del self._local_root._metrics[key]
 
@@ -401,7 +401,7 @@ def set_tag(self, key: _TagNameType, value: Any = None) -> None:
             # Also set the `version` tag to the same value
             # DEV: Note that we do no return, we want to set both
             self.set_tag(VERSION_KEY, value)
-        elif key == SPAN_MEASURED_KEY:
+        elif key == _SPAN_MEASURED_KEY:
             # Set `_dd.measured` tag as a metric
             # DEV: `set_metric` will ensure it is an integer 0 or 1
             if value is None:
@@ -458,7 +458,7 @@ def set_tags(self, tags: Dict[_TagNameType, Any]) -> None:
     def set_metric(self, key: _TagNameType, value: NumericType) -> None:
         """This method sets a numeric tag value for the given key."""
         # Enforce a specific constant for `_dd.measured`
-        if key == SPAN_MEASURED_KEY:
+        if key == _SPAN_MEASURED_KEY:
             try:
                 value = int(bool(value))
             except (ValueError, TypeError):

ddtrace/_trace/trace_handlers.py

@@ -18,11 +18,11 @@
 )
 from ddtrace._trace.utils_botocore.span_tags import set_botocore_response_metadata_tags
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
 from ddtrace.contrib.internal.botocore.constants import BOTOCORE_STEPFUNCTIONS_INPUT_KEY
-from ddtrace.contrib.trace_utils import _set_url_tag
+from ddtrace.contrib.internal.trace_utils import _set_url_tag
 from ddtrace.ext import SpanKind
 from ddtrace.ext import db
 from ddtrace.ext import http
@@ -334,7 +334,7 @@ def _on_request_span_modifier(
     # RequestContext` and possibly a url rule
     span.resource = " ".join((request.method, request.path))
 
-    span.set_tag(SPAN_MEASURED_KEY)
+    span.set_tag(_SPAN_MEASURED_KEY)
     # set analytics sample rate with global config enabled
     sample_rate = flask_config.get_analytics_sample_rate(use_global_config=True)
     if sample_rate is not None:
@@ -366,7 +366,7 @@ def _on_request_span_modifier_post(ctx, flask_config, request, req_body):
 
 def _on_traced_get_response_pre(_, ctx: core.ExecutionContext, request, before_request_tags):
     before_request_tags(ctx["pin"], ctx.span, request)
-    ctx.span._metrics[SPAN_MEASURED_KEY] = 1
+    ctx.span._metrics[_SPAN_MEASURED_KEY] = 1
 
 
 def _on_django_finalize_response_pre(ctx, after_request_tags, request, response):

ddtrace/_trace/tracer.py

@@ -30,8 +30,8 @@
 from ddtrace._trace.sampler import DatadogSampler
 from ddtrace._trace.span import Span
 from ddtrace.appsec._constants import APPSEC
+from ddtrace.constants import _HOSTNAME_KEY
 from ddtrace.constants import ENV_KEY
-from ddtrace.constants import HOSTNAME_KEY
 from ddtrace.constants import PID
 from ddtrace.constants import VERSION_KEY
 from ddtrace.internal import agent
@@ -966,7 +966,7 @@ def _start_span(
                 on_finish=[self._on_span_finish],
             )
             if config._report_hostname:
-                span.set_tag_str(HOSTNAME_KEY, hostname.get_hostname())
+                span.set_tag_str(_HOSTNAME_KEY, hostname.get_hostname())
 
         if not span._parent:
             span.set_tag_str("runtime-id", get_runtime_id())

ddtrace/_trace/utils_botocore/span_tags.py

@@ -7,8 +7,8 @@
 from ddtrace import config
 from ddtrace._trace.utils_botocore.aws_payload_tagging import AWSPayloadTagging
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.ext import SpanKind
 from ddtrace.ext import aws
 from ddtrace.ext import http
@@ -23,7 +23,7 @@ def set_botocore_patched_api_call_span_tags(span: Span, instance, args, params,
     span.set_tag_str(COMPONENT, config.botocore.integration_name)
     # set span.kind to the type of request being performed
     span.set_tag_str(SPAN_KIND, SpanKind.CLIENT)
-    span.set_tag(SPAN_MEASURED_KEY)
+    span.set_tag(_SPAN_MEASURED_KEY)
 
     if args:
         # DEV: join is the fastest way of concatenating strings that is compatible

ddtrace/_trace/utils_redis.py

@@ -7,10 +7,10 @@
 from typing import Optional
 
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
-from ddtrace.contrib.redis_utils import _extract_conn_tags
+from ddtrace.contrib.internal.redis_utils import _extract_conn_tags
 from ddtrace.ext import SpanKind
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import db
@@ -30,7 +30,7 @@ def _set_span_tags(
     span.set_tag_str(SPAN_KIND, SpanKind.CLIENT)
     span.set_tag_str(COMPONENT, config_integration.integration_name)
     span.set_tag_str(db.SYSTEM, redisx.APP)
-    span.set_tag(SPAN_MEASURED_KEY)
+    span.set_tag(_SPAN_MEASURED_KEY)
     if query is not None:
         span_name = schematize_cache_operation(redisx.RAWCMD, cache_provider=redisx.APP)  # type: ignore[operator]
         span.set_tag_str(span_name, query)