Merge branch '2.20' into backport-12029-to-2.20

.github/CODEOWNERS

@@ -164,12 +164,14 @@ ddtrace/internal/remoteconfig       @DataDog/remote-config @DataDog/apm-core-pyt
 tests/internal/remoteconfig         @DataDog/remote-config @DataDog/apm-core-python
 
 # API SDK
+ddtrace/trace/                                    @DataDog/apm-sdk-api-python
 ddtrace/_trace/                                    @DataDog/apm-sdk-api-python
 ddtrace/opentelemetry/                             @DataDog/apm-sdk-api-python
 ddtrace/internal/opentelemetry                     @DataDog/apm-sdk-api-python
 ddtrace/opentracer/                                @DataDog/apm-sdk-api-python
 ddtrace/propagation/                               @DataDog/apm-sdk-api-python
 ddtrace/filters.py                                 @DataDog/apm-sdk-api-python
+ddtrace/provider.py                                @DataDog/apm-sdk-api-python
 ddtrace/pin.py                                     @DataDog/apm-sdk-api-python
 ddtrace/sampler.py                                 @DataDog/apm-sdk-api-python
 ddtrace/sampling_rule.py                           @DataDog/apm-sdk-api-python

ddtrace/__init__.py

@@ -17,7 +17,7 @@
 # configure ddtrace logger before other modules log
 configure_ddtrace_logger()  # noqa: E402
 
-from .settings import _config as config
+from .settings import _global_config as config
 
 
 # Enable telemetry writer and excepthook as early as possible to ensure we capture any exceptions from initialization

ddtrace/_monkey.py

@@ -12,7 +12,7 @@
 from .internal import telemetry
 from .internal.logger import get_logger
 from .internal.utils import formats
-from .settings import _config as config
+from .settings import _global_config as config
 
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -157,9 +157,6 @@
 }
 
 
-DEFAULT_MODULES_PREFIX = "ddtrace.contrib"
-
-
 class PatchException(Exception):
     """Wraps regular `Exception` class when patching modules"""
 
@@ -170,19 +167,14 @@ class ModuleNotFoundException(PatchException):
     pass
 
 
-def _on_import_factory(module, prefix="ddtrace.contrib", raise_errors=True, patch_indicator=True):
+def _on_import_factory(module, path_f, raise_errors=True, patch_indicator=True):
     # type: (str, str, bool, Union[bool, List[str]]) -> Callable[[Any], None]
     """Factory to create an import hook for the provided module name"""
 
     def on_import(hook):
         # Import and patch module
         try:
-            try:
-                imported_module = importlib.import_module("%s.internal.%s.patch" % (prefix, module))
-            except ImportError:
-                # Some integrations do not have an internal patch module, so we use the public one
-                # FIXME: This is a temporary solution until we refactor the patching logic.
-                imported_module = importlib.import_module("%s.%s" % (prefix, module))
+            imported_module = importlib.import_module(path_f % (module,))
             imported_module.patch()
             if hasattr(imported_module, "patch_submodules"):
                 imported_module.patch_submodules(patch_indicator)
@@ -209,7 +201,7 @@ def on_import(hook):
                         name, True, PATCH_MODULES.get(module) is True, "", version=v
                     )
             elif hasattr(imported_module, "get_version"):
-                # TODO: Ensure every integration defines either get_version or get_versions in their patch.py module
+                # Some integrations/iast patchers do not define get_version
                 version = imported_module.get_version()
                 telemetry.telemetry_writer.add_integration(
                     module, True, PATCH_MODULES.get(module) is True, "", version=version
@@ -258,8 +250,8 @@ def patch_all(**patch_modules):
     load_common_appsec_modules()
 
 
-def patch(raise_errors=True, patch_modules_prefix=DEFAULT_MODULES_PREFIX, **patch_modules):
-    # type: (bool, str, Union[List[str], bool]) -> None
+def patch(raise_errors=True, **patch_modules):
+    # type: (bool, Union[List[str], bool]) -> None
     """Patch only a set of given modules.
 
     :param bool raise_errors: Raise error if one patch fail.
@@ -270,17 +262,22 @@ def patch(raise_errors=True, patch_modules_prefix=DEFAULT_MODULES_PREFIX, **patc
     contribs = {c: patch_indicator for c, patch_indicator in patch_modules.items() if patch_indicator}
     for contrib, patch_indicator in contribs.items():
         # Check if we have the requested contrib.
-        if not os.path.isfile(os.path.join(os.path.dirname(__file__), "contrib", contrib, "__init__.py")):
+        if not os.path.isfile(os.path.join(os.path.dirname(__file__), "contrib", "internal", contrib, "patch.py")):
             if raise_errors:
                 raise ModuleNotFoundException(
-                    "integration module ddtrace.contrib.%s does not exist, "
-                    "module will not have tracing available" % contrib
+                    "integration module ddtrace.contrib.internal.%s.patch does not exist, "
+                    "automatic instrumentation is disabled for this library" % contrib
                 )
         modules_to_patch = _MODULES_FOR_CONTRIB.get(contrib, (contrib,))
         for module in modules_to_patch:
             # Use factory to create handler to close over `module` and `raise_errors` values from this loop
             when_imported(module)(
-                _on_import_factory(contrib, raise_errors=raise_errors, patch_indicator=patch_indicator)
+                _on_import_factory(
+                    contrib,
+                    "ddtrace.contrib.internal.%s.patch",
+                    raise_errors=raise_errors,
+                    patch_indicator=patch_indicator,
+                )
             )
 
         # manually add module to patched modules

ddtrace/_trace/context.py

@@ -12,9 +12,9 @@
 from ddtrace._trace._span_link import SpanLink
 from ddtrace._trace.types import _MetaDictType
 from ddtrace._trace.types import _MetricDictType
-from ddtrace.constants import ORIGIN_KEY
-from ddtrace.constants import SAMPLING_PRIORITY_KEY
-from ddtrace.constants import USER_ID_KEY
+from ddtrace.constants import _ORIGIN_KEY
+from ddtrace.constants import _SAMPLING_PRIORITY_KEY
+from ddtrace.constants import _USER_ID_KEY
 from ddtrace.internal.compat import NumericType
 from ddtrace.internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS
 from ddtrace.internal.constants import W3C_TRACEPARENT_KEY
@@ -72,9 +72,9 @@ def __init__(
         self._is_remote: bool = is_remote
 
         if dd_origin is not None and _DD_ORIGIN_INVALID_CHARS_REGEX.search(dd_origin) is None:
-            self._meta[ORIGIN_KEY] = dd_origin
+            self._meta[_ORIGIN_KEY] = dd_origin
         if sampling_priority is not None:
-            self._metrics[SAMPLING_PRIORITY_KEY] = sampling_priority
+            self._metrics[_SAMPLING_PRIORITY_KEY] = sampling_priority
         if span_links is not None:
             self._span_links = span_links
         else:
@@ -127,16 +127,16 @@ def _update_tags(self, span: "Span") -> None:
     @property
     def sampling_priority(self) -> Optional[NumericType]:
         """Return the context sampling priority for the trace."""
-        return self._metrics.get(SAMPLING_PRIORITY_KEY)
+        return self._metrics.get(_SAMPLING_PRIORITY_KEY)
 
     @sampling_priority.setter
     def sampling_priority(self, value: Optional[NumericType]) -> None:
         with self._lock:
             if value is None:
-                if SAMPLING_PRIORITY_KEY in self._metrics:
-                    del self._metrics[SAMPLING_PRIORITY_KEY]
+                if _SAMPLING_PRIORITY_KEY in self._metrics:
+                    del self._metrics[_SAMPLING_PRIORITY_KEY]
                 return
-            self._metrics[SAMPLING_PRIORITY_KEY] = value
+            self._metrics[_SAMPLING_PRIORITY_KEY] = value
 
     @property
     def _traceparent(self) -> str:
@@ -180,22 +180,22 @@ def _tracestate(self) -> str:
     @property
     def dd_origin(self) -> Optional[Text]:
         """Get the origin of the trace."""
-        return self._meta.get(ORIGIN_KEY)
+        return self._meta.get(_ORIGIN_KEY)
 
     @dd_origin.setter
     def dd_origin(self, value: Optional[Text]) -> None:
         """Set the origin of the trace."""
         with self._lock:
             if value is None:
-                if ORIGIN_KEY in self._meta:
-                    del self._meta[ORIGIN_KEY]
+                if _ORIGIN_KEY in self._meta:
+                    del self._meta[_ORIGIN_KEY]
                 return
-            self._meta[ORIGIN_KEY] = value
+            self._meta[_ORIGIN_KEY] = value
 
     @property
     def dd_user_id(self) -> Optional[Text]:
         """Get the user ID of the trace."""
-        user_id = self._meta.get(USER_ID_KEY)
+        user_id = self._meta.get(_USER_ID_KEY)
         if user_id:
             return str(base64.b64decode(user_id), encoding="utf-8")
         return None
@@ -205,10 +205,10 @@ def dd_user_id(self, value: Optional[Text]) -> None:
         """Set the user ID of the trace."""
         with self._lock:
             if value is None:
-                if USER_ID_KEY in self._meta:
-                    del self._meta[USER_ID_KEY]
+                if _USER_ID_KEY in self._meta:
+                    del self._meta[_USER_ID_KEY]
                 return
-            self._meta[USER_ID_KEY] = str(base64.b64encode(bytes(value, encoding="utf-8")), encoding="utf-8")
+            self._meta[_USER_ID_KEY] = str(base64.b64encode(bytes(value, encoding="utf-8")), encoding="utf-8")
 
     @property
     def _trace_id_64bits(self):

ddtrace/_trace/processor/__init__.py

@@ -14,7 +14,7 @@
 from ddtrace._trace.span import _get_64_highest_order_bits_as_hex
 from ddtrace._trace.span import _is_top_level
 from ddtrace.constants import _APM_ENABLED_METRIC_KEY as MK_APM_ENABLED
-from ddtrace.constants import SAMPLING_PRIORITY_KEY
+from ddtrace.constants import _SAMPLING_PRIORITY_KEY
 from ddtrace.constants import USER_KEEP
 from ddtrace.internal import gitmetadata
 from ddtrace.internal import telemetry
@@ -165,7 +165,7 @@ def process_trace(self, trace: List[Span]) -> Optional[List[Span]]:
                                 # In order to ensure that the agent does not update priority sampling rates
                                 # due to single spans sampling, we set all of these spans to manual keep.
                                 if config._trace_compute_stats:
-                                    span.set_metric(SAMPLING_PRIORITY_KEY, USER_KEEP)
+                                    span.set_metric(_SAMPLING_PRIORITY_KEY, USER_KEEP)
                                 break
 
             return trace

ddtrace/_trace/sampler.py

@@ -12,7 +12,7 @@
 from typing import Tuple  # noqa:F401
 
 from ddtrace import config
-from ddtrace.constants import SAMPLING_LIMIT_DECISION
+from ddtrace.constants import _SAMPLING_LIMIT_DECISION
 
 from ..constants import ENV_KEY
 from ..internal.constants import _PRIORITY_CATEGORY
@@ -22,7 +22,7 @@
 from ..internal.rate_limiter import RateLimiter
 from ..internal.sampling import _get_highest_precedence_rule_matching
 from ..internal.sampling import _set_sampling_tags
-from ..settings import _config as ddconfig
+from ..settings import _global_config as ddconfig
 from .sampling_rule import SamplingRule
 
 
@@ -342,7 +342,7 @@ def sample(self, span):
             # uses DatadogSampler._rate_limit_always_on to override this functionality.
             if sampled:
                 sampled = self.limiter.is_allowed()
-                span.set_metric(SAMPLING_LIMIT_DECISION, self.limiter.effective_rate)
+                span.set_metric(_SAMPLING_LIMIT_DECISION, self.limiter.effective_rate)
         _set_sampling_tags(
             span,
             sampled,

ddtrace/_trace/span.py

@@ -24,17 +24,17 @@
 from ddtrace._trace.types import _MetricDictType
 from ddtrace._trace.types import _TagNameType
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SAMPLING_AGENT_DECISION
+from ddtrace.constants import _SAMPLING_LIMIT_DECISION
+from ddtrace.constants import _SAMPLING_RULE_DECISION
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import ERROR_MSG
 from ddtrace.constants import ERROR_STACK
 from ddtrace.constants import ERROR_TYPE
 from ddtrace.constants import MANUAL_DROP_KEY
 from ddtrace.constants import MANUAL_KEEP_KEY
-from ddtrace.constants import SAMPLING_AGENT_DECISION
-from ddtrace.constants import SAMPLING_LIMIT_DECISION
-from ddtrace.constants import SAMPLING_RULE_DECISION
 from ddtrace.constants import SERVICE_KEY
 from ddtrace.constants import SERVICE_VERSION_KEY
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.constants import USER_KEEP
 from ddtrace.constants import USER_REJECT
 from ddtrace.constants import VERSION_KEY
@@ -327,7 +327,7 @@ def _override_sampling_decision(self, decision: Optional[NumericType]):
         self.context.sampling_priority = decision
         set_sampling_decision_maker(self.context, SamplingMechanism.MANUAL)
         if self._local_root:
-            for key in (SAMPLING_RULE_DECISION, SAMPLING_AGENT_DECISION, SAMPLING_LIMIT_DECISION):
+            for key in (_SAMPLING_RULE_DECISION, _SAMPLING_AGENT_DECISION, _SAMPLING_LIMIT_DECISION):
                 if key in self._local_root._metrics:
                     del self._local_root._metrics[key]
 
@@ -401,7 +401,7 @@ def set_tag(self, key: _TagNameType, value: Any = None) -> None:
             # Also set the `version` tag to the same value
             # DEV: Note that we do no return, we want to set both
             self.set_tag(VERSION_KEY, value)
-        elif key == SPAN_MEASURED_KEY:
+        elif key == _SPAN_MEASURED_KEY:
             # Set `_dd.measured` tag as a metric
             # DEV: `set_metric` will ensure it is an integer 0 or 1
             if value is None:
@@ -458,7 +458,7 @@ def set_tags(self, tags: Dict[_TagNameType, Any]) -> None:
     def set_metric(self, key: _TagNameType, value: NumericType) -> None:
         """This method sets a numeric tag value for the given key."""
         # Enforce a specific constant for `_dd.measured`
-        if key == SPAN_MEASURED_KEY:
+        if key == _SPAN_MEASURED_KEY:
             try:
                 value = int(bool(value))
             except (ValueError, TypeError):

ddtrace/_trace/trace_handlers.py

@@ -18,8 +18,8 @@
 )
 from ddtrace._trace.utils_botocore.span_tags import set_botocore_response_metadata_tags
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
 from ddtrace.contrib.internal.botocore.constants import BOTOCORE_STEPFUNCTIONS_INPUT_KEY
 from ddtrace.contrib.internal.trace_utils import _set_url_tag
@@ -334,7 +334,7 @@ def _on_request_span_modifier(
     # RequestContext` and possibly a url rule
     span.resource = " ".join((request.method, request.path))
 
-    span.set_tag(SPAN_MEASURED_KEY)
+    span.set_tag(_SPAN_MEASURED_KEY)
     # set analytics sample rate with global config enabled
     sample_rate = flask_config.get_analytics_sample_rate(use_global_config=True)
     if sample_rate is not None:
@@ -366,7 +366,7 @@ def _on_request_span_modifier_post(ctx, flask_config, request, req_body):
 
 def _on_traced_get_response_pre(_, ctx: core.ExecutionContext, request, before_request_tags):
     before_request_tags(ctx["pin"], ctx.span, request)
-    ctx.span._metrics[SPAN_MEASURED_KEY] = 1
+    ctx.span._metrics[_SPAN_MEASURED_KEY] = 1
 
 
 def _on_django_finalize_response_pre(ctx, after_request_tags, request, response):

ddtrace/_trace/tracer.py

@@ -30,8 +30,8 @@
 from ddtrace._trace.sampler import DatadogSampler
 from ddtrace._trace.span import Span
 from ddtrace.appsec._constants import APPSEC
+from ddtrace.constants import _HOSTNAME_KEY
 from ddtrace.constants import ENV_KEY
-from ddtrace.constants import HOSTNAME_KEY
 from ddtrace.constants import PID
 from ddtrace.constants import VERSION_KEY
 from ddtrace.internal import agent
@@ -966,7 +966,7 @@ def _start_span(
                 on_finish=[self._on_span_finish],
             )
             if config._report_hostname:
-                span.set_tag_str(HOSTNAME_KEY, hostname.get_hostname())
+                span.set_tag_str(_HOSTNAME_KEY, hostname.get_hostname())
 
         if not span._parent:
             span.set_tag_str("runtime-id", get_runtime_id())

ddtrace/_trace/utils_botocore/span_tags.py

@@ -7,8 +7,8 @@
 from ddtrace import config
 from ddtrace._trace.utils_botocore.aws_payload_tagging import AWSPayloadTagging
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.ext import SpanKind
 from ddtrace.ext import aws
 from ddtrace.ext import http
@@ -23,7 +23,7 @@ def set_botocore_patched_api_call_span_tags(span: Span, instance, args, params,
     span.set_tag_str(COMPONENT, config.botocore.integration_name)
     # set span.kind to the type of request being performed
     span.set_tag_str(SPAN_KIND, SpanKind.CLIENT)
-    span.set_tag(SPAN_MEASURED_KEY)
+    span.set_tag(_SPAN_MEASURED_KEY)
 
     if args:
         # DEV: join is the fastest way of concatenating strings that is compatible

ddtrace/_trace/utils_redis.py

@@ -7,8 +7,8 @@
 from typing import Optional
 
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
+from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
-from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
 from ddtrace.contrib.internal.redis_utils import _extract_conn_tags
 from ddtrace.ext import SpanKind
@@ -30,7 +30,7 @@ def _set_span_tags(
     span.set_tag_str(SPAN_KIND, SpanKind.CLIENT)
     span.set_tag_str(COMPONENT, config_integration.integration_name)
     span.set_tag_str(db.SYSTEM, redisx.APP)
-    span.set_tag(SPAN_MEASURED_KEY)
+    span.set_tag(_SPAN_MEASURED_KEY)
     if query is not None:
         span_name = schematize_cache_operation(redisx.RAWCMD, cache_provider=redisx.APP)  # type: ignore[operator]
         span.set_tag_str(span_name, query)

ddtrace/appsec/_iast/_iast_request_context.py

@@ -14,7 +14,7 @@
 from ddtrace.appsec._iast._taint_tracking._context import create_context as create_propagation_context
 from ddtrace.appsec._iast._taint_tracking._context import reset_context as reset_propagation_context
 from ddtrace.appsec._iast.reporter import IastSpanReporter
-from ddtrace.constants import ORIGIN_KEY
+from ddtrace.constants import _ORIGIN_KEY
 from ddtrace.internal import core
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.utils.formats import asbool
@@ -133,8 +133,8 @@ def _create_and_attach_iast_report_to_span(req_span: Span, existing_data: Option
     set_iast_request_enabled(False)
     end_iast_context(req_span)
 
-    if req_span.get_tag(ORIGIN_KEY) is None:
-        req_span.set_tag_str(ORIGIN_KEY, APPSEC.ORIGIN_VALUE)
+    if req_span.get_tag(_ORIGIN_KEY) is None:
+        req_span.set_tag_str(_ORIGIN_KEY, APPSEC.ORIGIN_VALUE)
 
     oce.release_request()