Merge branch 'main' into upgrade-latest-starlette-version

.circleci/config.templ.yml

@@ -4,7 +4,7 @@ default_resource_class: &default_resource_class medium
 ubuntu_base_image: &ubuntu_base_img ubuntu-2004:2023.04.2
 cimg_base_image: &cimg_base_image cimg/base:2022.08
 python310_image: &python310_image cimg/python:3.10.12
-ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
+ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:8ca43d46ff34e078bd7bc0662e74e6be38547a98140a5cd4203805f6b214b583
 redis_image: &redis_image redis:4.0-alpine@sha256:3e99741f293147ff406657dda7644c2b88564b80a498cd00da8f905743449c9f
 memcached_image: &memcached_image memcached:1.5-alpine@sha256:48cb7207e3d34871893fa1628f3a4984375153e9942facf82e25935b0a633c8a
 cassandra_image: &cassandra_image cassandra:3.11.7@sha256:495e5752526f7e75d3ad85b6a6bbf3b79714321b17a44255a216c341e3baae11
@@ -188,6 +188,7 @@ commands:
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
                   DD_CIVISIBILITY_AGENTLESS_ENABLED: true
                 no_output_timeout: 5m
+                attempts: 2
                 command: |
                   ulimit -c unlimited
                   ./scripts/run-test-suite '<<parameters.pattern>>' <<pipeline.parameters.coverage>> 1
@@ -521,7 +522,7 @@ jobs:
 
   appsec_integrations:
     <<: *machine_executor
-    parallelism: 7
+    parallelism: 13
     steps:
       - run_test:
           pattern: 'appsec_integrations'

.github/workflows/check_old_target_branch.yml

@@ -0,0 +1,36 @@
+name: Check for Old Target Branch
+
+on:
+  pull_request:
+
+jobs:
+  check_target_branch:
+    name: "Check for old target branch"
+    runs-on: ubuntu-latest
+    permissions:
+       pull-requests: write
+    steps:
+      - name: Check if target branch is too old to backport
+        id: check-branch
+        run: |
+          # Define regex for branches with major version 0 or 1, or versions from 2.0 to 2.12
+          old_branch_regex="^(0|1)(\\.|$)|^2\\.([0-9]|1[0-2])(\\.|$)"
+          target_branch="${{ github.event.pull_request.base.ref }}"
+          
+          if [[ "$target_branch" =~ $old_branch_regex ]]; then
+              echo "Old target branch detected: $target_branch"
+              echo "old_branch=true" >> $GITHUB_ENV
+          else
+              echo "old_branch=false" >> $GITHUB_ENV
+          fi
+
+      - name: Old branch warning on PR
+        if: env.old_branch == 'true'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            🚫 **This target branch is too old or unsupported. Please update the target branch to continue.**
+
+      - name: Fail the job if branch is old
+        if: env.old_branch == 'true'
+        run: exit 1

.github/workflows/django-overhead-profile.yml

@@ -13,12 +13,19 @@ on:
 jobs:
   django-overhead-profile:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - suffix: "-v1"
+            stack_v2: "0"
+          - suffix: "-v2"
+            stack_v2: "1"
     env:
       PREFIX: ${{ github.workspace }}/prefix
       DD_CODE_ORIGIN_FOR_SPANS_ENABLED: "1"
       DD_PROFILING_ENABLED: "1"
-      DD_PROFILING_STACK_V2_ENABLED: "1"
-      DD_PROFILING_OUTPUT_PPROF: ${{ github.workspace }}/prefix/artifacts/ddtrace_profile.pprof
+      DD_PROFILING_STACK_V2_ENABLED: ${{ matrix.stack_v2 }}
+      DD_PROFILING_OUTPUT_PPROF: ${{ github.workspace }}/prefix/artifacts/ddtrace_profile
     defaults:
       run:
         working-directory: ddtrace
@@ -41,6 +48,6 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: django-overhead-profile
+          name: django-overhead-profile${{ matrix.suffix }}
           path: ${{ github.workspace }}/prefix/artifacts
 

.github/workflows/requirements-locks.yml

@@ -11,7 +11,7 @@ jobs:
   validate:
     name: Check requirements lockfiles
     runs-on: ubuntu-latest
-    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
+    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:8ca43d46ff34e078bd7bc0662e74e6be38547a98140a5cd4203805f6b214b583
     steps:
       - uses: actions/checkout@v4
         with:

.gitignore

@@ -162,6 +162,8 @@ template_venv/
 cloned_venvs/
 # CircleCI generated config
 .circleci/config.gen.yml
+# GitLab CI generated config
+.gitlab/*-gen.yml
 
 # Automatically generated fixtures
 tests/appsec/iast/fixtures/aspects/callers.py

.gitlab-ci.yml

@@ -1,6 +1,7 @@
 stages:
   - package
-  - tests
+  - tests-gen
+  - tests-trigger
   - shared-pipeline
   - dogfood
   - benchmarks
@@ -10,27 +11,37 @@ stages:
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
-  TESTRUNNER_IMAGE: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:ecc5741ff3e7c8a30363fcd9cca79a371dcea5b4
   # CI_DEBUG_SERVICES: "true"
 
-.testrunner:
-  image: $TESTRUNNER_IMAGE
-  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
-  tags: [ "arch:amd64" ]
-  timeout: 20m
-  before_script:
-    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
-    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
-
 include:
   - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml
   - local: ".gitlab/services.yml" # Include early so others can use the definitions
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/package.yml"
-  - local: ".gitlab/tests.yml"
   - local: ".gitlab/macrobenchmarks.yml"
   - local: ".gitlab/dogfood.yml"
   - local: ".gitlab/release.yml"
+  - local: ".gitlab/testrunner.yml"
+
+tests-gen:
+  stage: tests-gen
+  extends: .testrunner
+  script:
+    - pip install riot==0.20.0
+    - riot -v run --pass-env -s gitlab-gen-config -v
+  needs: []
+  artifacts:
+    paths:
+      - .gitlab/tests-gen.yml
+
+run-tests-trigger:
+ stage: tests-trigger
+ needs: [ tests-gen ]
+ trigger:
+   include:
+     - artifact: .gitlab/tests-gen.yml
+       job: tests-gen
+   strategy: depend
 
 requirements_json_test:
   rules:

.gitlab/download-wheels-from-gh-actions.sh

@@ -6,37 +6,48 @@ if [ -z "$CI_COMMIT_SHA" ]; then
   exit 1
 fi
 
-echo "Querying for RUN_ID"
-
-timeout=600 # 10 minutes
-start_time=$(date +%s)
-end_time=$((start_time + timeout))
-# Loop for 10 minutes waiting for run to appear in github
-while [ $(date +%s) -lt $end_time ]; do
-  RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId")
-  if [ -n "$RUN_ID" ]; then
-    break;
-  fi
-  echo "Waiting for RUN_ID"
-  sleep 20
-done
+RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId")
+if [ -n "$RUN_ID" ]; then
+  # The job has not started yet. Give it time to start
+  sleep 180 # 3 minutes
+
+  echo "Querying for RUN_ID"
+
+  timeout=600 # 10 minutes
+  start_time=$(date +%s)
+  end_time=$((start_time + timeout))
+  # Loop for 10 minutes waiting for run to appear in github
+  while [ $(date +%s) -lt $end_time ]; do
+    RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId")
+    if [ -n "$RUN_ID" ]; then
+      break;
+    fi
+    echo "Waiting for RUN_ID"
+    sleep 60
+  done
+fi
 
 if [ -z "$RUN_ID" ]; then
   echo "RUN_ID not found"
   exit 1
 fi
 
 echo "Found RUN_ID: $RUN_ID"
-echo "Waiting for workflow to finish"
-
-# wait for run to finish
-gh run watch $RUN_ID --interval 45 --exit-status 1 --repo DataDog/dd-trace-py
 
 mkdir pywheels
 cd pywheels
 
-echo "Github workflow finished. Downloading wheels"
+if [[ $(gh run view $RUN_ID --exit-status --json status --jq .status) != "completed" ]]; then
+  echo "Waiting for workflow to finish"
+
+  # Give time to the job to finish
+  sleep 300 # 5 minutes
 
+  # wait for run to finish
+  gh run watch $RUN_ID --interval 60 --exit-status 1 --repo DataDog/dd-trace-py
+fi
+
+echo "Github workflow finished. Downloading wheels"
 # download all wheels
 gh run download $RUN_ID --repo DataDog/dd-trace-py
 

.gitlab/package.yml

@@ -1,22 +1,3 @@
-build_base_venvs:
-  extends: .testrunner
-  stage: package
-  parallel:
-    matrix:
-      - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
-  variables:
-    CMAKE_BUILD_PARALLEL_LEVEL: 12
-    PIP_VERBOSE: 1
-  script:
-    - pip install riot==0.20.0
-    - riot -P -v generate --python=$PYTHON_VERSION
-  artifacts:
-    name: venv_$PYTHON_VERSION
-    paths:
-      - .riot/venv_*
-      - ddtrace/**/*.so*
-      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
-
 download_ddtrace_artifacts:
   image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
   tags: [ "arch:amd64" ]

.gitlab/testrunner.yml

@@ -0,0 +1,8 @@
+.testrunner:
+  image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:7a2e802af76051f82d698919d2837eff18dbb48e
+  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
+  tags: [ "arch:amd64" ]
+  timeout: 20m
+  before_script:
+    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
+    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/

.gitlab/tests.yml

@@ -1,12 +1,32 @@
+stages:
+  - tests
+
 variables:
   RIOT_RUN_CMD: riot -P -v run --exitfirst --pass-env -s
+  REPO_LANG: python # "python" is used everywhere rather than "py"
+  # CI_DEBUG_SERVICES: "true"
+
+
+.testrunner:
+  image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:7a2e802af76051f82d698919d2837eff18dbb48e
+  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
+  tags: [ "arch:amd64" ]
+  timeout: 20m
+  before_script:
+    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
+    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
+
+
+{{services.yml}}
 
 .test_base_hatch:
   extends: .testrunner
   stage: tests
   # Hatch doesn't use pre-built wheels or venvs so we can start them right away
   needs: []
   parallel: 4
+  # DEV: This is the max retries that GitLab currently allows for
+  retry: 2
   script:
     - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} --ddtrace"
     - export _DD_CIVISIBILITY_USE_CI_CONTEXT_PROVIDER=true
@@ -22,6 +42,7 @@ variables:
         hatch run ${env}:test
       done
 
+
 .test_base_hatch_snapshot:
   extends: .test_base_hatch
   services:
@@ -32,13 +53,37 @@ variables:
     # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent
     - export DD_TRACE_AGENT_URL="http://testagent:9126"
 
+
+build_base_venvs:
+  extends: .testrunner
+  stage: tests
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+  variables:
+    CMAKE_BUILD_PARALLEL_LEVEL: 12
+    PIP_VERBOSE: 1
+    DD_PROFILING_NATIVE_TESTS: 1
+  script:
+    - pip install riot==0.20.0
+    - riot -P -v generate --python=$PYTHON_VERSION
+  artifacts:
+    name: venv_$PYTHON_VERSION
+    paths:
+      - .riot/venv_*
+      - ddtrace/**/*.so*
+      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
+      - ddtrace/internal/datadog/profiling/test/test_*
+
 .test_base_riot:
   extends: .testrunner
   stage: tests
   needs: [ build_base_venvs ]
   parallel: 4
   services:
     - !reference [.services, ddagent]
+  # DEV: This is the max retries that GitLab currently allows for
+  retry: 2
   script:
     - pip install riot==0.20.0
     - unset DD_SERVICE
@@ -71,13 +116,20 @@ variables:
     # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the
     # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent
     - export DD_TRACE_AGENT_URL="http://testagent:9126"
+    - ln -s "${CI_PROJECT_DIR}" "/root/project"
+
+slotscheck:
+  extends: .testrunner
+  stage: tests
+  needs: []
+  script:
+    - hatch run slotscheck:_
+
+conftests:
+  extends: .testrunner
+  stage: tests
+  needs: []
+  script:
+    - hatch run meta-testing:meta-testing
 
-include:
-  - local: ".gitlab/tests/appsec.yml"
-  - local: ".gitlab/tests/ci_visibility.yml"
-  - local: ".gitlab/tests/contrib.yml"
-  - local: ".gitlab/tests/core.yml"
-  - local: ".gitlab/tests/debugging.yml"
-  - local: ".gitlab/tests/llmobs.yml"
-  - local: ".gitlab/tests/tracer.yml"
-  - local: ".gitlab/tests/profiling.yml"
+# Required jobs will appear here