Merge branch 'main' into rachel.yang/baggage-max-items-fix

DataDog · Nov 15, 2024 · b5aeca8 · b5aeca8
2 parents 4436db9 + 1aaed62
commit b5aeca8
Show file tree

Hide file tree

Showing 429 changed files with 16,552 additions and 3,579 deletions.
diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml
@@ -4,7 +4,7 @@ default_resource_class: &default_resource_class medium
 ubuntu_base_image: &ubuntu_base_img ubuntu-2004:2023.04.2
 cimg_base_image: &cimg_base_image cimg/base:2022.08
 python310_image: &python310_image cimg/python:3.10.12
-ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
+ddtrace_dev_image: &ddtrace_dev_image ghcr.io/datadog/dd-trace-py/testrunner@sha256:8ca43d46ff34e078bd7bc0662e74e6be38547a98140a5cd4203805f6b214b583
 redis_image: &redis_image redis:4.0-alpine@sha256:3e99741f293147ff406657dda7644c2b88564b80a498cd00da8f905743449c9f
 memcached_image: &memcached_image memcached:1.5-alpine@sha256:48cb7207e3d34871893fa1628f3a4984375153e9942facf82e25935b0a633c8a
 cassandra_image: &cassandra_image cassandra:3.11.7@sha256:495e5752526f7e75d3ad85b6a6bbf3b79714321b17a44255a216c341e3baae11
@@ -188,6 +188,7 @@ commands:
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
                   DD_CIVISIBILITY_AGENTLESS_ENABLED: true
                 no_output_timeout: 5m
+                attempts: 2
                 command: |
                   ulimit -c unlimited
                   ./scripts/run-test-suite '<<parameters.pattern>>' <<pipeline.parameters.coverage>> 1
@@ -521,7 +522,7 @@ jobs:
 
   appsec_integrations:
     <<: *machine_executor
-    parallelism: 7
+    parallelism: 13
     steps:
       - run_test:
           pattern: 'appsec_integrations'

diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml
@@ -33,7 +33,7 @@ jobs:
               cibuildwheel --print-build-identifiers --platform linux --arch x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest"}' \
               && cibuildwheel --print-build-identifiers --platform linux --arch aarch64  | jq -cR '{only: ., os: "arm-4core-linux"}' \
               && cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \
-              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,universal2 | jq -cR '{only: ., os: "macos-12"}'
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,universal2 | jq -cR '{only: ., os: "macos-13"}'
             } | jq -sc
           )
           echo $MATRIX_INCLUDE

diff --git a/.github/workflows/check_old_target_branch.yml b/.github/workflows/check_old_target_branch.yml
@@ -0,0 +1,36 @@
+name: Check for Old Target Branch
+
+on:
+  pull_request:
+
+jobs:
+  check_target_branch:
+    name: "Check for old target branch"
+    runs-on: ubuntu-latest
+    permissions:
+       pull-requests: write
+    steps:
+      - name: Check if target branch is too old to backport
+        id: check-branch
+        run: |
+          # Define regex for branches with major version 0 or 1, or versions from 2.0 to 2.12
+          old_branch_regex="^(0|1)(\\.|$)|^2\\.([0-9]|1[0-2])(\\.|$)"
+          target_branch="${{ github.event.pull_request.base.ref }}"
+          
+          if [[ "$target_branch" =~ $old_branch_regex ]]; then
+              echo "Old target branch detected: $target_branch"
+              echo "old_branch=true" >> $GITHUB_ENV
+          else
+              echo "old_branch=false" >> $GITHUB_ENV
+          fi
+
+      - name: Old branch warning on PR
+        if: env.old_branch == 'true'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            🚫 **This target branch is too old or unsupported. Please update the target branch to continue.**
+
+      - name: Fail the job if branch is old
+        if: env.old_branch == 'true'
+        run: exit 1
diff --git a/.github/workflows/django-overhead-profile.yml b/.github/workflows/django-overhead-profile.yml
@@ -13,9 +13,19 @@ on:
 jobs:
   django-overhead-profile:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - suffix: "-v1"
+            stack_v2: "0"
+          - suffix: "-v2"
+            stack_v2: "1"
     env:
       PREFIX: ${{ github.workspace }}/prefix
       DD_CODE_ORIGIN_FOR_SPANS_ENABLED: "1"
+      DD_PROFILING_ENABLED: "1"
+      DD_PROFILING_STACK_V2_ENABLED: ${{ matrix.stack_v2 }}
+      DD_PROFILING_OUTPUT_PPROF: ${{ github.workspace }}/prefix/artifacts/ddtrace_profile
     defaults:
       run:
         working-directory: ddtrace
@@ -38,6 +48,6 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: django-overhead-profile
+          name: django-overhead-profile${{ matrix.suffix }}
           path: ${{ github.workspace }}/prefix/artifacts
 
diff --git a/.github/workflows/generate-package-versions.yml b/.github/workflows/generate-package-versions.yml
@@ -99,4 +99,30 @@ jobs:
           base: main
           title: "chore: update ${{ env.VENV_NAME }} latest version to ${{ env.NEW_LATEST }}"
           labels: changelog/no-changelog
-          body-path: .github/PULL_REQUEST_TEMPLATE.md
+          body: |
+            Update ${{ env.VENV_NAME }} lockfiles and dependency package lockfiles.
+            This performs the following updates:
+              1) Some ${{ env.VENV_NAME }} lockfiles use ${{ env.VENV_NAME }} `latest`. This will update ${{ env.VENV_NAME }} and dependencies.
+              2) Some ${{ env.VENV_NAME }} lockfiles use a pinned (non-latest) version of ${{ env.VENV_NAME }}, but require the `latest` version of another package. This will update all such packages.
+            
+            ## Checklist
+            - [x] PR author has checked that all the criteria below are met
+            - The PR description includes an overview of the change
+            - The PR description articulates the motivation for the change
+            - The change includes tests OR the PR description describes a testing strategy
+            - The PR description notes risks associated with the change, if any
+            - Newly-added code is easy to change
+            - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
+            - The change includes or references documentation updates if necessary
+            - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
+
+            ## Reviewer Checklist
+            - [ ] Reviewer has checked that all the criteria below are met 
+            - Title is accurate
+            - All changes are related to the pull request's stated goal
+            - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes
+            - Testing strategy adequately addresses listed risks
+            - Newly-added code is easy to change
+            - Release note makes sense to a user of the library
+            - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment
+            - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
diff --git a/.github/workflows/requirements-locks.yml b/.github/workflows/requirements-locks.yml
@@ -11,7 +11,7 @@ jobs:
   validate:
     name: Check requirements lockfiles
     runs-on: ubuntu-latest
-    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:4c8afd048321e702f3605b4ae4d206fcd00e74bac708089cfe7f9c24383dc53b
+    container: ghcr.io/datadog/dd-trace-py/testrunner@sha256:8ca43d46ff34e078bd7bc0662e74e6be38547a98140a5cd4203805f6b214b583
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -165,6 +165,10 @@ jobs:
         if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_MISSING_RULES
 
+      - name: Run APPSEC_AUTO_EVENTS_EXTENDED
+        if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec'
+        run: ./run.sh APPSEC_AUTO_EVENTS_EXTENDED
+
       - name: Run APPSEC_CUSTOM_RULES
         if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CUSTOM_RULES

diff --git a/.github/workflows/test_frameworks.yml b/.github/workflows/test_frameworks.yml
@@ -396,7 +396,7 @@ jobs:
           repository: tiangolo/fastapi
           ref: 0.92.0
           path: fastapi
-      - uses: actions/cache@v3.3.1
+      - uses: actions/cache@v4.1.2
         if: needs.needs-run.outputs.outcome == 'success'
         id: cache
         with:

diff --git a/.gitignore b/.gitignore
@@ -162,6 +162,8 @@ template_venv/
 cloned_venvs/
 # CircleCI generated config
 .circleci/config.gen.yml
+# GitLab CI generated config
+.gitlab/*-gen.yml
 
 # Automatically generated fixtures
 tests/appsec/iast/fixtures/aspects/callers.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 stages:
   - package
-  - tests
+  - tests-gen
+  - tests-trigger
   - shared-pipeline
   - dogfood
   - benchmarks
@@ -10,27 +11,44 @@ stages:
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
-  TESTRUNNER_IMAGE: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:ecc5741ff3e7c8a30363fcd9cca79a371dcea5b4
   # CI_DEBUG_SERVICES: "true"
 
-.testrunner:
-  image: $TESTRUNNER_IMAGE
-  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
-  tags: [ "arch:amd64" ]
-  timeout: 20m
-  before_script:
-    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
-    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
-
 include:
   - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml
   - local: ".gitlab/services.yml" # Include early so others can use the definitions
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/package.yml"
-  - local: ".gitlab/tests.yml"
   - local: ".gitlab/macrobenchmarks.yml"
   - local: ".gitlab/dogfood.yml"
   - local: ".gitlab/release.yml"
+  - local: ".gitlab/testrunner.yml"
+
+tests-gen:
+  stage: tests-gen
+  extends: .testrunner
+  script:
+    - pip install riot==0.20.0
+    - riot -v run --pass-env -s gitlab-gen-config -v
+  needs: []
+  artifacts:
+    paths:
+      - .gitlab/tests-gen.yml
+
+run-tests-trigger:
+ stage: tests-trigger
+ needs: [ tests-gen ]
+ trigger:
+   include:
+     - artifact: .gitlab/tests-gen.yml
+       job: tests-gen
+   strategy: depend
+
+requirements_json_test:
+  rules:
+    - when: on_success
+  variables:
+    REQUIREMENTS_BLOCK_JSON_PATH: ".gitlab/requirements_block.json"
+    REQUIREMENTS_ALLOW_JSON_PATH: ".gitlab/requirements_allow.json"
 
 package-oci:
   needs: [ download_dependency_wheels, download_ddtrace_artifacts ]

diff --git a/.gitlab/download-wheels-from-gh-actions.sh b/.gitlab/download-wheels-from-gh-actions.sh
@@ -6,37 +6,48 @@ if [ -z "$CI_COMMIT_SHA" ]; then
   exit 1
 fi
 
-echo "Querying for RUN_ID"
-
-timeout=600 # 10 minutes
-start_time=$(date +%s)
-end_time=$((start_time + timeout))
-# Loop for 10 minutes waiting for run to appear in github
-while [ $(date +%s) -lt $end_time ]; do
-  RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId")
-  if [ -n "$RUN_ID" ]; then
-    break;
-  fi
-  echo "Waiting for RUN_ID"
-  sleep 20
-done
+RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId")
+if [ -n "$RUN_ID" ]; then
+  # The job has not started yet. Give it time to start
+  sleep 180 # 3 minutes
+
+  echo "Querying for RUN_ID"
+
+  timeout=600 # 10 minutes
+  start_time=$(date +%s)
+  end_time=$((start_time + timeout))
+  # Loop for 10 minutes waiting for run to appear in github
+  while [ $(date +%s) -lt $end_time ]; do
+    RUN_ID=$(gh run ls --repo DataDog/dd-trace-py --commit=$CI_COMMIT_SHA --workflow=build_deploy.yml --json databaseId --jq "first (.[]) | .databaseId")
+    if [ -n "$RUN_ID" ]; then
+      break;
+    fi
+    echo "Waiting for RUN_ID"
+    sleep 60
+  done
+fi
 
 if [ -z "$RUN_ID" ]; then
   echo "RUN_ID not found"
   exit 1
 fi
 
 echo "Found RUN_ID: $RUN_ID"
-echo "Waiting for workflow to finish"
-
-# wait for run to finish
-gh run watch $RUN_ID --interval 45 --exit-status 1 --repo DataDog/dd-trace-py
 
 mkdir pywheels
 cd pywheels
 
-echo "Github workflow finished. Downloading wheels"
+if [[ $(gh run view $RUN_ID --exit-status --json status --jq .status) != "completed" ]]; then
+  echo "Waiting for workflow to finish"
+
+  # Give time to the job to finish
+  sleep 300 # 5 minutes
 
+  # wait for run to finish
+  gh run watch $RUN_ID --interval 60 --exit-status 1 --repo DataDog/dd-trace-py
+fi
+
+echo "Github workflow finished. Downloading wheels"
 # download all wheels
 gh run download $RUN_ID --repo DataDog/dd-trace-py
 

diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml
@@ -42,6 +42,7 @@ variables:
     DD_RUNTIME_METRICS_ENABLED: "true"
     DD_REMOTE_CONFIGURATION_ENABLED: "false"
     DD_INSTRUMENTATION_TELEMETRY_ENABLED: "false"
+    DD_CRASHTRACKING_ENABLED: "false"
 
     K6_OPTIONS_WARMUP_RATE: 40
     K6_OPTIONS_WARMUP_DURATION: 1m

diff --git a/.gitlab/package.yml b/.gitlab/package.yml
@@ -1,22 +1,3 @@
-build_base_venvs:
-  extends: .testrunner
-  stage: package
-  parallel:
-    matrix:
-      - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
-  variables:
-    CMAKE_BUILD_PARALLEL_LEVEL: 12
-    PIP_VERBOSE: 1
-  script:
-    - pip install riot==0.20.0
-    - riot -P -v generate --python=$PYTHON_VERSION
-  artifacts:
-    name: venv_$PYTHON_VERSION
-    paths:
-      - .riot/venv_*
-      - ddtrace/**/*.so*
-      - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*
-
 download_ddtrace_artifacts:
   image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
   tags: [ "arch:amd64" ]

diff --git a/.gitlab/requirements_allow.json b/.gitlab/requirements_allow.json
@@ -0,0 +1,10 @@
+[
+    {"name": "min glibc x64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "glibc:2.17"}},
+    {"name": "ok glibc x64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "glibc:2.23"}},
+    {"name": "high glibc x64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "glibc:3.0"}},
+    {"name": "musl x64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "musl:1.2.2"}},
+    {"name": "min glibc arm64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm64", "libc":  "glibc:2.17"}},
+    {"name": "ok glibc arm64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm64", "libc":  "glibc:2.23"}},
+    {"name": "high glibc arm64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm64", "libc":  "glibc:3.0"}},
+    {"name": "musl arm64", "filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm64", "libc":  "musl:1.2.2"}}
+  ]
diff --git a/.gitlab/requirements_block.json b/.gitlab/requirements_block.json
@@ -0,0 +1,11 @@
+[
+  {"name": "unsupported 2.x glibc x64","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "glibc:2.16"}},
+  {"name": "unsupported 1.x glibc x64","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "glibc:1.22"}},
+  {"name": "unsupported 2.x.x glibc x64","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x64", "libc":  "glibc:2.16.9"}},
+  {"name": "unsupported 2.x glibc arm64","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm64", "libc":  "glibc:2.15"}},
+  {"name": "unsupported 2.x.x glibc x64","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm64", "libc":  "glibc:2.14.9"}},
+  {"name": "glibx x86","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x86", "libc":  "glibc:2.23"}},
+  {"name": "musl x86","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "x86", "libc":  "musl:1.2.2"}},
+  {"name": "glibx arm","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm", "libc":  "glibc:2.23"}},
+  {"name": "musl arm","filepath":  "/some/path", "args": [], "envars":  [], "host": {"os": "linux", "arch": "arm", "libc":  "musl:1.2.2"}}
+]
diff --git a/.gitlab/testrunner.yml b/.gitlab/testrunner.yml
@@ -0,0 +1,8 @@
+.testrunner:
+  image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:7a2e802af76051f82d698919d2837eff18dbb48e
+  # DEV: we have a larger pool of amd64 runners, prefer that over arm64
+  tags: [ "arch:amd64" ]
+  timeout: 20m
+  before_script:
+    - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
+    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/