Merge branch 'main' into close-apply-async-celery-spans

.circleci/config.templ.yml

@@ -581,25 +581,6 @@ jobs:
           wait: cassandra
           pattern: 'cassandra'
 
-  celery:
-    <<: *contrib_job_large
-    docker:
-      - image: *ddtrace_dev_image
-      - image: *redis_image
-      - image: *rabbitmq_image
-      - image: *testagent_image
-        environment:
-        - LOG_LEVEL=DEBUG
-        - SNAPSHOT_DIR=/snapshots
-        - PORT=9126
-        - SNAPSHOT_CI=1
-        - DD_POOL_TRACE_CHECK_FAILURES=true
-        - DD_DISABLE_ERROR_RESPONSES=true
-        - ENABLED_CHECKS=trace_stall,meta_tracer_version_header,trace_content_length,trace_peer_service,trace_dd_service # disable flaky content length check
-    steps:
-      - run_test:
-          pattern: 'celery'
-
   consul:
     <<: *contrib_job_small
     docker:
@@ -722,17 +703,6 @@ jobs:
           wait: postgres
           pattern: 'aiopg'
 
-  kombu:
-    <<: *contrib_job
-    docker:
-      - image: *ddtrace_dev_image
-      - image: *rabbitmq_image
-      - *testagent
-    steps:
-      - run_test:
-          wait: rabbitmq
-          pattern: 'kombu'
-
   build_docs:
     # build documentation and store as an artifact
     executor: ddtrace_dev

.gitlab/services.yml

@@ -74,4 +74,6 @@
       POSTGRES_PASSWORD: postgres
       POSTGRES_USER: postgres
       POSTGRES_DB: postgres
-
+  rabbitmq:
+    name: registry.ddbuild.io/images/mirror/rabbitmq:3.12.7-management-alpine
+    alias: rabbitmq

.gitlab/tests/appsec.yml

@@ -29,6 +29,8 @@ appsec iast memcheck:
   parallel: 5
   variables:
     SUITE_NAME: "appsec_iast_memcheck"
+    CI_DEBUG_TRACE: "true" 
+    PYTEST_ADDOPTS: "-v -s" 
   retry: 2
 
 appsec threats django:
@@ -54,7 +56,7 @@ appsec threats fastapi:
 
 appsec aggregated leak testing:
   extends: .test_base_hatch
-  parallel: 6
+  parallel: 5
   variables:
     SUITE_NAME: "appsec_aggregated_leak_testing"
   retry: 2

.gitlab/tests/contrib.yml

@@ -33,6 +33,23 @@ bottle:
   variables:
     SUITE_NAME: "bottle"
 
+celery:
+  extends: .test_base_riot_snapshot    
+  services:
+    - !reference [.test_base_riot_snapshot, services]
+    - !reference [.services, rabbitmq]
+    - !reference [.services, redis]
+  variables:
+    SUITE_NAME: "celery"
+    LOG_LEVEL: DEBUG
+    SNAPSHOT_DIR: /snapshots
+    PORT: 9126
+    SNAPSHOT_CI: 1
+    DD_POOL_TRACE_CHECK_FAILURES: true
+    DD_DISABLE_ERROR_RESPONSES: true
+    ENABLED_CHECKS: trace_stall,meta_tracer_version_header,trace_content_length,trace_peer_service,trace_dd_service # disable flaky content length check
+
+
 cherrypy:
   extends: .test_base_riot_snapshot
   variables:
@@ -55,7 +72,7 @@ django_hosts:
   extends: .test_base_riot_snapshot
   variables:
     SUITE_NAME: 'django_hosts$'
-    
+
 djangorestframework:
   extends: .test_base_riot_snapshot
   services:
@@ -159,6 +176,14 @@ kafka:
     TEST_KAFKA_HOST: "kafka"
     TEST_KAFKA_PORT: "29092"
 
+kombu:
+  extends: .test_base_riot_snapshot
+  services:
+    - !reference [.test_base_riot_snapshot, services]
+    - !reference [.services, rabbitmq]
+  variables:
+    SUITE_NAME: "kombu"
+
 logbook:
   extends: .test_base_riot_snapshot
   variables:
@@ -188,7 +213,7 @@ opentracer:
   extends: .test_base_riot
   variables:
     SUITE_NAME: "opentracer"
-    
+
 pylibmc:
   extends: .test_base_riot_snapshot
   services:
@@ -290,7 +315,7 @@ structlog:
   extends: .test_base_riot_snapshot
   variables:
     SUITE_NAME: "structlog"
-    
+
 sourcecode:
   extends: .test_base_riot
   variables:

hatch.toml

@@ -331,7 +331,7 @@ test = [
 ]
 
 [[envs.appsec_aggregated_leak_testing.matrix]]
-python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
 
 # if you add or remove a version here, please also update the parallelism parameter

scripts/iast/test_leak_functions.py

@@ -23,13 +23,14 @@ def parse_arguments():
 
 
 def test_iast_leaks(iterations: int, fail_percent: float, print_every: int):
-    if iterations < 100000:
+    if iterations < 60000:
         print(
-            "Warning: running with %d iterations. At least 100.000 are recommended to stabilize the RSS info"
-            % iterations
+            "Error: not running with %d iterations. At least 60.000 are needed to stabilize the RSS info" % iterations
         )
+        sys.exit(1)
+
     try:
-        half_iterations = iterations // 2
+        mem_reference_iterations = 50000
         print("Test %d iterations" % iterations)
         current_rss = 0
         half_rss = 0
@@ -44,8 +45,9 @@ def test_iast_leaks(iterations: int, fail_percent: float, print_every: int):
             assert is_pyobject_tainted(result)
             reset_context()
 
-            if i == half_iterations:
+            if i == mem_reference_iterations:
                 half_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
+                print("Reference usage taken at %d iterations: %f" % (i, half_rss))
 
             current_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
 
@@ -59,13 +61,13 @@ def test_iast_leaks(iterations: int, fail_percent: float, print_every: int):
         percent_increase = ((final_rss - half_rss) / half_rss) * 100
         if percent_increase > fail_percent:
             print(
-                f"Failed: memory increase from half-point ({half_iterations} iterations) is "
+                f"Failed: memory increase from reference-point ({mem_reference_iterations} iterations) is "
                 f"{percent_increase:.2f}% which is greater than {fail_percent}%"
             )
             return 1
 
         print(
-            f"Success: memory increase is {percent_increase:.2f}% from half-point ({half_iterations} "
+            f"Success: memory increase is {percent_increase:.2f}% from reference-point ({mem_reference_iterations} "
             f"iterations) which is less than {fail_percent}%"
         )
         return 0

tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py

@@ -5,4 +5,4 @@ def test_aggregated_leaks():
     with override_env({"DD_IAST_ENABLED": "True"}):
         from scripts.iast.test_leak_functions import test_iast_leaks
 
-        assert test_iast_leaks(100000, 2.0, 100) == 0
+        assert test_iast_leaks(75000, 2.0, 100) == 0