From baad07ac4fafbc477e0a969a84861339ff2a7217 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Tue, 1 Oct 2024 15:57:23 +0200 Subject: [PATCH] ci: always run system tests (#10878) CI: Removes conditions to check wether should run or "dry run" system tests, and just runs them ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) (cherry picked from commit 7f6f3dbe7c2e00fe84cdf944e1121304306fb649) --- .github/workflows/system-tests.yml | 92 +++++++++--------------------- 1 file changed, 28 insertions(+), 64 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index e6bd299e0d5..f57efa3dea5 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -13,44 +13,24 @@ on: types: [checks_requested] jobs: - needs-run: - runs-on: ubuntu-latest - outputs: - outcome: ${{ steps.run_needed.outcome }} - steps: - - uses: actions/checkout@v4 - - id: run_needed - name: Check if run is needed - run: | - git fetch origin ${{ github.event.pull_request.base.sha || github.sha }} - export PATHS=$(git diff --name-only HEAD ${{ github.event.pull_request.base.sha || github.sha }}) - python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/system-tests.yml'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))" - continue-on-error: true - system-tests-build-agent: runs-on: ubuntu-latest - needs: needs-run steps: - name: Checkout system tests - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - name: Build agent - id: build - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' run: ./build.sh -i agent - name: Save id: save - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' run: | docker image save system_tests/agent:latest | gzip > agent_${{ github.sha }}.tar.gz - uses: actions/upload-artifact@v4 - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' with: name: agent_${{ github.sha }} path: | @@ -59,7 +39,6 @@ jobs: system-tests-build-weblog: runs-on: ubuntu-latest - needs: needs-run strategy: matrix: include: @@ -85,13 +64,11 @@ jobs: steps: - name: Checkout system tests - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - name: Checkout dd-trace-py - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/checkout@v4 with: path: 'binaries/dd-trace-py' @@ -101,18 +78,14 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Build - id: build - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' run: ./build.sh -i weblog - name: Save id: save - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' run: | docker image save system_tests/weblog:latest | gzip > ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz - uses: actions/upload-artifact@v4 - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' with: name: ${{ matrix.weblog-variant }}_${{ github.sha }} path: | @@ -121,7 +94,7 @@ jobs: system-tests: runs-on: ubuntu-latest - needs: [needs-run, system-tests-build-agent, system-tests-build-weblog] + needs: [system-tests-build-agent, system-tests-build-weblog] strategy: matrix: weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12] @@ -143,131 +116,126 @@ jobs: steps: - name: Checkout system tests - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - name: Build runner - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: ./.github/actions/install_runner - uses: actions/download-artifact@v4 - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' with: name: ${{ matrix.weblog-variant }}_${{ github.sha }} path: images_artifacts/ - uses: actions/download-artifact@v4 - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' with: name: agent_${{ github.sha }} path: images_artifacts/ - name: docker load - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' id: docker_load run: | docker load < images_artifacts/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz docker load < images_artifacts/agent_${{ github.sha }}.tar.gz - name: Run DEFAULT - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'other' run: ./run.sh DEFAULT - name: Run SAMPLING - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'other' run: ./run.sh SAMPLING - name: Run INTEGRATIONS - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'other' run: ./run.sh INTEGRATIONS - name: Run CROSSED_TRACING_LIBRARIES - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'other' run: ./run.sh CROSSED_TRACING_LIBRARIES - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - name: Run APPSEC_MISSING_RULES - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_MISSING_RULES - name: Run APPSEC_CUSTOM_RULES - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CUSTOM_RULES - name: Run APPSEC_CORRUPTED_RULES - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CORRUPTED_RULES - name: Run APPSEC_RULES_MONITORING_WITH_ERRORS - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_RULES_MONITORING_WITH_ERRORS - name: Run APPSEC_LOW_WAF_TIMEOUT - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_LOW_WAF_TIMEOUT - name: Run APPSEC_CUSTOM_OBFUSCATION - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CUSTOM_OBFUSCATION - name: Run APPSEC_RATE_LIMITER - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec' run: ./run.sh APPSEC_RATE_LIMITER - name: Run APPSEC_STANDALONE - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_STANDALONE - name: Run APPSEC_RUNTIME_ACTIVATION - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RUNTIME_ACTIVATION - name: Run APPSEC_WAF_TELEMETRY - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_WAF_TELEMETRY - name: Run APPSEC_DISABLED - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_DISABLED - name: Run APPSEC_BLOCKING - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_BLOCKING - name: Run APPSEC_BLOCKING_FULL_DENYLIST - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST - name: Run APPSEC_REQUEST_BLOCKING - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_REQUEST_BLOCKING - name: Run APPSEC_RASP - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RASP # The compress step speed up a lot the upload artifact process - name: Compress artifact - if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') + if: always() && steps.docker_load.outcome == 'success' id: compress-artifact run: tar -czvf artifact.tar.gz $(ls | grep logs) - name: Upload artifact uses: actions/upload-artifact@v4 - if: always() && steps.docker_load.outcome == 'success' && (steps.compress-artifact.outcome == 'success' || github.event_name == 'schedule') + if: always() && steps.docker_load.outcome == 'success' with: name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }} path: artifact.tar.gz @@ -276,17 +244,14 @@ jobs: parametric: runs-on: group: "APM Larger Runners" - needs: needs-run env: TEST_LIBRARY: python steps: - name: Checkout system tests - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - name: Checkout dd-trace-py - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/checkout@v4 with: path: 'binaries/dd-trace-py' @@ -295,20 +260,19 @@ jobs: - name: Build runner id: build_runner - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: ./.github/actions/install_runner - name: Run - if: always() && steps.build_runner.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') + if: always() && steps.build_runner.outcome == 'success' run: ./run.sh PARAMETRIC - name: Compress artifact - if: always() && steps.build_runner.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') + if: always() && steps.build_runner.outcome == 'success' run: tar -czvf artifact.tar.gz $(ls | grep logs) - name: Upload artifact uses: actions/upload-artifact@v4 - if: always() && steps.build_runner.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') + if: always() && steps.build_runner.outcome == 'success' with: name: logs_parametric path: artifact.tar.gz