From b35e25ce21b6b5f1b0307b1514a5660fe1c54dd5 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Tue, 10 Sep 2024 16:13:29 +0200 Subject: [PATCH] ci: system-tests not stopping on failure (#10336) CI: Ensure all scenarios are run, but mark the job as failure if any of them failed Sample forced failure: https://github.com/DataDog/dd-trace-py/actions/runs/10790477805?pr=10336 - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) (cherry picked from commit d0de49e53eeb261eb2148a3c043e174832d8fce5) --- .github/workflows/system-tests.yml | 57 +++++++++++++++--------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index f67b78cb0b6..3d4343f2975 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -128,6 +128,7 @@ jobs: - name: docker load if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + id: docker_load run: | docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz @@ -139,98 +140,98 @@ jobs: chmod -R +x venv/bin/* - name: Run DEFAULT - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' run: ./run.sh DEFAULT - name: Run SAMPLING - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' run: ./run.sh SAMPLING - name: Run INTEGRATIONS - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' run: ./run.sh INTEGRATIONS - name: Run CROSSED_TRACING_LIBRARIES - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' run: ./run.sh CROSSED_TRACING_LIBRARIES - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - name: Run APPSEC_MISSING_RULES - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_MISSING_RULES - name: Run APPSEC_CUSTOM_RULES - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CUSTOM_RULES - name: Run APPSEC_CORRUPTED_RULES - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CORRUPTED_RULES - name: Run APPSEC_RULES_MONITORING_WITH_ERRORS - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_RULES_MONITORING_WITH_ERRORS - name: Run APPSEC_LOW_WAF_TIMEOUT - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_LOW_WAF_TIMEOUT - name: Run APPSEC_CUSTOM_OBFUSCATION - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CUSTOM_OBFUSCATION - name: Run APPSEC_RATE_LIMITER - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_RATE_LIMITER - name: Run APPSEC_RUNTIME_ACTIVATION - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RUNTIME_ACTIVATION - name: Run APPSEC_WAF_TELEMETRY - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_WAF_TELEMETRY - name: Run APPSEC_DISABLED - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_DISABLED - name: Run APPSEC_BLOCKING - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_BLOCKING - name: Run APPSEC_BLOCKING_FULL_DENYLIST - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST - name: Run APPSEC_REQUEST_BLOCKING - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_REQUEST_BLOCKING - name: Run APPSEC_RASP - if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RASP # The compress step speed up a lot the upload artifact process - name: Compress artifact - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: always() && steps.docker_load.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') id: compress-artifact run: tar -czvf artifact.tar.gz $(ls | grep logs) - name: Upload artifact uses: actions/upload-artifact@v4 - if: steps.compress-artifact.outcome == 'success' || github.event_name == 'schedule' + if: always() && steps.docker_load.outcome == 'success' && (steps.compress-artifact.outcome == 'success' || github.event_name == 'schedule') with: name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }} path: artifact.tar.gz @@ -259,22 +260,22 @@ jobs: with: python-version: '3.9' - - name: Build + - name: Build runner + id: build_runner if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' run: ./build.sh -i runner - name: Run - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: always() && steps.build_runner.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') run: ./run.sh PARAMETRIC - name: Compress artifact - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: always() && steps.build_runner.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') run: tar -czvf artifact.tar.gz $(ls | grep logs) - name: Upload artifact uses: actions/upload-artifact@v4 - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: always() && steps.build_runner.outcome == 'success' && (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') with: name: logs_parametric path: artifact.tar.gz -