tTc #16
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: regression_tests | ||
| on: | ||
| push: | ||
| pull_request: | ||
| schedule: | ||
| - cron: '0 0 * * *' # every day at midnight | ||
| workflow_dispatch: | ||
| inputs: | ||
| default_only: | ||
| description: 'Test default scenarios only' | ||
| required: false | ||
| type: boolean | ||
| commit_id: | ||
| description: 'Commit ID' | ||
| required: false | ||
| default: '' | ||
| marker: | ||
| description: 'Pytest Marker (defined in pytest.ini)' | ||
| required: false | ||
| default: '' | ||
| type: choice | ||
| options: | ||
| - '' | ||
| - basic | ||
| - ipfix | ||
| - bmp | ||
| - bgp | ||
| - avro | ||
| - json | ||
| - redis | ||
| - signals | ||
| - ha | ||
| jobs: | ||
| ### Step 0: Build Traffic Reproducer Images | ||
| ### Step 1: Build Traffic Reproducer Images | ||
| traf-repro-docker: | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout pmacct | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: pmacct | ||
| submodules: recursive | ||
| - name: Create Traffic Reproducer Docker Images | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install docker | ||
| cd pmacct/test-framework | ||
| tools/pcap_player/build_docker_image.sh | ||
| - name: Check Images and Save as Artifacts | ||
| run: | | ||
| echo "Checking Images..." | ||
| docker images | grep _build | ||
| echo | ||
| echo "Saving images as artifacts..." | ||
| mkdir -p /tmp/docker/ | ||
| docker save -o /tmp/docker/traffic_reproducer_docker_images.tar traffic-reproducer:_build | ||
| - name: Upload Artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| retention-days: 1 | ||
| name: traffic_reproducer_docker_images | ||
| path: /tmp/docker | ||
| ### Step 2: Retrieve all other necessary images from Docker Hub and store as artifacts | ||
| ### (to avoid possible Docker Hub pull limits) | ||
| hub-pulled-docker: | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout pmacct | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: pmacct | ||
| - name: Download images and prepare artifacts | ||
| run: | | ||
| cd pmacct/test-framework | ||
| source settings.conf | ||
| docker image pull $ZOOKEEPER_IMG | ||
| docker image pull $KAFKA_IMG | ||
| docker image pull $SCHEMAREGISTRY_IMG | ||
| docker image pull $REDIS_IMG | ||
| echo "List Images" | ||
| docker images | grep 'confluentinc\|redis' | ||
| echo | ||
| echo "Saving images as artifacts..." | ||
| mkdir -p /tmp/docker/ | ||
| docker save -o /tmp/docker/hub_pulled_docker_images.tar $ZOOKEEPER_IMG $KAFKA_IMG $SCHEMAREGISTRY_IMG $REDIS_IMG | ||
| - name: Upload Artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| retention-days: 1 | ||
| name: hub_pulled_docker_images | ||
| path: /tmp/docker | ||
| ### Step 3: Collect tests from framework | ||
| collect-tests: | ||
| runs-on: ubuntu-22.04 | ||
| outputs: | ||
| matrix: ${{ steps.set-matrix.outputs.matrix }} | ||
| steps: | ||
| - name: Checkout pmacct | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: pmacct | ||
| - name: Collect list of tests from tests/ folder | ||
| id: set-matrix | ||
| run: | | ||
| cd pmacct/tests | ||
| MATRIX="{"test": $(find . -mindepth 1 -maxdepth 1 -type d | cut -c 3- | cut -c 1-3 | sort | jq -R -s -c 'split("\n")[:-1]')}" | ||
| echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT | ||
| echo "Collected tests:" | ||
| echo $MATRIX | ||
| ### Step 4: Setup Framework and Run Tests | ||
| pytest-runtests: | ||
| needs: [collect-tests, traf-repro-docker, hub-pulled-docker] | ||
| runs-on: ubuntu-22.04 | ||
| env: | ||
| SKIP: 'false' | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: ${{ fromJson(needs.collect-tests.outputs.matrix) }} | ||
| steps: | ||
| - name: Checkout pmacct | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: pmacct | ||
| submodules: recursive | ||
| - name: Set up Python 3.10 | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Install Framework Requirements | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install librdkafka-dev docker | ||
| pip install --upgrade pip | ||
| pip install -r pmacct/test-framework/requirements.txt | ||
| - name: Dry-run to check collected tests | ||
| if: ${{ success() && inputs.marker != '' }} | ||
| env: | ||
| MARKER: ${{ inputs.marker }} | ||
| run: | | ||
| function intercept_pytest_no_tests_collected { | ||
| exit_code=$? | ||
| if [[ ${exit_code} -eq 5 ]] | ||
| then | ||
| echo "Intercepting pytest exit code 5 (no tests collected) and replacing with 0" | ||
| echo "Setting early_exit to true!" | ||
| echo "SKIP=true" >> "$GITHUB_ENV" | ||
| exit 0 | ||
| fi | ||
| } | ||
| trap intercept_pytest_no_tests_collected EXIT | ||
| cd pmacct/test-framework | ||
| sudo env PATH="$PATH" ./runtest.sh --dry ${{ matrix.test }} --mark="$MARKER" | ||
| trap - EXIT | ||
| - name: Build containers | ||
| uses: ./pmacct/.github/actions/build_containers/ | ||
| with: | ||
| commit-id: ${{GITHUB_SHA}} | ||
|
Check failure on line 174 in .github/workflows/regression_tests.yaml
|
||
| daemons: ${{DAEMONS}} | ||
| ci_num_workers: ${{CI_NUM_WORKERS}} | ||
| ci_deps_dont_check_cert: ${{CI_DEPS_DONT_CHECK_CERTIFICATE}} | ||
| - name: Download Artifacts | ||
| if: ${{ success() && env.SKIP == 'false' }} | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| pattern: '*_docker_images' | ||
| path: /tmp/docker | ||
| - name: Import images in the local registry | ||
| if: ${{ success() && env.SKIP == 'false' }} | ||
| run: | | ||
| docker load -i /tmp/docker/pmacct_docker_images/pmacct_docker_images.tar | ||
| docker load -i /tmp/docker/traffic_reproducer_docker_images/traffic_reproducer_docker_images.tar | ||
| docker load -i /tmp/docker/hub_pulled_docker_images/hub_pulled_docker_images.tar | ||
| echo "List Images" | ||
| docker images | grep 'confluentinc\|redis\|traffic\|_build' | ||
| - name: Run the test(s) | ||
| if: ${{ success() && env.SKIP == 'false' }} | ||
| env: | ||
| DEFAULT_ONLY: ${{ inputs.default_only }} | ||
| MARKER: ${{ inputs.marker }} | ||
| run: | | ||
| cd pmacct/test-framework | ||
| if [[ "$DEFAULT_ONLY" == "true" ]]; then | ||
| if [[ "$MARKER" == "" ]]; then | ||
| sudo env PATH="$PATH" ./runtest.sh ${{ matrix.test }}:00 | ||
| else | ||
| sudo env PATH="$PATH" ./runtest.sh ${{ matrix.test }}:00 --mark="$MARKER" | ||
| fi | ||
| else | ||
| if [[ "$MARKER" == "" ]]; then | ||
| sudo env PATH="$PATH" ./runtest.sh ${{ matrix.test }} | ||
| else | ||
| sudo env PATH="$PATH" ./runtest.sh ${{ matrix.test }} --mark="$MARKER" | ||
| fi | ||
| fi | ||
| - name: Prepare Results Folder for Upload (permissions and folder name) | ||
| if: ${{ !cancelled() && env.SKIP == 'false' }} # always run this step, unless job manually cancelled or we are skipping the test | ||
| run: | | ||
| cd pmacct/test-framework | ||
| sudo chown -R 1000:1000 results/ | ||
| sudo chmod -R 777 results/ | ||
| echo "Adjust results folder name (when : or * is used as part of ./runtest.sh argument...)" | ||
| TEST_FOLDER_NAME=$( echo ${{ matrix.test }} | sed 's/\*/x/g' ) | ||
| TEST_FOLDER_NAME=$( echo $TEST_FOLDER_NAME | sed 's/\:/_/g' ) | ||
| echo "TEST_FOLDER_NAME=$TEST_FOLDER_NAME" >> "$GITHUB_ENV" | ||
| - name: Upload Results Folder | ||
| if: ${{ !cancelled() && env.SKIP == 'false' }} # always run this step, unless job manually cancelled or we are skipping the test | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| retention-days: 7 | ||
| name: "test_results_${{ env.TEST_FOLDER_NAME }}" | ||
| path: pmacct/test-framework/results | ||
| # Step 5.5: Expose Results as Artifacts | ||
| pytest-results: | ||
| if: '!cancelled()' # always run this job, unless job manually cancelled | ||
| needs: pytest-runtests | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Download Results Folder | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| pattern: test_results_* | ||
| path: results | ||
| - name: Set up Python 3.10 | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Install Requirements | ||
| run: | | ||
| pip install --upgrade pip | ||
| pip install pytest-html-merger | ||
| - name: Merge results (test logs, global logs, and reports) in a single folder | ||
| run: | | ||
| mkdir test_results_merged | ||
| cat results/**/pytestlog.log > test_results_merged/pytestlog.log | ||
| cat results/**/monitor.log > test_results_merged/monitor.log | ||
| find results/**/* -maxdepth 0 -type d -exec cp -r {} test_results_merged/ \; | ||
| mkdir tmp_html | ||
| randomname() { head -c16 /dev/urandom | base64 | tr -dc a-zA-Z; } | ||
| for f in results/**/report.html; do mv "$f" tmp_html/`randomname`.html; done | ||
| cp -R test_results_merged/assets tmp_html | ||
| pytest_html_merger -i tmp_html -o test_results_merged/report_merged.html | ||
| cp -R tmp_html/assets test_results_merged | ||
| - name: Upload Merged Results Folder | ||
| id: upload-artifact-results | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| retention-days: 15 | ||
| name: test_results | ||
| path: test_results_merged | ||
| # Workaround until they support multiple URLs deployments to github pages | ||
| - name: Upload HTML report as standard artifact | ||
| id: upload-artifact-html-report | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| retention-days: 15 | ||
| name: pytest_html_report | ||
| path: test_results_merged/report_merged.html | ||
| - name: Create folder to deploy to pages and fix permissions | ||
| if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} | ||
| run: | | ||
| mkdir github-pages | ||
| cp test_results_merged/report_merged.html github-pages/index.html | ||
| echo | ||
| echo "Fix permissions (if necessary)..." | ||
| chmod -c -R +rX github-pages | while read line; do | ||
| echo "::warning title=Invalid file permissions automatically fixed::$line" | ||
| done | ||
| - name: Upload HTML report as github pages artifact (will be deployed by next job) | ||
| if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} | ||
| uses: actions/upload-pages-artifact@v3 | ||
| with: | ||
| path: github-pages/ | ||
| - name: Add info to markdown summary | ||
| env: | ||
| MARKER: ${{ inputs.marker }} | ||
| COMMIT_ID: ${{ inputs.commit_id }} | ||
| DEFAULT_ONLY: ${{ inputs.default_only }} | ||
| run: | | ||
| echo "## :loudspeaker: Pytest Run Information: :loudspeaker:" >> $GITHUB_STEP_SUMMARY | ||
| echo "### Test Results:" >> $GITHUB_STEP_SUMMARY | ||
| echo "The Pytest HTML report is only deployed on github pages for runs triggered from the master branch (for security reasons), \ | ||
| and is only available for the latest CI run. This is due to current Github Actions limitations of not supporting \ | ||
| different URLs for deployments. Nonetheless, reports are anyway available for download as artifacts for up to 15 days \ | ||
| after the test run (see Artifacts section above)." >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Quick links for downloading:**" >> $GITHUB_STEP_SUMMARY | ||
| echo "- Pytest HTML Report: ${{ steps.upload-artifact-html-report.outputs.artifact-url }}'" >> $GITHUB_STEP_SUMMARY | ||
| echo "- Complete Pytest Results (with fine-grained logs for all tests): \ | ||
| ${{ steps.upload-artifact-results.outputs.artifact-url }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo | ||
| echo "### Input Arguments (use for manual workflow dispatch of the CI only):" >> $GITHUB_STEP_SUMMARY | ||
| echo "Marker provided: $MARKER" >> $GITHUB_STEP_SUMMARY | ||
| echo "Commit ID provided: $COMMIT_ID" >> $GITHUB_STEP_SUMMARY | ||
| echo "Default_only: $DEFAULT_ONLY" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo | ||
| # Step 5.6: Deploy HTML report with github pages | ||
| pytest-html-report-deploy: | ||
| if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} # We can only deploy pages from master (security reasons) | ||
| needs: pytest-results | ||
| runs-on: ubuntu-22.04 | ||
| permissions: # Grant GITHUB_TOKEN the permissions required to make a Pages deployment | ||
| pages: write # --> to deploy to Pages | ||
| id-token: write # --> to verify the deployment originates from an appropriate source | ||
| environment: | ||
| name: github-pages | ||
| url: ${{ steps.deployment.outputs.page_url }} # The deployment URL cannot be changed (for now...) | ||
| steps: | ||
| - name: Deploy artifact to Github Pages | ||
| uses: actions/deploy-pages@v4 | ||
| id: deployment | ||
