test.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Test

on:
  push:
  pull_request:
    types: [opened, synchronize, reopened, edited]

jobs:

  milestone-check:
    runs-on: ubuntu-22.04

    continue-on-error: true

    env:
      PR_MILESTONE: "${{ github.event.pull_request.milestone.number }}"

    steps:
      - uses: actions/checkout@v4
      - name: Check if PR is assigned to a milestone
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          if [ -z "$PR_MILESTONE" ]; then
            echo 'No milestone selected for PR'
            exit 1
          fi
          exit 0

  changelog-check:
    runs-on: ubuntu-22.04

    env:
      PR_BODY: "${{ github.event.pull_request.body }}"
      CHANGELOG_ISSUE: ":issue:`${{ github.event.pull_request.number }}`"

    steps:
      - uses: actions/checkout@v4
      - name: Check if PR is mentioned in changelog
        if: ${{ always() }}
        run: |
          if [ -z "${{ github.event.pull_request.number }}" ]; then
            echo 'No PR defined'
          else
            if grep -qE '^\[no changelog\]' <<<"$PR_BODY"; then
              echo 'Marker "[no changelog]" found in PR body'
              if [ "$(grep -F "$CHANGELOG_ISSUE" CHANGELOG.rst)" != "" ]; then
                echo "ERROR: $CHANGELOG_ISSUE found in CHANGELOG.rst."
                exit 1
              else
                echo "OK: $CHANGELOG_ISSUE not found in CHANGELOG.rst"
              fi
            else
              echo 'Marker "[no changelog]" not found in PR body'
              if [ "$(grep -F "$CHANGELOG_ISSUE" CHANGELOG.rst)" == "" ]; then
                echo "ERROR: $CHANGELOG_ISSUE not found in CHANGELOG.rst."
                exit 1
              else
                echo "OK: $CHANGELOG_ISSUE found in CHANGELOG.rst"
              fi
            fi
          fi
          exit 0

  build:
    runs-on: ${{ matrix.os }}

    needs: [milestone-check, changelog-check]

    env:
      FORCE_COLOR: "1"

    # Testing strategy
    # ----------------
    #
    # We have different axes of the testing matrix:
    #
    # OS: Linux, Windows
    # Compiler: GCC-5, GCC-6, GCC-8, Clang-10
    # Python: 3.7 -- 3.10, pypy3
    #
    # Instead of testing all combinations, we try to achieve full coverage
    # across each axis. The main test matrix just represents the Python axis on
    # Linux. The OS=Windows and Compiler axis are added manually.
    #
    # Some cases (Clang compiler) are handled via the Docker-tests.
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04]
        gcc: [gcc-8]
        python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.8']

        include:

          # Test additional compilers with Linux.
          # Note that gcc-5, gcc-6, gcc-8, gcc-12, gcc-13, clang-10, clang-13, clang-14 and clang-15 is handled via Docker.
          - os: ubuntu-20.04
            gcc: gcc-9
            python-version: '3.9'
          - os: ubuntu-22.04
            gcc: gcc-11
            python-version: '3.11'

          # Test minimum and maximum Python version on Windows.
          - os: windows-2019
            gcc: gcc
            python-version: '3.8'
          - os: windows-2019
            gcc: gcc
            python-version: '3.11'

          # Test minimum and maximum Python version on Windows.
          - os: macos-12
            gcc: gcc
            python-version: '3.8'
          - os: macos-12
            gcc: gcc
            python-version: '3.11'

    steps:
      - uses: actions/checkout@v4
      - name: Setup environment
        run: |
          # Enable coverage for specific target configurations
          case "${{ matrix.os }}/${{ matrix.gcc }}/${{ matrix.python-version }}" in
            ubuntu-22.04/gcc-11/3.11) USE_COVERAGE=true ;;
            windows-2019/gcc-8/3.8)   USE_COVERAGE=true ;;
            macos-12/gcc-11/3.9)      USE_COVERAGE=true ;;
            *)                        USE_COVERAGE=false ;;
          esac
          echo "USE_COVERAGE=$USE_COVERAGE" >> $GITHUB_ENV

          # Set the CC environment
          echo "CC=${{ matrix.gcc }}" >> $GITHUB_ENV

        shell: bash
      - name: Install build commands, GCC and libxml2 (Linux)
        if: ${{ startsWith(matrix.os,'ubuntu-') }}
        run: |
          sudo apt update
          sudo apt-get install -y \
            make \
            ninja-build \
            ${{ matrix.gcc }} \
            $(echo ${{ matrix.gcc }} | sed -e 's/gcc/g\+\+/') \
            libxml2-utils
          sudo apt-get clean
      - name: Install msys with GCC (Windows)
        if: ${{ startsWith(matrix.os,'windows-') }}
        uses: msys2/setup-msys2@v2
        with:
          install: gcc make
      - name: Install ninja (Windows)
        if: ${{ startsWith(matrix.os,'windows-') }}
        run: |
          choco install ninja
      - name: Install ninja and libxml2 (MacOs)
        if: ${{ startsWith(matrix.os,'macos-') }}
        run: |
          brew update
          brew install ninja
          brew install libxml2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-${{ hashFiles('noxfile.py', 'doc/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python3 -m pip install nox requests
      - name: Lint files
        run: |
          nox --non-interactive --session lint
      - name: Test with pytest
        run: |
          nox --non-interactive --session tests -- --archive_differences
      - name: Upload pytest test results
        if: ${{ failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: diffs-${{ matrix.os }}-${{ matrix.gcc }}-${{ matrix.python-version }}
          path: tests/diff.zip
      - name: Upload coverage to Codecov
        if: ${{ env.USE_COVERAGE == 'true' }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          env_vars: OS,PYTHON
          fail_ci_if_error: true
          disable_search: true
          plugins: pycoverage
          files: ./coverage.xml
          name: ${{ matrix.os }}-${{ matrix.gcc }}
          verbose: true
      - name: Generate documentation
        if: ${{ (! startsWith(matrix.python-version,'pypy')) && (! startsWith(matrix.os,'windows-'))}}
        run: |
          nox --non-interactive --session doc || exit 1

          # Check if files are modified (outdated screenshots)
          Status=$(git status --porcelain | grep -F '.jpeg' || exit 0)
          if [ -n "$Status" ] ; then
            echo "Following files are modified:"
            echo "$Status"

            echo "Please regenerate the screenshots!"
            exit 1
          fi
          exit 0
      - name: Test bundle of app
        run: |
          nox --non-interactive --session bundle_app
        if: ${{ ! startsWith(matrix.python-version,'pypy') }}
      - name: Build wheel
        run: |
          nox --non-interactive --session build_wheel
      - name: Check wheel
        run: |
          nox --non-interactive --session check_wheel

  run-docker:
    runs-on: ubuntu-22.04
    needs: [milestone-check, changelog-check]

    strategy:
      fail-fast: false
      matrix:
        gcc: [gcc-5, gcc-6, gcc-8, gcc-9, gcc-10, gcc-11, gcc-12, gcc-13, clang-10, clang-13, clang-14, clang-15]

    steps:
      - name: Setup environment
        run: |
          # Enable coverage for specific target configurations
          case "${{ matrix.gcc }}" in
            gcc-5)    USE_COVERAGE=true ;;
            gcc-13)   USE_COVERAGE=true ;;
            clang-10) USE_COVERAGE=true ;;
            clang-15) USE_COVERAGE=true ;;
            *)        USE_COVERAGE=false ;;
          esac
          echo "USE_COVERAGE=$USE_COVERAGE" >> $GITHUB_ENV
        shell: bash
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python3 -m pip install nox requests
      - uses: actions/checkout@v4
      - name: Build Docker
        run: |
          python3 -m nox --non-interactive --session "docker_build_compiler(${{ matrix.gcc }})"
      - name: Lint files (in container)
        run: |
          python3 -m nox --non-interactive --session "docker_run_compiler(${{ matrix.gcc }})" -- --session lint
      - name: Test with pytest (in container)
        run: |
          python3 -m nox --non-interactive --session "docker_run_compiler(${{ matrix.gcc }})" -- --session tests
      - name: Upload coverage to Codecov
        if: ${{ env.USE_COVERAGE == 'true' }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          env_vars: OS,PYTHON
          fail_ci_if_error: true
          disable_search: true
          plugins: pycoverage
          files: ./coverage.xml
          name: docker-${{ matrix.gcc }}
          verbose: true
      - name: Upload LCOV coverage to Codecov
        if: ${{ (matrix.gcc == 'gcc-5') || (matrix.gcc == 'clang-10') }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          env_vars: OS,PYTHON
          fail_ci_if_error: true
          disable_search: true
          files: ./tests/nested/reference/${{ matrix.gcc }}/coverage.lcov
          name: docker-${{ matrix.gcc }}-lcov
          verbose: true
      - name: Generate documentation (in container)
        run: |
          python3 -m nox --non-interactive --session "docker_run_compiler(${{ matrix.gcc }})" -- --session doc
      - name: Test bundle of app (in container)
        if: ${{ (matrix.gcc != 'gcc-5') && (matrix.gcc != 'gcc-6') }} # Uses Ubuntu 18.04
        run: |
          python3 -m nox --non-interactive --session "docker_run_compiler(${{ matrix.gcc }})" -- --session bundle_app
      - name: Build wheel (in container)
        run: |
          python3 -m nox --non-interactive --session "docker_run_compiler(${{ matrix.gcc }})" -- --session build_wheel
      - name: Check wheel (in container)
        run: |
          python3 -m nox --non-interactive --session "docker_run_compiler(${{ matrix.gcc }})" -- --session check_wheel