end_to_end.yml

name: End to end

on:
  push:
    branches: [ master ]
    paths-ignore:
      - '**.md'
      - '**.py'
      - 'Dockerfile'
      - '.gitignore'
      - 'LICENSE'
      - 'docker.yml'
      - 'formatting.yml'
      - 'test_pack_dock.yml'
      - 'triage.yml'
      - 'acceptance_test.yml'
      - 'web/**'
      - '.github/workflows/**'
  pull_request:
    branches: [ master ]
    paths-ignore:
      - '**.md'
      - '**.py'
      - 'Dockerfile'
      - '.gitignore'
      - 'LICENSE'
      - 'docker.yml'
      - 'formatting.yml'
      - 'test_pack_dock.yml'
      - 'triage.yml'
      - 'acceptance_test.yml'
      - 'web/**'
      - '.github/workflows/**'
  workflow_dispatch:

env:
  python_version: '3.11'
  java_version: '11'
  java_distribution: 'temurin'

jobs:
  validate_gradle_wrapper:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: gradle/wrapper-validation-action@v1
  pack-snapshot:
    needs: validate_gradle_wrapper
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK ${{ env.java_version }}-${{ env.java_distribution }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ env.java_version }}
          distribution: ${{ env.java_distribution }}
      - name: Cache Gradle packages
        uses: actions/cache@v3
        with:
          path: ~/.gradle/caches
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
          restore-keys: ${{ runner.os }}-gradle
      - name: Package cli app jar with Gradle
        uses: gradle/gradle-build-action@v2
        with:
          arguments: shadowJar
      - name: Persist gtfs-validator snapshot jar
        uses: actions/upload-artifact@v4
        with:
          name: gtfs-validator-snapshot
          path: cli/build/libs/gtfs-validator-*-cli.jar
  fetch-urls:
    runs-on: ubuntu-latest-4-cores
    steps:
      - name: Checkout repository code
        uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python_version }}
      - name: Install dependencies
        run: |
          pip install -r scripts/mobility-database-harvester/requirements.txt
      - name: Set URL matrix
        id: set-matrix
        run: |
          DATASETS=$(python3 scripts/mobility-database-harvester/harvest_latest_versions.py -d scripts/mobility-database-harvester/datasets_metadata -l gtfs_latest_versions.json -s)
          echo $DATASETS
          echo "matrix=$DATASETS" >> $GITHUB_OUTPUT
      - name: Persist metadata
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: datasets_metadata
          path: scripts/mobility-database-harvester/datasets_metadata
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
  run-on-data:
    needs: [ fetch-urls, pack-snapshot ]
    # We use machines with more memory to run validation, as large feeds
    # can consume too much heap for default machine instances (see #1304).
    runs-on: ubuntu-latest-4-cores
    strategy:
      matrix: ${{ fromJson(needs.fetch-urls.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v3
      - name: Download latest changes .jar file from previous job
        uses: actions/download-artifact@v4
        with:
          name: gtfs-validator-snapshot
          path: gtfs-validator-snapshot
      - name: Run snapshot validator on queued URLs
        run: |
          queue="${{ matrix.data }}"
          bash ./scripts/queue_runner.sh --exclude-master $queue
        env:
          OUTPUT_BASE: ${{ github.sha }}
      - name: Extract ID from JSON
        id: extract-id
        run: echo "feed_id=$(echo '${{ matrix.data }}' | jq -r '.id')" >> $GITHUB_ENV
      - name: Persist reports
        uses: actions/upload-artifact@v4
        with:
          name: reports_snapshot_${{ env.feed_id }}
          path: ${{ github.sha }}/output
  merge-reports-artifacts:
    runs-on: ubuntu-latest
    needs: [ run-on-data ]
    steps:
      - name: Merge Artifacts
        uses: actions/upload-artifact/merge@v4
        with:
          name: reports_snapshot
          pattern: reports_snapshot_*
          delete-merged: true