main_pr.yml

name: Pull Request trigger

on:
  pull_request:


permissions:
  id-token: write
  contents: write
  pull-requests: write
  issues: write


jobs:
  initialize_workflow:
    runs-on: ubuntu-latest
    outputs:
      run_dataplex: ${{ steps.changed_files.outputs.dataplex_any_changed }}
      run_scenarios: ${{ steps.changed_files.outputs.scenarios_any_changed }}
      run_spark_dataproc: ${{ steps.changed_files.outputs.spark_dataproc_any_changed }}
      ol_release: ${{ steps.get-release.outputs.openlineage_release }}
      any_run: ${{ steps.changed_files.outputs.any_any_changed }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: check file structure
        id: check-structure
        run: ./scripts/check_structure.sh

      - name: check configs
        id: check-configs
        run: ./scripts/check_configs.sh

      - name: Get changed files
        id: changed_files
        uses: tj-actions/changed-files@v45
        with:
          files_yaml: |
            scenarios:
              - consumer/scenarios/**
            dataplex:
              - consumer/consumers/dataplex/**
            spark_dataproc:
              - producer/spark_dataproc/**
            any:
              - consumer/scenarios/**
              - consumer/consumers/**
              - producer/**

      - name: get openlineage release
        id: get-release
        run: |
          echo " any changed value is ${{ steps.changed_files.outputs.any_changed }}"
          openlineage_release=$(cat generated-files/releases.json | jq -c '.[] | select(.name | contains("openlineage")) | .latest_version ' -r)
          echo "openlineage_release=${openlineage_release}" >> $GITHUB_OUTPUT  

  ######## COMPONENT VALIDATION ########

  scenarios:
    needs: initialize_workflow
    if: ${{ needs.initialize_workflow.outputs.run_scenarios == 'true' }}
    uses: ./.github/workflows/check_scenarios.yml
    with:
      get-latest-snapshots: false
      release: ${{ needs.initialize_workflow.outputs.ol_release }}

  dataplex:
    needs:
      - initialize_workflow
      - scenarios
    if: ${{ !failure() && needs.initialize_workflow.outputs.run_dataplex == 'true' }}
    uses: ./.github/workflows/consumer_dataplex.yml
    secrets:
      gcpKey: ${{ secrets.GCP_SA_KEY }}
    with:
      release: ${{ needs.initialize_workflow.outputs.ol_release }}

  spark_dataproc:
    needs: initialize_workflow
    if: ${{ needs.initialize_workflow.outputs.run_spark_dataproc == 'true' }}
    uses: ./.github/workflows/producer_spark_dataproc.yml
    secrets:
      gcpKey: ${{ secrets.GCP_SA_KEY }}
      postgresqlUser: ${{ secrets.POSTGRESQL_USER }}
      postgresqlPassword: ${{ secrets.POSTGRESQL_PASSWORD }}
    with:
      release: ${{ needs.initialize_workflow.outputs.ol_release }}
      get-latest-snapshots: 'false'

  ######## COLLECTION OF REPORTS AND EXECUTE APPROPRIATE ACTIONS ########

  collect-and-compare-reports:
    needs:
      - initialize_workflow
      - scenarios
      - dataplex
      - spark_dataproc
    if: ${{ !failure() && needs.initialize_workflow.outputs.any_run  == 'true'}}
    uses: ./.github/workflows/collect_and_compare_reports.yml
    with:
      fail-for-new-failures: true

  generate-compatibility-tables:
    needs:
      - collect-and-compare-reports
    uses: ./.github/workflows/generate_compatibility_tables.yml