Add Dataproc CloudSql test

Signed-off-by: Dominik Dębowczyk <[email protected]>
OpenLineage · Jan 28, 2025 · 1482886 · 1482886
1 parent 2f033d7
commit 1482886
Show file tree

Hide file tree

Showing 22 changed files with 1,193 additions and 5 deletions.
diff --git a/.github/workflows/main_new_release.yml b/.github/workflows/main_new_release.yml
@@ -90,6 +90,8 @@ jobs:
     uses: ./.github/workflows/producer_spark_dataproc.yml
     secrets:
       gcpKey: ${{ secrets.GCP_SA_KEY }}
+      postgresqlUser: ${{ secrets.POSTGRESQL_USER }}
+      postgresqlPassword: ${{ secrets.POSTGRESQL_PASSWORD }}
     with:
       release: ${{ needs.initialize_workflow.outputs.openlineage_release }}
       get-latest-snapshots: 'false'

diff --git a/.github/workflows/main_pr.yml b/.github/workflows/main_pr.yml
@@ -82,6 +82,8 @@ jobs:
     uses: ./.github/workflows/producer_spark_dataproc.yml
     secrets:
       gcpKey: ${{ secrets.GCP_SA_KEY }}
+      postgresqlUser: ${{ secrets.POSTGRESQL_USER }}
+      postgresqlPassword: ${{ secrets.POSTGRESQL_PASSWORD }}
     with:
       release: ${{ needs.initialize_workflow.outputs.ol_release }}
       get-latest-snapshots: 'false'

diff --git a/.github/workflows/producer_spark_dataproc.yml b/.github/workflows/producer_spark_dataproc.yml
@@ -5,6 +5,10 @@ on:
     secrets:
       gcpKey:
         required: true
+      postgresqlUser:
+        required: true
+      postgresqlPassword:
+        required: true
     inputs:
       release:
         description: "release tag of OpenLineage to use"
@@ -80,6 +84,14 @@ jobs:
           local-file-path: producer/spark_dataproc/runner/get_openlineage_jar.sh
           gcs-path: "gs://open-lineage-e2e/scripts"
           credentials: ${{ steps.gcp-auth.outputs.credentials_file_path }}
+
+      - name: Upload CloudSQL init actions to GCS
+        id: upload-cloud-sql-initialization-actions
+        uses: ./.github/actions/upload_artifacts
+        with:
+          local-file-path: producer/spark_dataproc/runner/cloud_sql_proxy.sh
+          gcs-path: "gs://open-lineage-e2e/scripts"
+          credentials: ${{ steps.gcp-auth.outputs.credentials_file_path }}
 
       - name: Set up Python 3.11
         uses: actions/setup-python@v3
@@ -100,8 +112,8 @@ jobs:
           --region us-west1 \
           --cluster-name dataproc-producer-test-${{ github.run_id }} \
           --credentials-file ${{ steps.gcp-auth.outputs.credentials_file_path }} \
-          --metadata "SPARK_BQ_CONNECTOR_URL=gs://open-lineage-e2e/jars/spark-3.5-bigquery-0.0.1-SNAPSHOT.jar,OPENLINEAGE_SPARK_URL=${{ steps.upload-spark-integration.outputs.uploaded-file }},SPARK_SPANNER_CONNECTOR_URL=gs://open-lineage-e2e/jars/spark-3.1-spanner-1.1.0.jar" \
-          --initialization-actions="${{ steps.upload-initialization-actions.outputs.uploaded-file }}"
+          --metadata 'SPARK_BQ_CONNECTOR_URL=gs://open-lineage-e2e/jars/spark-3.5-bigquery-0.0.1-SNAPSHOT.jar,OPENLINEAGE_SPARK_URL=${{ steps.upload-spark-integration.outputs.uploaded-file }},SPARK_SPANNER_CONNECTOR_URL=gs://open-lineage-e2e/jars/spark-3.1-spanner-1.1.0.jar,enable-cloud-sql-hive-metastore=false,additional-cloud-sql-instances=gcp-open-lineage-testing:us-central1:open-lineage-e2e=tcp:3307,POSTGRESQL_USER=${{ secrets.postgresqlUser }},POSTGRESQL_PASSWORD=${{ secrets.postgresqlPassword }}' \
+          --initialization-actions="${{ steps.upload-initialization-actions.outputs.uploaded-file }},${{ steps.upload-cloud-sql-initialization-actions.outputs.uploaded-file }}"
 #          --metadata "SPARK_BQ_CONNECTOR_URL=${{ steps.upload-spark-bq-connector.outputs.uploaded-file }},OPENLINEAGE_SPARK_URL=${{ steps.upload-spark-integration.outputs.uploaded-file }}" \
 
       - name: Set producer output event dir