fix python version for spark2.4

Which needs a downgrade of pyarrow
criteo · Sep 4, 2024 · 527fdda · 527fdda
1 parent d5ebdaf
commit 527fdda
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -77,18 +77,18 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Build spark-docker
-      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=2.4.2 --build-arg PYTHON_VERSION=3.9.15
+      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=2.4.2 --build-arg PYTHON_VERSION=3.7.17
 
     - name: Build the docker-compose stack
       run: |
-        export PYTHON_VERSION=3.9
+        export PYTHON_VERSION=3.7
         docker compose -f ./examples/spark-with-S3/docker-compose.yml up -d
       
     - name: Check running containers
       run: docker ps -a
 
     - name: Run spark Job
-      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 2.4.2
+      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.7 2.4.2
 
   # hadoop_hdfs:
   #   runs-on: ubuntu-latest

diff --git a/examples/spark-with-S3/scripts/run_spark_example.sh b/examples/spark-with-S3/scripts/run_spark_example.sh
@@ -6,6 +6,9 @@ pip install -U pip setuptools wheel
 pip install pypandoc<1.8
 pip install s3fs pandas pyspark==$2
 pip install -e /cluster-pack
+if [[ $1 == python3.7 ]]; then
+  pip install pyarrow==6.0.1
+fi
 
 curr_dir=$(dirname "$0")
 python -V