Replace pip cache of session_latest (#712)

* Replace pip cache of session_latest * Setup pip cache during CI --------- Signed-off-by: javiermtorres <[email protected]>
mozilla-ai · Feb 7, 2025 · b64c7c2 · b64c7c2
1 parent 23a85a9
commit b64c7c2
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 2 deletions.
diff --git a/.devcontainer/docker-compose.override.yaml b/.devcontainer/docker-compose.override.yaml
@@ -2,13 +2,16 @@ name: lumigator
 
 services:
 
+  ray:
+    volumes:
+      - /tmp/ray_pip_cache:/tmp/ray_pip_cache
   backend:
     build:
       context: .
       dockerfile: "Dockerfile"
       target: "dev_image"
     volumes:
-      -  database_volume:/mzai/backend/local.db
+      - database_volume:/mzai/backend/local.db
     ports:
       - "5678:5678"
     develop:

diff --git a/.github/workflows/lumigator_pipeline.yaml b/.github/workflows/lumigator_pipeline.yaml
@@ -79,6 +79,11 @@ jobs:
           mkdir -p ${HOME}/.cache/huggingface
           chmod -R 777 ${HOME}/.cache/huggingface
 
+      - name: Create pip cache
+        run: |
+          mkdir -p /tmp/ray_pip_cache
+          chmod -R 777 /tmp/ray_pip_cache
+
       - name: Spinning up containers for the tests
         run: make start-lumigator-build
 
@@ -127,6 +132,11 @@ jobs:
           mkdir -p ${HOME}/.cache/huggingface
           chmod -R 777 ${HOME}/.cache/huggingface
 
+      - name: Create pip cache
+        run: |
+          mkdir -p /tmp/ray_pip_cache
+          chmod -R 777 /tmp/ray_pip_cache
+
       - name: Setup containers
         run: make start-lumigator-build
 

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -69,10 +69,24 @@ services:
     # RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR is set
     # Dead head nodes appear because of the GCS data being
     # persisted in Redis
-    command: bash -c "RAY_REDIS_ADDRESS=redis:6379 RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR=1 RAY_REDIS_ADDRESS=redis:6379 ray start --head --dashboard-port=8265 --port=6379 --dashboard-host=0.0.0.0 --ray-client-server-port 10001 --block" # pragma: allowlist secret
+    entrypoint:
+      - /bin/bash
+      - -c
+      - |
+          set -eaux
+          mkdir -p /tmp/ray_pip_cache
+          # If the file was mounted in a volume instead of
+          # a shared dir, permissions need to be setup
+          # ... || true allows this to fail (-e is set)
+          sudo chmod -R 777 /tmp/ray_pip_cache/ || true
+          RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR=1 RAY_REDIS_ADDRESS=redis:6379 ray start --head --dashboard-port=8265 --port=6379 --dashboard-host=0.0.0.0 --ray-client-server-port 10001
+          mkdir -p /tmp/ray/session_latest/runtime_resources/pip
+          rmdir /tmp/ray/session_latest/runtime_resources/pip/ && ln -s /tmp/ray_pip_cache /tmp/ray/session_latest/runtime_resources/pip
+          sleep infinity
     shm_size: 2g
     volumes:
       - ${HOME}/.cache/huggingface:/home/ray/.cache/huggingface
+      - ray-pip-cache:/tmp/ray_pip_cache
     deploy:
       resources:
         limits:
@@ -202,3 +216,4 @@ volumes:
     minio-data:
     database_volume:
     redis-data:
+    ray-pip-cache: