Restructured k8s deployment of e2e tests to add an auth token.

LCOGT · Feb 28, 2025 · 9a631c2 · 9a631c2
1 parent 7944032
commit 9a631c2
Show file tree

Hide file tree

Showing 4 changed files with 426 additions and 263 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -34,27 +34,35 @@ jobs:
           kind load docker-image banzai:test-latest
 
       - name: Start banzai
-        env:
-          AUTH_TOKEN: ${{ secrets.ArchiveAuthToken }}
         run: |
-          cat banzai/tests/e2e-k8s.yaml
+          kubectl create secret generic auth-token-secret --from-literal=auth-token=${{ secrets.ArchiveAuthToken }}
 
           # Deploy banzai stack
-          kubectl apply -f banzai/tests/e2e-k8s.yaml
+          kubectl apply -k banzai/tests/k8s 
 
           # Wait for banzai to be ready
-          kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test
+          kubectl wait --for=condition=Ready --timeout=60m pods -l group=banzai-e2e-test
+          
+          LISTENER_POD=$(kubectl get pods -l app=banzai-listener-deployment -o jsonpath='{.items[0].metadata.name}')
+          echo "LISTENER_POD=${LISTENER_POD}" >> $GITHUB_ENV
 
+          WORKERS_POD=$(kubectl get pods -l app=banzai-celery-workers-deployment -o jsonpath='{.items[0].metadata.name}')
+          echo "WORKERS_POD=${WORKERS_POD}" >> $GITHUB_ENV
+          
+          LARGE_WORKERS_POD=$(kubectl get pods -l app=banzai-large-celery-workers-deployment -o jsonpath='{.items[0].metadata.name}')
+          echo "LARGE_WORKERS_POD=${LARGE_WORKERS_POD}" >> $GITHUB_ENV
+          
       - name: Test Super Bias Creation
         run: |
           set +e
           export START=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
-          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
+          kubectl exec ${LISTENER_POD} -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
           EXIT_CODE=$?
           
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LISTENER_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${WORKERS_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LARGE_WORKERS_POD} --since-time=$START --all-containers --prefix=true
 
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
@@ -64,11 +72,13 @@ jobs:
           set +e
           export START=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 
-          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark
+          kubectl exec ${LISTENER_POD} -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark
           EXIT_CODE=$?
           
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LISTENER_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${WORKERS_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LARGE_WORKERS_POD} --since-time=$START --all-containers --prefix=true
           
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
@@ -78,11 +88,13 @@ jobs:
           set +e
           export START=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 
-          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat
+          kubectl exec ${LISTENER_POD} -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat
           EXIT_CODE=$?
           
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LISTENER_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${WORKERS_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LARGE_WORKERS_POD} --since-time=$START --all-containers --prefix=true
 
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
@@ -92,15 +104,18 @@ jobs:
           set +e
           export START=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 
-          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files
+          kubectl exec ${LISTENER_POD} -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files
           EXIT_CODE=$?
 
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LISTENER_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${WORKERS_POD} --since-time=$START --all-containers --prefix=true
+          kubectl logs ${LARGE_WORKERS_POD} --since-time=$START --all-containers --prefix=true
 
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
 
       - name: Cleanup
         run: |
-          kubectl delete pod banzai-e2e-test
+          kubectl delete service --selector=group=banzai-e2e-test
+          kubectl delete deployment --selector=group=banzai-e2e-test
diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml