Merge pull request #405 from LCOGT/bugfix/cals-by-block-migrate

Fixes to migration from cals by block.

.dockerignore

@@ -73,3 +73,7 @@ distribute-*.tar.gz
 *.xml
 
 *.iml
+
+.tmp
+
+.tox

.github/workflows/codestyle.yaml

@@ -0,0 +1,24 @@
+name: Code Style
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest  # Or macos-latest if needed
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12' # Specify your desired version
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E style
+      - name: Code Style
+        run: poetry run pycodestyle --count --max-line-length=120

.github/workflows/coverage.yaml

@@ -0,0 +1,24 @@
+name: Coverage
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest  # Or macos-latest if needed
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up 
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12' # Specify your desired version
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E test
+      - name: Run coverage
+        run: poetry run pytest --pyargs banzai.tests -m "not e2e" --cov banzai --cov-config="setup.cfg"

.github/workflows/docs.yaml

@@ -0,0 +1,25 @@
+name: Docs
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest  # Or macos-latest if needed
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12' # Specify your desired version
+      - name: Install dependencies
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E docs
+      - name: Build Docs
+        run: poetry run sphinx-build -W -b html docs docs/_build/html
+

.github/workflows/e2e.yaml

@@ -35,24 +35,25 @@ jobs:
 
       - name: Start banzai
         run: |
-          cat banzai/tests/e2e-k8s.yaml
+          kubectl create secret generic auth-token-secret --from-literal=auth-token=${{ secrets.ArchiveAuthToken }}
 
           # Deploy banzai stack
-          kubectl apply -f banzai/tests/e2e-k8s.yaml
+          kubectl apply -k banzai/tests/k8s 
 
           # Wait for banzai to be ready
           kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test
-
+          
       - name: Test Super Bias Creation
         run: |
           set +e
           export START=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
           kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
           EXIT_CODE=$?
           
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
 
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
@@ -66,8 +67,10 @@ jobs:
           EXIT_CODE=$?
           
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
-          
+          kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
+
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
 
@@ -80,7 +83,9 @@ jobs:
           EXIT_CODE=$?
           
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
 
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
@@ -94,11 +99,14 @@ jobs:
           EXIT_CODE=$?
 
           # Always print logs even if (especially if?) the reduction fails
-          kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
+          kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
 
           # Exit with the captured status so the job properly fails or succeeds
           exit $EXIT_CODE
 
       - name: Cleanup
         run: |
-          kubectl delete pod banzai-e2e-test
+          kubectl delete service --selector=group=banzai-e2e-test
+          kubectl delete deployment --selector=group=banzai-e2e-test

.github/workflows/unit-tests.yml

@@ -1,58 +1,41 @@
 name: Build/Test
 
 on:
-  # Run this workflow for pushes on all branches
   push:
+    branches: [main]
+  pull_request:
+    branches: [main]
 
 jobs:
-  tests:
-    name: ${{ matrix.name }} (${{ matrix.os }}, ${{ matrix.toxenv }})
-    runs-on: ${{ matrix.os }}
+  linuxtest:
+    runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
-        include:
-
-          - name: Documentation build
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: build_docs
-
-          - name: Python 3.9 with minimal dependencies
-            os: ubuntu-latest
-            python-version: '3.9'
-            toxenv: py39-test
-
-          - name: Python 3.10 with minimal dependencies
-            os: ubuntu-latest
-            python-version: '3.10'
-            toxenv: py310-test
-
-          - name: Python 3.11 with minimal dependencies
-            os: ubuntu-latest
-            python-version: '3.11'
-            toxenv: py311-test
-
-          - name: Code style checks
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: codestyle
-
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E test
+      - name: Run tests
+        run: poetry run pytest --pyargs banzai.tests -m "not e2e"
+  macos:  # New macOS job
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: ['3.12']  
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: | 
-       python -m pip install --upgrade pip
-       python -m pip install tox
-    - name: Install graphviz dependency
-      if: "endsWith(matrix.toxenv, 'build_docs')"
-      run: sudo apt-get -y install graphviz
-    - name: Run tests
-      run: |
-        tox -e ${{ matrix.toxenv }}
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E test
+      - name: Run tests
+        run: poetry run pytest --pyargs banzai.tests -m "not e2e"

.gitignore

@@ -19,7 +19,7 @@ a.out
 */version.py
 */cython_version.py
 htmlcov
-.coverage
+.coverage*
 MANIFEST
 .ipynb_checkpoints
 env
@@ -75,3 +75,4 @@ distribute-*.tar.gz
 *.iml
 
 .coverage.subprocess
+test.db

CHANGES.md

@@ -1,3 +1,9 @@
+1.20.2 (2025-02-10)
+-------------------
+- Bugfix to logging on reduction stopped
+- Updated the alembic scripts because the migration was taking too long
+- Dependencies and package build are now managed by poetry
+
 1.20.1 (2025-02-05)
 -------------------
 - Added tso proposals to be instantly public

Dockerfile

@@ -1,28 +1,30 @@
-FROM continuumio/miniconda3:23.5.2-0
-
-#  In principle I could remove the gcc to shrink the image, but pytorch is already so large it doesn't make much difference
-RUN apt-get -y update && apt-get -y install gcc && \
-        apt-get autoclean && \
-        rm -rf /var/lib/apt/lists/*
+FROM python:3.12-slim
 
+# Make a non-privledged user to run the pipeline
 RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \
         && /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \
         && chown -R archive:domainusers /home/archive
 
-USER archive
-
-ENV HOME=/home/archive
+RUN pip install poetry --no-cache
 
-WORKDIR /home/archive
+RUN apt-get -y update && apt-get -y install gcc procps && \
+        apt-get autoclean && \
+        rm -rf /var/lib/apt/lists/*
 
-COPY environment.yaml .
+RUN poetry config virtualenvs.create false
 
-RUN  . /opt/conda/etc/profile.d/conda.sh && conda config --set remote_read_timeout_secs 900 && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba
+COPY pyproject.toml poetry.lock /lco/banzai/
 
-COPY --chown=10087:10000 . /lco/banzai
+RUN poetry install --directory=/lco/banzai -E cpu --no-root --no-cache
 
-ENV PATH=/home/archive/envs/banzai/bin:$PATH
+COPY . /lco/banzai
 
-RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ 
+RUN poetry install --directory /lco/banzai -E cpu --no-cache
 
 RUN cp /lco/banzai/pytest.ini /home/archive/pytest.ini
+
+USER archive
+
+ENV HOME=/home/archive
+
+WORKDIR /home/archive