feat: initial eval workflow

.github/workflows/data_drift_retrain.yml → .github/workflows/check_data_drift.yml

@@ -16,8 +16,8 @@ jobs:
     permissions:
       contents: read
       issues: write  # Grant write access to issues
-    outputs:  # Define outputs for downstream jobs
-      drift_detected: ${{ steps.drift_check.outputs.drift_detected }}
+    # outputs:  # Define outputs for downstream jobs
+    #   drift_detected: ${{ steps.drift_check.outputs.drift_detected }}
     steps:
       - name: ⏬ Checkout repository
         uses: actions/checkout@v4

.github/workflows/eval_retrain.yml

@@ -0,0 +1,56 @@
+# Checks model performance, and retrains
+# if model degradation is detected
+name: Performance Evaluation
+
+on:
+  workflow_run: # Trigger after another workflow completes
+    workflows: ["Production Data Batching"]
+    types:
+      - completed
+  push:
+    branches: ijdoc/issue9
+
+jobs:
+  eval_check:
+    runs-on: ubuntu-latest
+    outputs:  # Define outputs for downstream jobs
+      drift_detected: ${{ steps.eval_check.outputs.degraded }}
+    steps:
+      - name: ⏬ Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 🐍 Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: 📦 Install dependencies
+        run: |
+          pip install pipenv
+          cd eval
+          pipenv sync
+
+      - name: ⚙️ Run Evaluation
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        id: eval_check
+        run: |
+          cd drift
+          output=$(pipenv run python eval.py)
+          echo "$output" >> $GITHUB_STEP_SUMMARY
+
+  # Uncomment the following block to automatically
+  # retrain the model when drift is detected
+  # retrain:
+  #   needs: eval_check
+  #   if: ${{ needs.eval_check.outputs.drift_detected == 'True' }}
+  #   runs-on: self-hosted
+  #   steps:
+  #     - name: ⏬ Checkout repository
+  #       uses: actions/checkout@v4
+
+  #     - name: ⚙️ Train!
+  #       env:
+  #         WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+  #       run: |
+  #         python train.py

drift/check_drift.py

@@ -93,7 +93,7 @@
     print(f"- [W&B Run]({run.url})")
     print(f"- [Full data drift report]({report_url})")
 
-    # Optionally the drift detection result in a parseable format.
+    # Optionally print the drift detection result in a parseable format.
     # Helpful if you want to use this result in a CI/CD pipeline
     # to automatically update the data and/or retrain your model.
     # print(f"DRIFT_DETECTED={drift_detected}")

eval/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+wandb = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"