updating readme [skip ci]

ersilia-os · Apr 25, 2024 · 9df2733 · 9df2733
1 parent 513a36e
commit 9df2733
Show file tree

Hide file tree

Showing 4 changed files with 225 additions and 63 deletions.
diff --git a/.github/scripts/verify_model_outcome.py b/.github/scripts/verify_model_outcome.py
@@ -0,0 +1,27 @@
+import csv
+
+def check_non_null_outcomes_in_output_csv(csv_file_path):
+    with open(csv_file_path, 'r') as csv_file:
+        csv_reader = csv.reader(csv_file)
+        header = next(csv_reader)
+        row = next(csv_reader)
+        for val in row[2:]: # Skip the first two columns (Inchikey and input)
+            if val not in ['', None]:
+                return False
+    return True
+
+if __name__ == '__main__':
+    # Read file path from command line
+    import sys
+    if len(sys.argv) < 2:
+        print('Usage: python verify_model_output.py <output_csv_file>')
+        exit(1)
+
+    output_csv_file = sys.argv[1]
+
+    if check_non_null_outcomes_in_output_csv(output_csv_file):
+        # If there are null outcomes, exit with status code 1
+        print('All outcomes are null')
+    else:
+        print('Some outcomes are not null')
+
diff --git a/.github/workflows/test-model-pr.yml b/.github/workflows/test-model-pr.yml
@@ -11,7 +11,23 @@ jobs:
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # [email protected]
         with:
           lfs: true
-
+
+# This might stop working in the future, so we need to keep an eye on it
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
       - name: Add conda to system path
         run: echo $CONDA/bin >> $GITHUB_PATH
 

diff --git a/.github/workflows/test-model.yml b/.github/workflows/test-model.yml
@@ -16,7 +16,23 @@ jobs:
           persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
           fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
           lfs: 'true'
-
+
+# This might stop working in the future, so we need to keep an eye on it
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
       - name: Add conda to system path
         run: echo $CONDA/bin >> $GITHUB_PATH
 
@@ -98,32 +114,4 @@ jobs:
           name: debug-logs
           retention-days: 1
           path: /home/runner/eos/console.log
-
-      - name: Shuffle assignees
-        id: shuffle
-        run: |
-          export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}')
-          echo "$assignees" >> $GITHUB_STEP_SUMMARY
-          echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT
-          echo "shuffled_assignee=$assignees" >> $GITHUB_ENV
 
-      - name: Check for existing issue
-        id: check_issue
-        run: |
-          gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }}
-          issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number')
-          echo "::set-output name=issue_number::$issue_number"
-   
-      - name: Create a Test issue
-        uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49
-        id: create_issue 
-        with:
-          github_token: ${{ secrets.github_token }}
-          title: ${{ vars.test_issue_title }}
-          assignees: |
-            ${{ steps.shuffle.outputs.shuffled_assignee }}
-          body: |
-            This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works!
-          labels: |
-            test
-        if: steps.check_issue.outputs.issue_number == ''
diff --git a/.github/workflows/upload-model-to-dockerhub.yml b/.github/workflows/upload-model-to-dockerhub.yml
@@ -36,6 +36,7 @@ jobs:
           fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
           lfs: 'true'
 
+# This might stop working in the future, so we need to keep an eye on it
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
         with:
@@ -50,16 +51,142 @@ jobs:
           haskell: true
           large-packages: true
           swap-storage: true
-
-      - name: Generate the Dockerfile
-        id: generate-dockerfile
+
+# Install ersilia in the next few steps to test the built image with ersilia CLI
+      - name: Add conda to system path
+        run: echo $CONDA/bin >> $GITHUB_PATH
+
+      - name: Source conda
+        run: source $CONDA/etc/profile.d/conda.sh
+
+      - name: Set Python to 3.10.10
+        run:
+         conda install -y python=3.10.10 
+
+      - name: Install dependencies
+        run: |
+          source activate
+          conda init
+          conda install git-lfs -c conda-forge
+          git-lfs install
+          conda install gh -c conda-forge
+
+      - name: Install ersilia
+        run: |
+          source activate
+          python --version
+          echo "After conda init"
+          conda init
+          python -m pip install git+https://github.com/ersilia-os/ersilia.git
+
+# Build a V2 image and test it with ersilia CLI
+
+      - name: Generate the V2 Dockerfile
+        id: generateDockerfileV2
         env:
           REPO_NAME: ${{ github.event.repository.name }}
         run: |
-          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py
+          wget -O place_a_dockerfile_in_current_eos_repo.py https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py
           python -m pip install requests
           python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME
-        
+          rm place_a_dockerfile_in_current_eos_repo.py
+
+      # We cannot tag it as anything other than latest because 
+      # ersilia cli only looks for the 'latest' tag
+      - name: Build only V2 AMD64 Image for Testing
+        id: buildForTestV2
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: ersiliaos/${{ github.event.repository.name }}:latest
+
+      # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this
+      - name: Test built v2 image
+        id: testBuiltImageV2
+        continue-on-error: true  # Allow this to fail
+        run: |
+          ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub
+          ersilia -v serve ${{ github.event.repository.name }}
+          ersilia example -n 1 -f input.csv --predefined
+          ersilia -v run -i "input.csv" -o "output.csv"
+          ersilia close
+          output=$(python .github/scripts/verify_model_outcome.py output.csv)
+          if echo "$output" | grep -q "All outcomes are null"; then
+            echo "Error in model outcome, aborting build"
+            exit 1
+          fi
+          rm output.csv
+        env:
+          PULL_IMAGE: n
+
+# If the test failed build a V1 image and test it with ersilia CLI
+
+      - name: Generate the V1 Dockerfile
+        id: generateDockerfileV1
+        if: steps.testBuiltImageV2.outcome == 'failure'
+        env:
+          REPO_NAME: ${{ github.event.repository.name }}
+        run: |
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py
+          python -m pip install requests
+          python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME v1
+          rm place_a_dockerfile_in_current_eos_repo.py
+
+      # We cannot tag it as anything other than latest because 
+      # ersilia cli only looks for the 'latest' tag
+      - name: Build only V1 AMD64 Image for Testing
+        id: buildForTestV1
+        if: steps.testBuiltImageV2.outcome == 'failure'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: ersiliaos/${{ github.event.repository.name }}:latest
+
+      # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this
+      - name: Test built v1 image
+        id: testBuiltImageV1
+        if: steps.testBuiltImageV2.outcome == 'failure'
+        run: |
+          ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub
+          ersilia -v serve ${{ github.event.repository.name }}
+          ersilia example -n 1 -f input.csv --predefined
+          ersilia -v run -i "input.csv" -o "output.csv"
+          ersilia close
+          output=$(python .github/scripts/verify_model_outcome.py output.csv)
+          if echo "$output" | grep -q "All outcomes are null"; then
+            echo "Error in model outcome, aborting build"
+            exit 1
+          fi
+          rm output.csv
+        env:
+          PULL_IMAGE: n
+
+# Create an issue to track that this model did not work with the V2 image
+
+      - name: Check for existing issue
+        id: check_existing_alert_issue
+        run: |
+          gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }}
+          issue_number=$(gh issue list --limit 100 --search "${{ vars.DOCKER_BUILD_FAILURE_ISSUE_TITLE }}" --json number --jq '.[0].number')
+          echo "::set-output name=issue_number::$issue_number"
+        if: steps.testBuiltImageV1.conclusion != 'skipped'
+
+      - name: Create an alert issue
+        uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49
+        id: create_alert_issue 
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          title: ${{ vars.DOCKER_BUILD_FAILURE_ISSUE_TITLE }}
+          body: |
+            This model failed to build with v2 image. Please investigate and fix the issue.
+          labels: |
+            bug
+        if: ${{ (steps.check_existing_alert_issue.conclusion != 'skipped') && (steps.check_existing_alert_issue.outputs.issue_number == '') }}
+
+# Finally build and push if everything is successful
+
       - name: Build and push
         id: buildMultiple
         continue-on-error: true
@@ -93,7 +220,7 @@ jobs:
           tags: ersiliaos/${{ github.event.repository.name }}:latest
 
       - name: Update Metadata JSON file with DockerHub info
-        id: UpdateMetadata
+        id: updateMetadata
         run: |
           mv Dockerfile_legacy Dockerfile
           python3 -c "
@@ -110,7 +237,6 @@ jobs:
              json.dump(data, f, indent=4)
           "
           rm arch.txt
-          rm place_a_dockerfile_in_current_eos_repo.py
           
       - name: Commit and push changes done to the Metadata JSON file
         uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
@@ -122,32 +248,6 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           amend: true
           force: true
-
-      - name: Add conda to system path
-        run: echo $CONDA/bin >> $GITHUB_PATH
-
-      - name: Source conda
-        run: source $CONDA/etc/profile.d/conda.sh
-
-      - name: Set Python to 3.10.10
-        run:
-         conda install -y python=3.10.10 
-
-      - name: Install dependencies
-        run: |
-          source activate
-          conda init
-          conda install git-lfs -c conda-forge
-          git-lfs install
-          conda install gh -c conda-forge
-
-      - name: Install ersilia
-        run: |
-          source activate
-          python --version
-          echo "After conda init"
-          conda init
-          python -m pip install git+https://github.com/ersilia-os/ersilia.git
 
       - name: Update metadata to AirTable
         id: update-metadata-to-airtable
@@ -193,3 +293,34 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
           repository: ersiliaos/${{ github.event.repository.name }}
           short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}"
+
+# Create an issue within the repository to track that this model is ready for testing
+
+      - name: Shuffle assignees
+        id: shuffle
+        run: |
+          export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}')
+          echo "$assignees" >> $GITHUB_STEP_SUMMARY
+          echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT
+          echo "shuffled_assignee=$assignees" >> $GITHUB_ENV
+
+      - name: Check for existing issue
+        id: check_existing_test_issue
+        run: |
+          gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }}
+          issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number')
+          echo "::set-output name=issue_number::$issue_number"
+   
+      - name: Create a Test issue
+        uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49
+        id: create_test_issue 
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          title: ${{ vars.TEST_ISSUE_TITLE }}
+          assignees: |
+            ${{ steps.shuffle.outputs.shuffled_assignee }}
+          body: |
+            This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works!
+          labels: |
+            test
+        if: steps.check_issue.outputs.issue_number == ''