diff --git a/.github/scripts/verify_model_outcome.py b/.github/scripts/verify_model_outcome.py new file mode 100644 index 0000000..e8944d2 --- /dev/null +++ b/.github/scripts/verify_model_outcome.py @@ -0,0 +1,27 @@ +import csv + +def check_non_null_outcomes_in_output_csv(csv_file_path): + with open(csv_file_path, 'r') as csv_file: + csv_reader = csv.reader(csv_file) + header = next(csv_reader) + row = next(csv_reader) + for val in row[2:]: # Skip the first two columns (Inchikey and input) + if val not in ['', None]: + return False + return True + +if __name__ == '__main__': + # Read file path from command line + import sys + if len(sys.argv) < 2: + print('Usage: python verify_model_output.py ') + exit(1) + + output_csv_file = sys.argv[1] + + if check_non_null_outcomes_in_output_csv(output_csv_file): + # If there are null outcomes, exit with status code 1 + print('All outcomes are null') + else: + print('Some outcomes are not null') + diff --git a/.github/workflows/test-model-pr.yml b/.github/workflows/test-model-pr.yml index 5d309d2..93cb8f2 100644 --- a/.github/workflows/test-model-pr.yml +++ b/.github/workflows/test-model-pr.yml @@ -11,7 +11,23 @@ jobs: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # pin@v3.5.2 with: lfs: true - + +# This might stop working in the future, so we need to keep an eye on it + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: Add conda to system path run: echo $CONDA/bin >> $GITHUB_PATH diff --git a/.github/workflows/test-model.yml b/.github/workflows/test-model.yml index f27d993..517aa33 100644 --- a/.github/workflows/test-model.yml +++ b/.github/workflows/test-model.yml @@ -16,7 +16,23 @@ jobs: persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token fetch-depth: 0 # otherwise, you will failed to push refs to dest repo lfs: 'true' - + +# This might stop working in the future, so we need to keep an eye on it + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: Add conda to system path run: echo $CONDA/bin >> $GITHUB_PATH @@ -98,32 +114,4 @@ jobs: name: debug-logs retention-days: 1 path: /home/runner/eos/console.log - - - name: Shuffle assignees - id: shuffle - run: | - export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') - echo "$assignees" >> $GITHUB_STEP_SUMMARY - echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT - echo "shuffled_assignee=$assignees" >> $GITHUB_ENV - - name: Check for existing issue - id: check_issue - run: | - gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} - issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') - echo "::set-output name=issue_number::$issue_number" - - - name: Create a Test issue - uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 - id: create_issue - with: - github_token: ${{ secrets.github_token }} - title: ${{ vars.test_issue_title }} - assignees: | - ${{ steps.shuffle.outputs.shuffled_assignee }} - body: | - This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works! - labels: | - test - if: steps.check_issue.outputs.issue_number == '' diff --git a/.github/workflows/upload-model-to-dockerhub.yml b/.github/workflows/upload-model-to-dockerhub.yml index 69275df..c1798cf 100644 --- a/.github/workflows/upload-model-to-dockerhub.yml +++ b/.github/workflows/upload-model-to-dockerhub.yml @@ -36,6 +36,7 @@ jobs: fetch-depth: 0 # otherwise, you will failed to push refs to dest repo lfs: 'true' +# This might stop working in the future, so we need to keep an eye on it - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main with: @@ -50,16 +51,142 @@ jobs: haskell: true large-packages: true swap-storage: true - - - name: Generate the Dockerfile - id: generate-dockerfile + +# Install ersilia in the next few steps to test the built image with ersilia CLI + - name: Add conda to system path + run: echo $CONDA/bin >> $GITHUB_PATH + + - name: Source conda + run: source $CONDA/etc/profile.d/conda.sh + + - name: Set Python to 3.10.10 + run: + conda install -y python=3.10.10 + + - name: Install dependencies + run: | + source activate + conda init + conda install git-lfs -c conda-forge + git-lfs install + conda install gh -c conda-forge + + - name: Install ersilia + run: | + source activate + python --version + echo "After conda init" + conda init + python -m pip install git+https://github.com/ersilia-os/ersilia.git + +# Build a V2 image and test it with ersilia CLI + + - name: Generate the V2 Dockerfile + id: generateDockerfileV2 env: REPO_NAME: ${{ github.event.repository.name }} run: | - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py + wget -O place_a_dockerfile_in_current_eos_repo.py https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py python -m pip install requests python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME - + rm place_a_dockerfile_in_current_eos_repo.py + + # We cannot tag it as anything other than latest because + # ersilia cli only looks for the 'latest' tag + - name: Build only V2 AMD64 Image for Testing + id: buildForTestV2 + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ersiliaos/${{ github.event.repository.name }}:latest + + # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this + - name: Test built v2 image + id: testBuiltImageV2 + continue-on-error: true # Allow this to fail + run: | + ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub + ersilia -v serve ${{ github.event.repository.name }} + ersilia example -n 1 -f input.csv --predefined + ersilia -v run -i "input.csv" -o "output.csv" + ersilia close + output=$(python .github/scripts/verify_model_outcome.py output.csv) + if echo "$output" | grep -q "All outcomes are null"; then + echo "Error in model outcome, aborting build" + exit 1 + fi + rm output.csv + env: + PULL_IMAGE: n + +# If the test failed build a V1 image and test it with ersilia CLI + + - name: Generate the V1 Dockerfile + id: generateDockerfileV1 + if: steps.testBuiltImageV2.outcome == 'failure' + env: + REPO_NAME: ${{ github.event.repository.name }} + run: | + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py + python -m pip install requests + python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME v1 + rm place_a_dockerfile_in_current_eos_repo.py + + # We cannot tag it as anything other than latest because + # ersilia cli only looks for the 'latest' tag + - name: Build only V1 AMD64 Image for Testing + id: buildForTestV1 + if: steps.testBuiltImageV2.outcome == 'failure' + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ersiliaos/${{ github.event.repository.name }}:latest + + # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this + - name: Test built v1 image + id: testBuiltImageV1 + if: steps.testBuiltImageV2.outcome == 'failure' + run: | + ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub + ersilia -v serve ${{ github.event.repository.name }} + ersilia example -n 1 -f input.csv --predefined + ersilia -v run -i "input.csv" -o "output.csv" + ersilia close + output=$(python .github/scripts/verify_model_outcome.py output.csv) + if echo "$output" | grep -q "All outcomes are null"; then + echo "Error in model outcome, aborting build" + exit 1 + fi + rm output.csv + env: + PULL_IMAGE: n + +# Create an issue to track that this model did not work with the V2 image + + - name: Check for existing issue + id: check_existing_alert_issue + run: | + gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} + issue_number=$(gh issue list --limit 100 --search "${{ vars.DOCKER_BUILD_FAILURE_ISSUE_TITLE }}" --json number --jq '.[0].number') + echo "::set-output name=issue_number::$issue_number" + if: steps.testBuiltImageV1.conclusion != 'skipped' + + - name: Create an alert issue + uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 + id: create_alert_issue + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + title: ${{ vars.DOCKER_BUILD_FAILURE_ISSUE_TITLE }} + body: | + This model failed to build with v2 image. Please investigate and fix the issue. + labels: | + bug + if: ${{ (steps.check_existing_alert_issue.conclusion != 'skipped') && (steps.check_existing_alert_issue.outputs.issue_number == '') }} + +# Finally build and push if everything is successful + - name: Build and push id: buildMultiple continue-on-error: true @@ -93,7 +220,7 @@ jobs: tags: ersiliaos/${{ github.event.repository.name }}:latest - name: Update Metadata JSON file with DockerHub info - id: UpdateMetadata + id: updateMetadata run: | mv Dockerfile_legacy Dockerfile python3 -c " @@ -110,7 +237,6 @@ jobs: json.dump(data, f, indent=4) " rm arch.txt - rm place_a_dockerfile_in_current_eos_repo.py - name: Commit and push changes done to the Metadata JSON file uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 @@ -122,32 +248,6 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} amend: true force: true - - - name: Add conda to system path - run: echo $CONDA/bin >> $GITHUB_PATH - - - name: Source conda - run: source $CONDA/etc/profile.d/conda.sh - - - name: Set Python to 3.10.10 - run: - conda install -y python=3.10.10 - - - name: Install dependencies - run: | - source activate - conda init - conda install git-lfs -c conda-forge - git-lfs install - conda install gh -c conda-forge - - - name: Install ersilia - run: | - source activate - python --version - echo "After conda init" - conda init - python -m pip install git+https://github.com/ersilia-os/ersilia.git - name: Update metadata to AirTable id: update-metadata-to-airtable @@ -193,3 +293,34 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} repository: ersiliaos/${{ github.event.repository.name }} short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" + +# Create an issue within the repository to track that this model is ready for testing + + - name: Shuffle assignees + id: shuffle + run: | + export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') + echo "$assignees" >> $GITHUB_STEP_SUMMARY + echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT + echo "shuffled_assignee=$assignees" >> $GITHUB_ENV + + - name: Check for existing issue + id: check_existing_test_issue + run: | + gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} + issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') + echo "::set-output name=issue_number::$issue_number" + + - name: Create a Test issue + uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 + id: create_test_issue + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + title: ${{ vars.TEST_ISSUE_TITLE }} + assignees: | + ${{ steps.shuffle.outputs.shuffled_assignee }} + body: | + This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works! + labels: | + test + if: steps.check_issue.outputs.issue_number == ''