From ef470ec33a38048628c6c6b5a2c00f00929da952 Mon Sep 17 00:00:00 2001 From: ersilia-bot Date: Wed, 18 Dec 2024 10:43:30 +0000 Subject: [PATCH] updating readme [skip ci] --- .github/scripts/resolve_dockerfile.py | 49 ++++ .github/scripts/verify_model_outcome.py | 2 +- .github/workflows/post-model-upload.yml | 194 +++++++++++++++ .github/workflows/test-model-pr.yml | 54 ++-- .github/workflows/test-model.yml | 124 ++++++---- .github/workflows/upload-bentoml.yml | 142 +++++++++++ .github/workflows/upload-ersilia-pack.yml | 122 ++++++++++ .../workflows/upload-model-to-dockerhub.yml | 230 ++---------------- .github/workflows/upload-model-to-s3.yml | 53 +++- 9 files changed, 679 insertions(+), 291 deletions(-) create mode 100644 .github/scripts/resolve_dockerfile.py create mode 100644 .github/workflows/post-model-upload.yml create mode 100644 .github/workflows/upload-bentoml.yml create mode 100644 .github/workflows/upload-ersilia-pack.yml diff --git a/.github/scripts/resolve_dockerfile.py b/.github/scripts/resolve_dockerfile.py new file mode 100644 index 0000000..4b06921 --- /dev/null +++ b/.github/scripts/resolve_dockerfile.py @@ -0,0 +1,49 @@ +import sys +import os +import requests + +from ersilia_pack.parsers import DockerfileInstallParser, YAMLInstallParser + +REPO_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) +PY_VERSION_MAP = { + '3.8': 'py38', + '3.9': 'py39', + '3.10': 'py310', + '3.11': 'py311', + '3.12': 'py312' +} +model_id = sys.argv[1] +def resolve_parser(): + if os.path.exists(os.path.abspath(os.path.join(REPO_PATH, "Dockerfile"))): + return DockerfileInstallParser(file_dir=REPO_PATH) + elif os.path.exists(os.path.join(REPO_PATH, "install.yml")): + return YAMLInstallParser(file_dir=REPO_PATH) + else: + raise ValueError("No install file found") + +def resolve_python_version(parser): + return parser._get_python_version() + +def read_dockerfile(parser): + commands = parser._get_commands() + has_conda = parser._has_conda(commands) + if has_conda: + file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.conda" + else: + file_url = "https://raw.githubusercontent.com/ersilia-os/ersilia/master/dockerfiles/dockerize-ersiliapack/model/Dockerfile.pip" + response = requests.get(file_url) + return response.text + +def write_version_and_model_id(file_content, python_version): + python_version = PY_VERSION_MAP[python_version] + file_content = file_content.replace("eos_identifier", model_id) + lines = file_content.split("\n") + lines[0] = lines[0].replace("VERSION", python_version) + with open(os.path.join(REPO_PATH, "../", "Dockerfile"), "w") as f: + f.write("\n".join(lines)) + +if __name__ == "__main__": + parser = resolve_parser() + python_version = resolve_python_version(parser) + dockerfile = read_dockerfile(parser) + write_version_and_model_id(dockerfile, python_version) \ No newline at end of file diff --git a/.github/scripts/verify_model_outcome.py b/.github/scripts/verify_model_outcome.py index e8944d2..5021e81 100644 --- a/.github/scripts/verify_model_outcome.py +++ b/.github/scripts/verify_model_outcome.py @@ -6,7 +6,7 @@ def check_non_null_outcomes_in_output_csv(csv_file_path): header = next(csv_reader) row = next(csv_reader) for val in row[2:]: # Skip the first two columns (Inchikey and input) - if val not in ['', None]: + if val not in ['', None]: # Returns if even one outcome is not null return False return True diff --git a/.github/workflows/post-model-upload.yml b/.github/workflows/post-model-upload.yml new file mode 100644 index 0000000..d33c69f --- /dev/null +++ b/.github/workflows/post-model-upload.yml @@ -0,0 +1,194 @@ +name: Post Model Upload actions + +on: + workflow_run: + workflows: ["Upload model to DockerHub"] + types: + - completed + +jobs: + post-model-upload: + if: ${{ github.repository != 'ersilia-os/eos-template' }} + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download arch.txt + uses: actions/download-artifact@v4 + with: + name: arch + path: . + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check metadata file + id: checkMetadata + continue-on-error: true + run: | + if [[ ! -f metadata.yml ]]; then + echo "metadata.yml file not found" + exit 1 + fi + + - name: Update Metadata YAML file with DockerHub info + id: updateMetadataYAML + if: steps.checkMetadata.outcome == 'success' + run: | + python3 -c " + import yaml + with open('metadata.yml', 'r') as f: + data = yaml.safe_load(f) + print(data) + with open('arch.txt', 'r') as f: + arch = f.read().rstrip() + arch = arch.split(',') + data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) + data['Docker Architecture'] = arch + with open('metadata.yml', 'w') as f: + yaml.dump(data, f, default_flow_style=False, sort_keys=False) + " + rm arch.txt + + - name: Update Metadata JSON file with DockerHub info + id: updateMetadataJSON + if: steps.checkMetadata.outcome == 'failure' + run: | + python3 -c " + import json + with open('metadata.json', 'r') as f: + data = json.load(f) + print(data) + with open('arch.txt', 'r') as f: + arch = f.read().rstrip() + arch = arch.split(',') + data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) + data['Docker Architecture'] = arch + with open('metadata.json', 'w') as f: + json.dump(data, f, indent=4) + " + rm arch.txt + + + - name: Commit and push changes done to the Metadata file + uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 + with: + author_name: "ersilia-bot" + author_email: "ersilia-bot@users.noreply.github.com" + message: "updating metadata [skip ci]" + repository: "ersilia-os/${{ github.event.repository.name }}" + github_token: ${{ secrets.GITHUB_TOKEN }} + amend: true + force: true + + # Setup conda + - name: Setup conda + id: setupConda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.10.10" + + # Install ersilia + - name: Install dependencies in Conda environment + id: installDependenciesInConda + run: | + conda install gh -c conda-forge + python -m pip install git+https://github.com/ersilia-os/ersilia.git + + - name: Update metadata to AirTable + id: update-metadata-to-airtable + env: + USER_NAME: ${{ github.repository_owner }} + BRANCH: "main" + REPO_NAME: ${{ github.event.repository.name }} + AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + run: | + source activate + pip install requests pyairtable + echo "Updating metadata to AirTable looking at owner: $USER_NAME" + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py + python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY + rm airtableops.py + + - name: sync metadata to S3 JSON + id: sync-metadata-to-s3 + env: + AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + source activate + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py + pip install boto3 requests pyairtable + python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY + rm convert_airtable_to_json.py + + - name: Update README file + id: update-readme-file + env: + MODEL_ID: ${{ github.event.repository.name }} + run: | + echo "Updating README file with AirTable metadata for model: $MODEL_ID" + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py + python3 airtableops.py readme-update --repo $MODEL_ID --path . + rm airtableops.py + less README.md + + - name: Commit and push changes done to the README file + uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 + with: + author_name: "ersilia-bot" + author_email: "ersilia-bot@users.noreply.github.com" + message: "updating readme [skip ci]" + repository: "ersilia-os/${{ github.event.repository.name }}" + github_token: ${{ secrets.GITHUB_TOKEN }} + amend: true + force: true + + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ersiliaos/${{ github.event.repository.name }} + short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" + + # Create an issue within the repository to track that this model is ready for testing + + - name: Shuffle assignees + id: shuffle + run: | + export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') + echo "$assignees" >> $GITHUB_STEP_SUMMARY + echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT + echo "shuffled_assignee=$assignees" >> $GITHUB_ENV + + - name: Check for existing issue + id: check_existing_test_issue + run: | + gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} + issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') + echo "::set-output name=issue_number::$issue_number" + + - name: Create a Test issue + uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 + id: create_test_issue + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + title: ${{ vars.TEST_ISSUE_TITLE }} + assignees: | + ${{ steps.shuffle.outputs.shuffled_assignee }} + body: | + This model is a new incorporation to the Ersilia Model Hub or it has been modified. If you are assigned to this issue, please try it out and ensure everything works! + To test a model, first clone it in your local system (ideally, from dockerhub) using the CLI commands: + ``` + ersilia -v fetch eosxxxx --from_dockerhub + ersilia serve eosxxxx + ersilia test + ``` + The test command will automatically check that the model can handle null outputs and whether it produces consistent results. Please copy here the result of the test command. If it passes, simply close the issue as completed. If it fails, please detail at which step and whether you have taken any steps to solve it. Please tag the original model contributor and one of Ersilia's maintainers for support. + labels: | + test + if: steps.check_existing_test_issue.outputs.issue_number == '' diff --git a/.github/workflows/test-model-pr.yml b/.github/workflows/test-model-pr.yml index 5d309d2..a76fb5c 100644 --- a/.github/workflows/test-model-pr.yml +++ b/.github/workflows/test-model-pr.yml @@ -11,7 +11,23 @@ jobs: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # pin@v3.5.2 with: lfs: true - + +# This might stop working in the future, so we need to keep an eye on it + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: Add conda to system path run: echo $CONDA/bin >> $GITHUB_PATH @@ -38,32 +54,26 @@ jobs: conda init python -m pip install git+https://github.com/ersilia-os/ersilia.git - - name: Check metadata before updating to AirTable - id: check-metadata - env: - USER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} - BRANCH: "main" - REPO_NAME: ${{ github.event.repository.name }} - run: | - source activate - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py - python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH - - name: Predict output env: MODEL_ID: ${{ github.event.repository.name }} - run: | - source activate - echo "Sample model id selected: $MODEL_ID" - ersilia -v fetch $MODEL_ID --repo_path . - ersilia -v serve $MODEL_ID - ersilia sample -n 5 -f input.csv - ersilia -v api -i input.csv - ersilia close + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + source activate + echo "Sample model id selected: $MODEL_ID" + ersilia -v fetch $MODEL_ID --from_dir . + ersilia -v serve $MODEL_ID + ersilia example -n 5 -f input.csv --predefined + ersilia -v api -i input.csv + ersilia close - name: Upload log output + if: always() uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb #pin v3.1.1 with: name: debug-logs - retention-days: 1 - path: /home/runner/eos/console.log + retention-days: 14 + path: /home/runner/eos/*.log diff --git a/.github/workflows/test-model.yml b/.github/workflows/test-model.yml index f27d993..db5ce34 100644 --- a/.github/workflows/test-model.yml +++ b/.github/workflows/test-model.yml @@ -1,4 +1,4 @@ -name: Model Test on push +name: Model test on push on: push: @@ -16,7 +16,23 @@ jobs: persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token fetch-depth: 0 # otherwise, you will failed to push refs to dest repo lfs: 'true' - + +# This might stop working in the future, so we need to keep an eye on it + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: Add conda to system path run: echo $CONDA/bin >> $GITHUB_PATH @@ -50,12 +66,30 @@ jobs: BRANCH: "main" REPO_NAME: ${{ github.event.repository.name }} AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + source activate + pip install requests pyairtable + echo "Updating metadata to AirTable looking at owner: $USER_NAME" + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py + python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY + rm airtableops.py + + - name: sync metadata to S3 JSON + id: sync-metadata-to-s3 + env: + AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | source activate - echo "Updating metadata to AirTable looking at owner: $USER_NAME" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py - python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY - rm update_metadata_to_airtable.py + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py + pip install boto3 requests pyairtable + python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY + rm convert_airtable_to_json.py - name: Update README file id: update-readme-file @@ -64,9 +98,9 @@ jobs: run: | source activate echo "Updating README file with AirTable metadata for model: $MODEL_ID" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py - python3 update_readme_from_airtable.py $MODEL_ID . - rm update_readme_from_airtable.py + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py + python3 airtableops.py readme-update --repo $MODEL_ID --path . + rm airtableops.py less README.md - name: Commit and push changes done to the README file @@ -80,50 +114,42 @@ jobs: amend: true force: true - - name: Predict output + - name: Fetch model env: MODEL_ID: ${{ github.event.repository.name }} run: | - source activate - echo "Sample model id selected: $MODEL_ID" - ersilia -v fetch $MODEL_ID --repo_path . - ersilia -v serve $MODEL_ID - ersilia sample -n 5 -f input.csv - ersilia -v api -i input.csv - ersilia close - + source activate + ersilia -v fetch $MODEL_ID --from_dir . + FOLDER="$HOME/eos/repository/$MODEL_ID" + if [ ! -d "$FOLDER" ]; then + echo "Error: Folder '$FOLDER' does not exist." >&2 + exit 1 + fi + + - name: Generate input and run model + env: + MODEL_ID: ${{ github.event.repository.name }} + run: | + source activate + ersilia -v serve $MODEL_ID + ersilia example -n 5 -f input.csv --predefined + ersilia -v run -i "input.csv" -o "output.csv" + ersilia close + cat output.csv + + - name: Test output + run: | + output=$(python .github/scripts/verify_model_outcome.py output.csv) + if echo "$output" | grep -q "All outcomes are null"; then + echo "Error in model outcome, aborting test" + exit 1 + fi + rm output.csv + - name: Upload log output + if: always() uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb #pin v3.1.1 with: name: debug-logs - retention-days: 1 - path: /home/runner/eos/console.log - - - name: Shuffle assignees - id: shuffle - run: | - export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') - echo "$assignees" >> $GITHUB_STEP_SUMMARY - echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT - echo "shuffled_assignee=$assignees" >> $GITHUB_ENV - - - name: Check for existing issue - id: check_issue - run: | - gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} - issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') - echo "::set-output name=issue_number::$issue_number" - - - name: Create a Test issue - uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 - id: create_issue - with: - github_token: ${{ secrets.github_token }} - title: ${{ vars.test_issue_title }} - assignees: | - ${{ steps.shuffle.outputs.shuffled_assignee }} - body: | - This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works! - labels: | - test - if: steps.check_issue.outputs.issue_number == '' + retention-days: 14 + path: /home/runner/eos/*.log diff --git a/.github/workflows/upload-bentoml.yml b/.github/workflows/upload-bentoml.yml new file mode 100644 index 0000000..456876a --- /dev/null +++ b/.github/workflows/upload-bentoml.yml @@ -0,0 +1,142 @@ +name: Upload BentoML Dockerized Model with or without Multi-stage Conda Pack + +on: + workflow_call: + inputs: + version: + required: true + type: string + +jobs: + build-bentoml-image: + if: ${{ github.repository != 'ersilia-os/eos-template' }} + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check if we can use this workflow + run: | + if [[ -f install.yml ]]; then + echo "This workflow is not supported for this repository" + exit 1 + fi + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + # log in to dockerhub + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + # This might stop working in the future, so we need to keep an eye on it + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Setup conda + id: setupConda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.10.10" + + - name: Install dependencies in Conda environment + id: installDependenciesInConda + run: | + conda install git-lfs -c conda-forge + git-lfs install + conda install gh -c conda-forge + python -m pip install git+https://github.com/ersilia-os/ersilia.git + + - name: Generate the Dockerfile + id: generateDockerfile + env: + REPO_NAME: ${{ github.event.repository.name }} + VERSION: ${{ inputs.version}} + run: | + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py + python -m pip install requests + python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME $VERSION + rm place_a_dockerfile_in_current_eos_repo.py + + # We cannot tag it as anything other than latest because + # ersilia cli only looks for the 'latest' tag + - name: Build only AMD64 Image for Testing + id: buildForTest + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ersiliaos/${{ github.event.repository.name }}:latest + + # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this + - name: Test built image + id: testBuiltImage + env: + PULL_IMAGE: n + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + run: | + ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub + ersilia -v serve ${{ github.event.repository.name }} --track #Added --track here + ersilia example -n 1 -f input.csv --predefined + ersilia -v run -i "input.csv" -o "output.csv" + ersilia close + output=$(python .github/scripts/verify_model_outcome.py output.csv) + if echo "$output" | grep -q "All outcomes are null"; then + echo "Error in model outcome, aborting build" + exit 1 + fi + rm output.csv + + - name: Build and push + id: buildMultiple + continue-on-error: true + uses: docker/build-push-action@v6.7.0 + timeout-minutes: 45 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ersiliaos/${{ github.event.repository.name }}:latest + + - name: Set build failure output + id: buildCheck + run: | + if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then + echo "::set-output name=failed::true" + echo "AMD64" > arch.txt + else + echo "::set-output name=failed::false" + echo "AMD64,ARM64" > arch.txt + fi + + - name: Upload arch.txt + uses: actions/upload-artifact@v4 + with: + name: arch + path: arch.txt diff --git a/.github/workflows/upload-ersilia-pack.yml b/.github/workflows/upload-ersilia-pack.yml new file mode 100644 index 0000000..fe4bd5c --- /dev/null +++ b/.github/workflows/upload-ersilia-pack.yml @@ -0,0 +1,122 @@ +name: Upload Ersilia Pack Dockerized Model + +on: + workflow_call: + +jobs: + build-ersilia-pack-image: + if: ${{ github.repository != 'ersilia-os/eos-template' }} + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + lfs: true + + - run: git lfs pull + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + # log in to dockerhub + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + # This might stop working in the future, so we need to keep an eye on it + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + # Install ersilia-pack, requests, and ersilia to test the built image with ersilia CLI + - name: Setup Python for Ersilia Pack + id: setupPythonForErsiliaPack + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 + with: + python-version: '3.10' + + - name: Install ersilia-pack and generate the right Dockerfile + env: + REPO_NAME: ${{ github.event.repository.name }} + run: | + python -m pip install git+https://github.com/ersilia-os/ersilia-pack.git + python -m pip install requests + python -m pip install git+https://github.com/ersilia-os/ersilia.git + python .github/scripts/resolve_dockerfile.py $REPO_NAME + + - name: Build only AMD64 Image for Testing + id: buildForTestErsiliaPack + uses: docker/build-push-action@v6.7.0 + with: + context: ../ # We need to go back to the root directory to find the Dockerfile and copy the model repository + load: true + tags: ersiliaos/${{ github.event.repository.name }}:latest + + - name: Test built image + id: testBuiltImageErsiliaPack + env: + PULL_IMAGE: n + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + if: steps.buildForTestErsiliaPack.outcome == 'success' + run: | + ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub + ersilia -v serve ${{ github.event.repository.name }} --track #Added --track here + ersilia example -n 1 -f input.csv --predefined + ersilia -v run -i "input.csv" -o "output.csv" + ersilia close + output=$(python .github/scripts/verify_model_outcome.py output.csv) + if echo "$output" | grep -q "All outcomes are null"; then + echo "Error in model outcome, aborting build" + exit 1 + fi + rm output.csv + + - name: Build and push + id: buildMultiple + continue-on-error: true + uses: docker/build-push-action@v6.7.0 + timeout-minutes: 45 + with: + context: ../ + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ersiliaos/${{ github.event.repository.name }}:latest + + - name: Set build failure output + id: buildCheck + run: | + if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then + echo "::set-output name=failed::true" + echo "AMD64" > arch.txt + else + echo "::set-output name=failed::false" + echo "AMD64,ARM64" > arch.txt + fi + + - name: Upload arch.txt + uses: actions/upload-artifact@v4 + with: + name: arch + path: arch.txt diff --git a/.github/workflows/upload-model-to-dockerhub.yml b/.github/workflows/upload-model-to-dockerhub.yml index 1bdcd1b..e272e93 100644 --- a/.github/workflows/upload-model-to-dockerhub.yml +++ b/.github/workflows/upload-model-to-dockerhub.yml @@ -8,214 +8,22 @@ on: - completed jobs: - upload_model_to_dockerhub: - if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - # log in to dockerhub - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout persist credentials - uses: actions/checkout@master - with: - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - lfs: 'true' - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: true - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true - - - name: Generate the Dockerfile - id: generate-dockerfile - env: - REPO_NAME: ${{ github.event.repository.name }} - run: | - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py - python -m pip install requests - python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME - - # We cannot tag it as anything other than latest because - # ersilia cli only looks for the 'latest' tag - - name: Build only AMD64 Image for Testing - id: buildForTest - uses: docker/build-push-action@v5 - with: - context: . - load: true - tags: ersiliaos/${{ github.event.repository.name }}:latest - - - name: Add conda to system path - run: echo $CONDA/bin >> $GITHUB_PATH - - - name: Source conda - run: source $CONDA/etc/profile.d/conda.sh - - - name: Set Python to 3.10.10 - run: - conda install -y python=3.10.10 - - - name: Install dependencies - run: | - source activate - conda init - conda install git-lfs -c conda-forge - git-lfs install - conda install gh -c conda-forge - - - name: Install ersilia - run: | - source activate - python --version - echo "After conda init" - conda init - python -m pip install git+https://github.com/ersilia-os/ersilia.git - - # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this - - name: Test Built Image - id: testBuiltImage - run: | - ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub - ersilia -v serve ${{ github.event.repository.name }} - ersilia -v run -i "CCCC" -o "output.csv" - ersilia close - output=$(python .github/scripts/verify_model_outcome.py output.csv) - if echo "$output" | grep -q "All outcomes are null"; then - echo "Error in model outcome, aborting build" - exit 1 - fi - env: - PULL_IMAGE: n - - - name: Build and push - id: buildMultiple - continue-on-error: true - uses: docker/build-push-action@v5 - timeout-minutes: 45 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ersiliaos/${{ github.event.repository.name }}:latest - - - name: Set build failure output - id: buildCheck - run: | - if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then - echo "::set-output name=failed::true" - echo "AMD64" > arch.txt - else - echo "::set-output name=failed::false" - echo "AMD64,ARM64" > arch.txt - fi - - - name: Build only for Linux/amd64 - id: buildSingle - if: steps.buildCheck.outputs.failed == 'true' - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ersiliaos/${{ github.event.repository.name }}:latest - - - name: Update Metadata JSON file with DockerHub info - id: updateMetadata - run: | - mv Dockerfile_legacy Dockerfile - python3 -c " - import json - with open('metadata.json', 'r') as f: - data = json.load(f) - print(data) - with open('arch.txt', 'r') as f: - arch = f.read().rstrip() - arch = arch.split(',') - data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) - data['Docker Architecture'] = arch - with open('metadata.json', 'w') as f: - json.dump(data, f, indent=4) - " - rm arch.txt - rm place_a_dockerfile_in_current_eos_repo.py - - - name: Commit and push changes done to the Metadata JSON file - uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 - with: - author_name: "ersilia-bot" - author_email: "ersilia-bot@users.noreply.github.com" - message: "updating metadata [skip ci]" - repository: "ersilia-os/${{ github.event.repository.name }}" - github_token: ${{ secrets.GITHUB_TOKEN }} - amend: true - force: true - - - name: Update metadata to AirTable - id: update-metadata-to-airtable - env: - USER_NAME: ${{ github.repository_owner }} - BRANCH: "main" - REPO_NAME: ${{ github.event.repository.name }} - AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} - run: | - source activate - echo "Updating metadata to AirTable looking at owner: $USER_NAME" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py - python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY - rm update_metadata_to_airtable.py - - - name: Update README file - id: update-readme-file - env: - MODEL_ID: ${{ github.event.repository.name }} - run: | - source activate - echo "Updating README file with AirTable metadata for model: $MODEL_ID" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py - python3 update_readme_from_airtable.py $MODEL_ID . - rm update_readme_from_airtable.py - less README.md - - - name: Commit and push changes done to the README file - uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 - with: - author_name: "ersilia-bot" - author_email: "ersilia-bot@users.noreply.github.com" - message: "updating readme [skip ci]" - repository: "ersilia-os/${{ github.event.repository.name }}" - github_token: ${{ secrets.GITHUB_TOKEN }} - amend: true - force: true - - - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ersiliaos/${{ github.event.repository.name }} - short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" + upload-ersilia-pack: + uses: ./.github/workflows/upload-ersilia-pack.yml + secrets: inherit + + upload-multi-stage-condapack: + needs: [upload-ersilia-pack] + if: ${{ failure() }} + uses: ./.github/workflows/upload-bentoml.yml + secrets: inherit + with: + version: multistage-condapack + + upload-legacy-bentoml: + needs: [upload-multi-stage-condapack] + if: ${{ failure() }} + uses: ./.github/workflows/upload-bentoml.yml + secrets: inherit + with: + version: legacy-bentoml diff --git a/.github/workflows/upload-model-to-s3.yml b/.github/workflows/upload-model-to-s3.yml index 163a87c..f3a48d5 100644 --- a/.github/workflows/upload-model-to-s3.yml +++ b/.github/workflows/upload-model-to-s3.yml @@ -45,8 +45,18 @@ jobs: conda init python -m pip install git+https://github.com/ersilia-os/ersilia.git + - name: Check metadata file + id: checkMetadata + continue-on-error: true + run: | + if [[ ! -f metadata.yml ]]; then + echo "metadata.yml file not found" + exit 1 + fi + - name: Update Metadata JSON file with DockerHub info - id: UpdateMetadata + if: steps.checkMetadata.outcome == 'failure' + id: UpdateMetadataJSON run: | python3 -c " import json @@ -56,6 +66,19 @@ jobs: with open('metadata.json', 'w') as f: json.dump(data, f, indent=4) " + + - name: Update Metadata YAML file with DockerHub info + if: steps.checkMetadata.outcome == 'success' + id: UpdateMetadataYAML + run: | + python3 -c " + import yaml + with open('metadata.yml', 'r') as f: + data = yaml.safe_load(f) + data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier']) + with open('metadata.yml', 'w') as f: + yaml.dump(data, f, default_flow_style=False, sort_keys=False) + " - name: Commit and push changes done to the Metadata JSON file uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 @@ -77,11 +100,25 @@ jobs: AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} run: | source activate + pip install requests pyairtable echo "Updating metadata to AirTable looking at owner: $USER_NAME" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py - python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY - rm update_metadata_to_airtable.py - + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py + python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY + rm airtableops.py + + - name: sync metadata to S3 JSON + id: sync-metadata-to-s3 + env: + AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + source activate + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py + pip install boto3 requests pyairtable + python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY + rm convert_airtable_to_json.py + - name: Update README file id: update-readme-file env: @@ -89,9 +126,9 @@ jobs: run: | source activate echo "Updating README file with AirTable metadata for model: $MODEL_ID" - wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py - python3 update_readme_from_airtable.py $MODEL_ID . - rm update_readme_from_airtable.py + wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py + python3 airtableops.py readme-update --repo $MODEL_ID --path . + rm airtableops.py less README.md - name: Commit and push changes done to the README file