generated from ersilia-os/eos-template
-
Notifications
You must be signed in to change notification settings - Fork 1
224 lines (200 loc) · 8.18 KB
/
upload-model-to-dockerhub.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
name: Upload model to DockerHub
on:
workflow_dispatch:
workflow_run:
workflows: ["Upload model to S3"]
types:
- completed
jobs:
upload_model_to_dockerhub:
if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
steps:
# https://github.com/docker/setup-qemu-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
# log in to dockerhub
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Checkout persist credentials
uses: actions/checkout@master
with:
persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
lfs: 'true'
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: true
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: true
swap-storage: true
- name: Generate the Dockerfile
id: generate-dockerfile
env:
REPO_NAME: ${{ github.event.repository.name }}
run: |
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py
python -m pip install requests
python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME
# We cannot tag it as anything other than latest because
# ersilia cli only looks for the 'latest' tag
- name: Build only AMD64 Image for Testing
id: buildForTest
uses: docker/build-push-action@v5
with:
context: .
load: true
tags: ersiliaos/${{ github.event.repository.name }}:latest
- name: Add conda to system path
run: echo $CONDA/bin >> $GITHUB_PATH
- name: Source conda
run: source $CONDA/etc/profile.d/conda.sh
- name: Set Python to 3.10.10
run:
conda install -y python=3.10.10
- name: Install dependencies
run: |
source activate
conda init
conda install git-lfs -c conda-forge
git-lfs install
conda install gh -c conda-forge
- name: Install ersilia
run: |
source activate
python --version
echo "After conda init"
conda init
python -m pip install git+https://github.com/ersilia-os/ersilia.git
# TODO This is very hacky, maybe we want to use the ersilia test command in the future for this
- name: Test Built Image
id: testBuiltImage
run: |
ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub
ersilia -v serve ${{ github.event.repository.name }}
ersilia -v run -i "CCCC" -o "output.json"
expected=$(jq '.[0].output.outcome[]' output.json | wc -l)
result=$(jq '.[0].output.outcome[] | select(. != null)' output.json | wc -l)
echo "Expected correct outcomes $expected"
echo "Returned correct outcomes $result"
if [ $expected -ne $result ]
then
echo "Error in model output, aborting build and push"
exit 1
fi
env:
PULL_IMAGE: n
- name: Build and push
id: buildMultiple
continue-on-error: true
uses: docker/build-push-action@v5
timeout-minutes: 45
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ersiliaos/${{ github.event.repository.name }}:latest
- name: Set build failure output
id: buildCheck
run: |
if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then
echo "::set-output name=failed::true"
echo "AMD64" > arch.txt
else
echo "::set-output name=failed::false"
echo "AMD64,ARM64" > arch.txt
fi
- name: Build only for Linux/amd64
id: buildSingle
if: steps.buildCheck.outputs.failed == 'true'
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ersiliaos/${{ github.event.repository.name }}:latest
- name: Update Metadata JSON file with DockerHub info
id: updateMetadata
run: |
mv Dockerfile_legacy Dockerfile
python3 -c "
import json
with open('metadata.json', 'r') as f:
data = json.load(f)
print(data)
with open('arch.txt', 'r') as f:
arch = f.read().rstrip()
arch = arch.split(',')
data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier'])
data['Docker Architecture'] = arch
with open('metadata.json', 'w') as f:
json.dump(data, f, indent=4)
"
rm arch.txt
rm place_a_dockerfile_in_current_eos_repo.py
- name: Commit and push changes done to the Metadata JSON file
uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
with:
author_name: "ersilia-bot"
author_email: "[email protected]"
message: "updating metadata [skip ci]"
repository: "ersilia-os/${{ github.event.repository.name }}"
github_token: ${{ secrets.GITHUB_TOKEN }}
amend: true
force: true
- name: Update metadata to AirTable
id: update-metadata-to-airtable
env:
USER_NAME: ${{ github.repository_owner }}
BRANCH: "main"
REPO_NAME: ${{ github.event.repository.name }}
AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
run: |
source activate
echo "Updating metadata to AirTable looking at owner: $USER_NAME"
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py
python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY
rm update_metadata_to_airtable.py
- name: Update README file
id: update-readme-file
env:
MODEL_ID: ${{ github.event.repository.name }}
run: |
source activate
echo "Updating README file with AirTable metadata for model: $MODEL_ID"
wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py
python3 update_readme_from_airtable.py $MODEL_ID .
rm update_readme_from_airtable.py
less README.md
- name: Commit and push changes done to the README file
uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected]
with:
author_name: "ersilia-bot"
author_email: "[email protected]"
message: "updating readme [skip ci]"
repository: "ersilia-os/${{ github.event.repository.name }}"
github_token: ${{ secrets.GITHUB_TOKEN }}
amend: true
force: true
- name: Docker Hub Description
uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
repository: ersiliaos/${{ github.event.repository.name }}
short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}"