From 94d544337323195747de9e8ba9c847ecc2e2e9a2 Mon Sep 17 00:00:00 2001 From: Atharva Arya Date: Tue, 18 Feb 2025 16:38:49 +0530 Subject: [PATCH 1/4] Add lfs-cache workflow from TARDIS and update steps --- .github/actions/setup_env/action.yml | 4 +- .github/actions/setup_lfs/action.yml | 36 ++++--------- .github/workflows/lfs-cache.yml | 75 ++++++++++++++++++++++++++++ .github/workflows/tests.yml | 9 ++-- 4 files changed, 94 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/lfs-cache.yml diff --git a/.github/actions/setup_env/action.yml b/.github/actions/setup_env/action.yml index c6d579ce..fa9d3773 100644 --- a/.github/actions/setup_env/action.yml +++ b/.github/actions/setup_env/action.yml @@ -11,13 +11,13 @@ runs: using: "composite" steps: - name: Download Lock File - run: wget -q https://raw.githubusercontent.com/tardis-sn/stardis/main/conda-${{ inputs.os-label }}.lock + run: wget -q https://raw.githubusercontent.com/stardis-sn/stardis/main/conda-${{ inputs.os-label }}.lock shell: bash - name: Generate Cache Key run: | file_hash=$(cat conda-${{ inputs.os-label }}.lock | shasum -a 256 | cut -d' ' -f1) - echo "file_hash=$file_hash" >> "${GITHUB_OUTPUT}" + echo "file_hash=stardis-conda-env-${{ inputs.os-label }}-${file_hash}-v1" >> "${GITHUB_OUTPUT}" id: cache-environment-key shell: bash diff --git a/.github/actions/setup_lfs/action.yml b/.github/actions/setup_lfs/action.yml index 3855931b..34f12716 100644 --- a/.github/actions/setup_lfs/action.yml +++ b/.github/actions/setup_lfs/action.yml @@ -1,52 +1,38 @@ name: "Setup LFS" -description: "Pull LFS repositories and caches them" - +description: "Sets up Git LFS, retrieves LFS cache and fails if cache is not available" inputs: regression-data-repo: - description: "stardis regression data repository" + description: "Repository containing regression data (format: owner/repo)" required: false - default: "tardis-sn/stardis-regression-data" + default: "stardis-sn/stardis-regression-data" runs: using: "composite" steps: - - name: Clone tardis-sn/stardis-regression-data + - name: Clone stardis-sn/stardis-regression-data uses: actions/checkout@v4 with: repository: ${{ inputs.regression-data-repo }} path: stardis-regression-data + lfs: false - name: Create LFS file list - run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + run: | + git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-files-list working-directory: stardis-regression-data shell: bash - + - name: Restore LFS cache uses: actions/cache/restore@v4 id: lfs-cache-regression-data with: path: stardis-regression-data/.git/lfs - key: ${{ runner.os }}-lfs-${{ hashFiles('stardis-regression-data/.lfs-assets-id') }}-v1 - - - name: Git LFS Pull - run: git lfs pull - working-directory: stardis-regression-data - if: steps.lfs-cache-regression-data.outputs.cache-hit != 'true' - shell: bash + key: stardis-regression-full-data-${{ hashFiles('stardis-regression-data/.lfs-files-list') }}-${{ inputs.regression-data-repo }}-v1 + fail-on-cache-miss: true - name: Git LFS Checkout + if: steps.lfs-cache-regression-data.outputs.cache-hit == 'true' run: git lfs checkout working-directory: stardis-regression-data - if: steps.lfs-cache-regression-data.outputs.cache-hit == 'true' shell: bash - - - name: Save LFS cache if not found - # uses fake ternary - # for reference: https://github.com/orgs/community/discussions/26738#discussioncomment-3253176 - if: ${{ steps.lfs-cache-regression-data.outputs.cache-hit != 'true' && !contains(github.ref, 'merge') && always() || false }} - uses: actions/cache/save@v4 - id: lfs-cache-regression-data-save - with: - path: stardis-regression-data/.git/lfs - key: ${{ runner.os }}-lfs-${{ hashFiles('stardis-regression-data/.lfs-assets-id') }}-v1 diff --git a/.github/workflows/lfs-cache.yml b/.github/workflows/lfs-cache.yml new file mode 100644 index 00000000..4c9dac4a --- /dev/null +++ b/.github/workflows/lfs-cache.yml @@ -0,0 +1,75 @@ +name: Save LFS Cache + +on: + workflow_call: + inputs: + regression-data-repo: + description: "Repository containing regression data (format: owner/repo)" + required: false + default: "stardis-sn/stardis-regression-data" + type: string + allow_lfs_pull: + description: "If true, allows LFS pull operations" + required: false + default: false + type: boolean + +defaults: + run: + shell: bash -l {0} + +concurrency: + # Only one workflow can run at a time + # the workflow group is a unique identifier and contains the workflow name, pull request number, and regression data repo + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ inputs.regression-data-repo }} + cancel-in-progress: true + + +jobs: + lfs-cache: + if: github.repository_owner == 'tardis-sn' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ inputs.regression-data-repo }} + path: stardis-regression-data + + - name: Create LFS file list + run: | + git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-files-list + working-directory: stardis-regression-data + + - name: Test cache availability + uses: actions/cache/restore@v4 + id: test-lfs-cache-regression-data + with: + path: stardis-regression-data/.git/lfs + key: stardis-regression-full-data-${{ hashFiles('stardis-regression-data/.lfs-files-list') }}-${{ inputs.regression-data-repo }}-v1 + lookup-only: true + + - name: Fail if LFS pull is needed but not allowed + if: | + steps.test-lfs-cache-regression-data.outputs.cache-hit != 'true' && + inputs.allow_lfs_pull != true + run: | + echo "Error: LFS pull is required but not allowed (allow_lfs_pull is false)" + exit 1 + + - name: Git LFS Pull + if: | + steps.test-lfs-cache-regression-data.outputs.cache-hit != 'true' && + inputs.allow_lfs_pull == true + run: git lfs pull + working-directory: stardis-regression-data + + - name: Git LFS Checkout + run: git lfs checkout + working-directory: stardis-regression-data + + - name: Save LFS cache if not found + uses: actions/cache/save@v4 + if: ${{ steps.test-lfs-cache-regression-data.outputs.cache-hit != 'true' && !contains(github.ref, 'merge') }} + with: + path: stardis-regression-data/.git/lfs + key: stardis-regression-full-data-${{ hashFiles('stardis-regression-data/.lfs-files-list') }}-${{ inputs.regression-data-repo }}-v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ca114ef6..1634e47f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,6 +19,12 @@ defaults: shell: bash -l {0} jobs: + lfs-cache: + uses: ./.github/workflows/lfs-cache.yml + with: + regression-data-repo: tardis-sn/tardis-regression-data + allow_lfs_pull: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'git-lfs-pull') }} + build: if: github.repository_owner == 'tardis-sn' strategy: @@ -27,9 +33,6 @@ jobs: - os: ubuntu-latest label: linux-64 prefix: /usr/share/miniconda3/envs/stardis - # - os: ubuntu-latest - # label: linux-64-cuda - # prefix: /usr/share/miniconda3/envs/stardis - os: macos-latest label: osx-64 prefix: /Users/runner/miniconda3/envs/stardis From c4cc0f1d558e0fc8bf0966aefed0bd0d49e302e8 Mon Sep 17 00:00:00 2001 From: Atharva Arya Date: Tue, 18 Feb 2025 16:53:37 +0530 Subject: [PATCH 2/4] Update build docs workflow --- .github/workflows/build-docs.yml | 131 +++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 9f920dfb..acc263ee 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -1,46 +1,97 @@ name: build-docs on: push: - workflow_dispatch: + branches: + - main + pull_request_target: + branches: + - main + + types: + - opened + - reopened + - synchronize + - labeled # requires the `build-docs` label + - ready_for_review + + workflow_dispatch: # manual trigger env: - CACHE_NUMBER: 0 - DEPLOY_BRANCH: gh-pages # deployed docs branch + CACHE_NUMBER: 0 # increase to reset cache manually + DEPLOY_BRANCH: gh-pages # deployed docs branch + HDF5_USE_FILE_LOCKING: "FALSE" # disable file locking + + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true defaults: run: shell: bash -le {0} + jobs: + check-for-changes: + runs-on: ubuntu-latest + if: ${{ !github.event.pull_request.draft }} + outputs: + trigger-check-outcome: ${{ steps.trigger_check.outcome }} + docs-check-outcome: ${{ steps.docs_check.outcome }} + steps: + - uses: actions/checkout@v4 + if: github.event_name != 'pull_request_target' + + - name: Checkout pull/${{ github.event.number }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + if: github.event_name == 'pull_request_target' + + - name: Check for trigger by push event, manual dispatch, build-docs label on a PR + id: trigger_check + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'build-docs') + run: | + echo "Building docs as a test." + exit 0 + continue-on-error: true + + - name: Check for changes in documentation + run: | + if git diff origin/main..."$(git rev-parse --abbrev-ref HEAD)" --name-only | cat | grep '^docs/' | grep -q .; then + num_files=$(git diff --name-only origin/main...HEAD | grep '^docs/' | wc -l) + echo "Changes found in documentation files: $num_files" + exit 0 + else + echo "No changes found in documentation files - will stop running the pipeline." + exit 1 + fi + id: docs_check + if: steps.trigger_check.outcome != 'success' + continue-on-error: true + + build-sphinx-html: - if: ((github.repository == 'tardis-sn/stardis') && (${{ github.head_ref || github.ref_name}} == 'main')) || (github.repository_owner != 'tardis-sn') + if: (github.repository == 'tardis-sn/stardis' && (github.head_ref || github.ref_name) == 'main') || github.repository_owner != 'tardis-sn' # The above line makes this action run if it is either not on the upstream/main or the main branch of upstream/main. # If there is a better way to implement this, I'd like someone to please share. # The context to get the branch name is from https://stackoverflow.com/a/71158878 runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + if: github.event_name != 'pull_request_target' - - uses: actions/checkout@v2 - - - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@v2 + - name: Checkout pull/${{ github.event.number }} + uses: actions/checkout@v4 with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: stardis - use-mamba: true - - - name: Cache Environment Lockfile - id: cache-env-lockfile - uses: actions/cache@v3 + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + if: github.event_name == 'pull_request_target' + + - name: Setup environment + uses: ./.github/actions/setup_env with: - path: /usr/share/miniconda3/envs/stardis - key: conda-linux-64-${{ hashFiles('conda-linux-64.lock') }}-${{ env.CACHE_NUMBER }} - - - name: Update Conda Environment - id: update-env - run: | - mamba update -n stardis --file conda-linux-64.lock - if: steps.cache-env-lockfile.outputs.cache-hit != 'true' + os-label: linux-64 - name: Install TARDIS id: install-tardis @@ -58,7 +109,7 @@ jobs: id: make-sphinx-html run: | make -C docs html - + - name: Set destination directory run: | BRANCH=$(echo ${GITHUB_REF#refs/heads/}) @@ -72,7 +123,7 @@ jobs: elif [[ $EVENT == pull_request_target ]]; then echo "DEST_DIR=pull/$PR" >> $GITHUB_ENV - + else echo "Unexpected event trigger $EVENT" exit 1 @@ -85,8 +136,22 @@ jobs: EVENT: ${{ github.event_name }} PR: ${{ github.event.number }} + - name: Set clean branch option + run: | + if [[ $EVENT == workflow_dispatch ]]; then + echo "CLEAN_BRANCH=true" >> $GITHUB_ENV + + else + echo "CLEAN_BRANCH=false" >> $GITHUB_ENV + + fi + + cat $GITHUB_ENV + env: + EVENT: ${{ github.event_name }} + - name: Deploy ${{ env.DEST_DIR }} - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.BOT_TOKEN }} publish_branch: ${{ env.DEPLOY_BRANCH }} @@ -94,8 +159,8 @@ jobs: destination_dir: ${{ env.DEST_DIR }} keep_files: true force_orphan: ${{ env.CLEAN_BRANCH }} - user_name: 'TARDIS Bot' - user_email: 'tardis.sn.bot@gmail.com' + user_name: "TARDIS Bot" + user_email: "tardis.sn.bot@gmail.com" - name: Find comment uses: peter-evans/find-comment@v1 @@ -108,7 +173,7 @@ jobs: - name: Post comment (success) uses: peter-evans/create-or-update-comment@v1 with: - token: ${{ secrets.BOT_TOKEN }} + token: ${{ secrets.BOT_TOKEN }} issue-number: ${{ github.event.number }} comment-id: ${{ steps.fc.outputs.comment-id }} edit-mode: replace @@ -118,7 +183,7 @@ jobs: Hi, human. The **`${{ github.workflow }}`** workflow has **succeeded** :heavy_check_mark: - + [**Click here**](${{ env.URL }}) to see your results. env: URL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/pull/${{ github.event.number }}/index.html @@ -127,7 +192,7 @@ jobs: - name: Post comment (failure) uses: peter-evans/create-or-update-comment@v1 with: - token: ${{ secrets.BOT_TOKEN }} + token: ${{ secrets.BOT_TOKEN }} issue-number: ${{ github.event.number }} comment-id: ${{ steps.fc.outputs.comment-id }} edit-mode: replace @@ -137,7 +202,7 @@ jobs: Hi, human. The **`${{ github.workflow }}`** workflow has **failed** :x: - + [**Click here**](${{ env.URL }}) to see the build log. env: URL: https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}?check_suite_focus=true From 1248e987054943db201794638c302f082815bbaf Mon Sep 17 00:00:00 2001 From: Atharva Arya Date: Mon, 24 Feb 2025 21:31:09 +0530 Subject: [PATCH 3/4] label check From 928fe3941f91d9502d9aef11015c295aaa98a32a Mon Sep 17 00:00:00 2001 From: Atharva Arya Date: Mon, 24 Feb 2025 21:32:09 +0530 Subject: [PATCH 4/4] label check --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1634e47f..c009ad4d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,6 +26,7 @@ jobs: allow_lfs_pull: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'git-lfs-pull') }} build: + needs: lfs-cache if: github.repository_owner == 'tardis-sn' strategy: matrix: