Merge branch 'dev' into nf-core-template-merge-3.1.0

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+* @FriederikeHanssen @maxulysse
+*.nf.test* @nf-core/nf-test
+.github/workflows/ @nf-core/a-team

.github/RELEASE_CHECKLIST.md

@@ -0,0 +1,47 @@
+# Release checklist
+
+> This checklist is for our own reference, to help us prepare a new release
+
+1. Check that everything is ready to go
+
+- Desired [PRs](https://github.com/nf-core/sarek/pulls) are merged
+- [GHA CI](https://github.com/nf-core/sarek/actions/workflows/ci.yml) are passing on `dev`
+- [GHA linting](https://github.com/nf-core/sarek/actions/workflows/linting.yml) are passing on `dev`
+
+2. Increase version number following [semantic versioning](http://semver.org/spec/v2.0.0.html)
+3. Choose an appropriate codename for the release (if major or minor)
+
+- i.e. Peaks in [Sarek National Park](https://en.wikipedia.org/wiki/Sarek_National_Park#Topography)
+
+4. Sync `dev` and checkout a new branch for the release
+5. Bump version:
+
+- `nf-core bump-version . <VERSION>`
+- edit `CHANGELOG`
+
+6. Make a PR to `master`
+7. Wait for reviews
+8. Merge said PR
+9. Make a [release](https://github.com/nf-core/sarek/releases) on GitHub
+10. Update [bio.tools](https://bio.tools/nf-core-sarek) with the new release details
+11. RT the nf-core automated tweet about the new released version
+12. Make a new branch from `dev`
+13. Checkout the `CHANGELOG.md` from `master`
+
+- `git checkout upstream/master -- CHANGELOG.md`
+
+14. Add a new `Unreleased` section in `CHANGELOG.md` for the `dev` version
+15. Checkout figures from `master`
+
+- `git checkout upstream/master -- docs/images/sarek_indices_subway.svg`
+- `git checkout upstream/master -- docs/images/sarek_indices_subway.png`
+- `git checkout upstream/master -- docs/images/sarek_subway.svg`
+- `git checkout upstream/master -- docs/images/sarek_subway.png`
+- `git checkout upstream/master -- docs/images/sarek_workflow.svg`
+- `git checkout upstream/master -- docs/images/sarek_workflow.png`
+
+16. Make a PR to `dev`
+17. Wait for review
+18. Merge said PR
+19. Commit and push. Continue making more awesome :metal:
+20. Have fika :cake:

.github/nf-test-tags.yml

@@ -0,0 +1,50 @@
+exclude:
+  - tags: "bcftools/annotate"
+  - tags: "bcftools/concat"
+  - tags: "bcftools/mpileup"
+  - tags: "bcftools/sort"
+  - tags: "bwa/index"
+  - tags: "bwa/mem"
+  - tags: "bwamem2/index"
+  - tags: "bwamem2/mem"
+  - tags: "cat/cat"
+  - tags: "cat/fastq"
+  - tags: "cnvkit/antitarget"
+  - tags: "cnvkit/batch"
+  - tags: "cnvkit/reference"
+  - tags: "deepvariant"
+  - tags: "dragmap/align"
+  - tags: "dragmap/hashtable"
+  - tags: "ensemblvep/download"
+  - tags: "ensemblvep/vep"
+  - tags: "fastp"
+  - tags: "fastqc"
+  - tags: "fgbio/fastqtobam"
+  - tags: "freebayes"
+  - tags: "gatk4/applybqsr"
+  - tags: "gatk4/baserecalibrator"
+  - tags: "gatk4/estimatelibrarycomplexity"
+  - tags: "gatk4/genomicsdbimport"
+  - tags: "gatk4/haplotypecaller"
+  - tags: "gatk4/markduplicates"
+  - tags: "gatk4/mergevcfs"
+  - tags: "gatk4/mutect2"
+  - tags: "gatk4spark/applybqsr"
+  - tags: "gatk4spark/markduplicates"
+  - tags: "gawk"
+  - tags: "lofreq/callparallel"
+  - tags: "mosdepth"
+  - tags: "multiqc"
+  - tags: "ngscheckmate/ncm"
+  - tags: "samblaster"
+  - tags: "samtools/convert"
+  - tags: "samtools/mpileup"
+  - tags: "samtools/stats"
+  - tags: "snpeff/snpeff"
+  - tags: "strelka/germline"
+  - tags: "strelka/somatic"
+  - tags: "subworkflows/utils_nfvalidation_plugin"
+  - tags: "tabix/bgziptabix"
+  - tags: "tabix/tabix"
+  - tags: "tiddit/sv"
+  - tags: "untar"

.github/workflows/ci.yml

@@ -10,49 +10,80 @@ on:
   workflow_dispatch:
 
 env:
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+  NFT_VER: "0.9.0"
+  NFT_WORKDIR: "~"
   NXF_ANSI_LOG: false
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+  TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
   test:
-    name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})"
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/sarek') }}"
     runs-on: ubuntu-latest
+    name: "Test ${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/5"
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "24.04.2"
           - "latest-everything"
-        profile:
-          - "conda"
-          - "docker"
-          - "singularity"
-        test_name:
-          - "test"
+        filter: ["workflow", "function", "pipeline"]
+        # filter: ["process", "workflow", "function", "pipeline"]
+        profile: ["conda", "docker", "singularity"]
+        shard: [1, 2, 3, 4, 5]
         isMaster:
           - ${{ github.base_ref == 'master' }}
-        # Exclude conda and singularity on dev
         exclude:
           - isMaster: false
             profile: "conda"
           - isMaster: false
             profile: "singularity"
+
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+
+      - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
 
       - name: Set up Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Set up Apptainer
+      - name: Set up Nextflow secrets
+        if: env.SENTIEON_LICENSE_BASE64 != null
+        run: |
+          nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+          nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
+          SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
+          SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
+          SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
+          SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
+          nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
+
+      - name: Set up nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
+
+      - name: Set up apptainer
         if: matrix.profile == 'singularity'
         uses: eWaterCycle/setup-apptainer@main
 
@@ -62,7 +93,17 @@ jobs:
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
-      - name: Set up Miniconda
+      - name: Cache pdiff
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
+        id: cache-pip-pdiff
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-pdiff
+
+      - name: Install pdiff
+        run: python -m pip install --upgrade pip pdiff cryptography
+
+      - name: Set up miniconda
         if: matrix.profile == 'conda'
         uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
         with:
@@ -77,9 +118,59 @@ jobs:
           echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
           echo $(realpath python) >> $GITHUB_PATH
 
-      - name: Clean up Disk space
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+      - name: Disk space cleanup
+        uses: jlumbroso/[email protected]
+
+      - name: Start summary
+        id: print-test
+        run: |
+          echo "## nf-test tests summary :rocket:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "This \`${{ matrix.filter }}\` ${{ matrix.shard }}/5 shard was run on \`${{ matrix.profile }}\` | \`NXF_VER=${{ matrix.NXF_VER }}\`, and contains the following test(s):" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          nf-test test \
+            --ci \
+            --dryRun \
+            --junitxml="TEST-${{ matrix.filter }}_${{ matrix.profile }}_${{ matrix.shard }}.xml" \
+            --shard ${{ matrix.shard }}/5 \
+            --changed-since HEAD^ \
+            --follow-dependencies \
+            --profile "+${{ matrix.profile }}" \
+            --filter ${{ matrix.filter }} \
+            | grep PASSED | cut -d "'" -f 2 | sed 's/^/- /' | sort -u >> $GITHUB_STEP_SUMMARY
+
+      - name: "Run tests | ${{ matrix.filter }}_${{ matrix.profile }} | ${{ matrix.shard }}/5"
+        run: |
+          nf-test test \
+              --ci \
+              --debug \
+              --verbose \
+              --junitxml="TEST-${{ matrix.filter }}_${{ matrix.profile }}_${{ matrix.shard }}.xml" \
+              --shard ${{ matrix.shard }}/5 \
+              --changed-since HEAD^ \
+              --follow-dependencies \
+              --profile "+${{ matrix.profile }}" \
+              --filter ${{ matrix.filter }}
+
+      - name: Print success in summary
+        if: success()
+        run: |
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "All test(s) successfull :tada:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Print failure in summary
+        if: failure()
+        run: |
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Some test(s) failed :cold_sweat:" >> $GITHUB_STEP_SUMMARY
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: "TEST-*.xml"
 
-      - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+      - name: Clean up
+        if: success() || failure()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+          sudo rm -rf /home/ubuntu/tests/