-
Notifications
You must be signed in to change notification settings - Fork 426
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: Move Sarek specific settings to ci
- Loading branch information
1 parent
6839ad2
commit 6fdf037
Showing
3 changed files
with
94 additions
and
163 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| exclude: | ||
| - tags: "bcftools/annotate" | ||
| - tags: "bcftools/concat" | ||
| - tags: "bcftools/mpileup" | ||
| - tags: "bcftools/sort" | ||
| - tags: "bwa/index" | ||
| - tags: "bwa/mem" | ||
| - tags: "bwamem2/index" | ||
| - tags: "bwamem2/mem" | ||
| - tags: "cat/cat" | ||
| - tags: "cat/fastq" | ||
| - tags: "cnvkit/antitarget" | ||
| - tags: "cnvkit/batch" | ||
| - tags: "cnvkit/reference" | ||
| - tags: "deepvariant" | ||
| - tags: "dragmap/align" | ||
| - tags: "dragmap/hashtable" | ||
| - tags: "ensemblvep/download" | ||
| - tags: "ensemblvep/vep" | ||
| - tags: "fastp" | ||
| - tags: "fastqc" | ||
| - tags: "fgbio/fastqtobam" | ||
| - tags: "freebayes" | ||
| - tags: "gatk4/applybqsr" | ||
| - tags: "gatk4/baserecalibrator" | ||
| - tags: "gatk4/estimatelibrarycomplexity" | ||
| - tags: "gatk4/genomicsdbimport" | ||
| - tags: "gatk4/haplotypecaller" | ||
| - tags: "gatk4/markduplicates" | ||
| - tags: "gatk4/mergevcfs" | ||
| - tags: "gatk4/mutect2" | ||
| - tags: "gatk4spark/applybqsr" | ||
| - tags: "gatk4spark/markduplicates" | ||
| - tags: "gawk" | ||
| - tags: "lofreq/callparallel" | ||
| - tags: "mosdepth" | ||
| - tags: "multiqc" | ||
| - tags: "ngscheckmate/ncm" | ||
| - tags: "samblaster" | ||
| - tags: "samtools/convert" | ||
| - tags: "samtools/mpileup" | ||
| - tags: "samtools/stats" | ||
| - tags: "snpeff/snpeff" | ||
| - tags: "strelka/germline" | ||
| - tags: "strelka/somatic" | ||
| - tags: "subworkflows/utils_nfvalidation_plugin" | ||
| - tags: "tabix/bgziptabix" | ||
| - tags: "tabix/tabix" | ||
| - tags: "tiddit/sv" | ||
| - tags: "untar" | ||
| - tags: "pipeline_sarek" | ||
| include: | ||
| - tags: "pipeline_sarek" | ||
| profile: "test,docker" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,6 +14,10 @@ env: | |
| NFT_WORKDIR: "~" | ||
| NFT_DIFF: "pdiff" | ||
| NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" | ||
| TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" | ||
| SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} | ||
| NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity | ||
| NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity | ||
|
|
||
| concurrency: | ||
| group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" | ||
|
|
@@ -26,8 +30,9 @@ jobs: | |
| fail-fast: false | ||
| matrix: | ||
| shard: [1, 2, 3, 4, 5] | ||
| NXF_VER: ["latest-stable"] # TODO Test minimum version on release | ||
| NXF_VER: ["23.04.0", "latest-everything"] # TODO Test minimum version on release | ||
| profile: ["docker"] | ||
| filter: ["pipeline", "workflow"] # TODO Modules and Functions | ||
| steps: | ||
| - name: Check out pipeline code | ||
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 | ||
|
|
@@ -52,14 +57,47 @@ jobs: | |
| with: | ||
| version: ${{ env.NFT_VER }} | ||
|
|
||
| - name: Set up miniconda | ||
| if: matrix.profile == 'conda' | ||
| uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 | ||
| with: | ||
| miniconda-version: "latest" | ||
| auto-update-conda: true | ||
| channels: conda-forge,bioconda,defaults | ||
|
|
||
| - name: Conda setup | ||
| if: matrix.profile == 'conda' | ||
| run: | | ||
| conda clean -a | ||
| conda install -n base conda-libmamba-solver | ||
| conda config --set solver libmamba | ||
| echo $(realpath $CONDA)/condabin >> $GITHUB_PATH | ||
| echo $(realpath python) >> $GITHUB_PATH | ||
| - name: Set up nextflow secrets | ||
| if: env.SENTIEON_LICENSE_BASE64 != null | ||
| run: | | ||
| nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} | ||
| nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} | ||
| SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) | ||
| SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) | ||
| SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") | ||
| SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) | ||
| nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 | ||
| - name: Disk space cleanup | ||
| uses: jlumbroso/[email protected] | ||
|
|
||
| - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) | ||
| run: | | ||
| nf-test test \ | ||
| --ci \ | ||
| --tap=test.tap \ | ||
| --verbose \ | ||
| --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ | ||
| --changed-since HEAD^ \ | ||
| --profile "+${{ matrix.profile }}" \ | ||
| --filter pipeline | ||
| --filter ${{ matrix.filter }} | ||
| - name: Publish Test Report | ||
| uses: mikepenz/action-junit-report@v3 | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,6 @@ on: | |
| - dev | ||
|
|
||
| env: | ||
| NFTEST_VER: "0.9.0" | ||
| NXF_ANSI_LOG: false | ||
| TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" | ||
| SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} | ||
|
|
@@ -189,170 +188,10 @@ jobs: | |
| !/home/ubuntu/pytest_workflow_*/*/work/singularity | ||
| !${{ github.workspace }}/.singularity | ||
| nftest-changes: | ||
| name: Check for changes (nf-test) | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| tags: ${{ steps.filter.outputs.changes }} | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 | ||
|
|
||
| - name: Combine all tags.yml files | ||
| id: get_tags | ||
| run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml | ||
|
|
||
| - uses: frouioui/paths-filter@main | ||
| id: filter | ||
| with: | ||
| filters: ".github/tags.yml" | ||
| token: "" | ||
|
|
||
| nftest: | ||
| name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} | ||
| runs-on: ubuntu-latest | ||
| needs: nftest-changes | ||
| if: needs.nftest-changes.outputs.tags != '[]' | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"] | ||
| profile: ["docker"] | ||
| # profile: ["docker", "singularity", "conda"] | ||
| TEST_DATA_BASE: | ||
| - "test-datasets/data" | ||
| NXF_VER: | ||
| - "23.04.0" | ||
| - "latest-everything" | ||
| exclude: | ||
| - tags: "bcftools/annotate" | ||
| - tags: "bcftools/concat" | ||
| - tags: "bcftools/mpileup" | ||
| - tags: "bcftools/sort" | ||
| - tags: "bwa/index" | ||
| - tags: "bwa/mem" | ||
| - tags: "bwamem2/index" | ||
| - tags: "bwamem2/mem" | ||
| - tags: "cat/cat" | ||
| - tags: "cat/fastq" | ||
| - tags: "cnvkit/antitarget" | ||
| - tags: "cnvkit/batch" | ||
| - tags: "cnvkit/reference" | ||
| - tags: "deepvariant" | ||
| - tags: "dragmap/align" | ||
| - tags: "dragmap/hashtable" | ||
| - tags: "ensemblvep/download" | ||
| - tags: "ensemblvep/vep" | ||
| - tags: "fastp" | ||
| - tags: "fastqc" | ||
| - tags: "fgbio/fastqtobam" | ||
| - tags: "freebayes" | ||
| - tags: "gatk4/applybqsr" | ||
| - tags: "gatk4/baserecalibrator" | ||
| - tags: "gatk4/estimatelibrarycomplexity" | ||
| - tags: "gatk4/genomicsdbimport" | ||
| - tags: "gatk4/haplotypecaller" | ||
| - tags: "gatk4/markduplicates" | ||
| - tags: "gatk4/mergevcfs" | ||
| - tags: "gatk4/mutect2" | ||
| - tags: "gatk4spark/applybqsr" | ||
| - tags: "gatk4spark/markduplicates" | ||
| - tags: "gawk" | ||
| - tags: "lofreq/callparallel" | ||
| - tags: "mosdepth" | ||
| - tags: "multiqc" | ||
| - tags: "ngscheckmate/ncm" | ||
| - tags: "samblaster" | ||
| - tags: "samtools/convert" | ||
| - tags: "samtools/mpileup" | ||
| - tags: "samtools/stats" | ||
| - tags: "snpeff/snpeff" | ||
| - tags: "strelka/germline" | ||
| - tags: "strelka/somatic" | ||
| - tags: "subworkflows/utils_nfvalidation_plugin" | ||
| - tags: "tabix/bgziptabix" | ||
| - tags: "tabix/tabix" | ||
| - tags: "tiddit/sv" | ||
| - tags: "untar" | ||
| - tags: "pipeline_sarek" | ||
| include: | ||
| - tags: "pipeline_sarek" | ||
| profile: "test,docker" | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 | ||
|
|
||
| - uses: actions/setup-java@2dfa2011c5b2a0f1489bf9e433881c92c1631f88 # v4 | ||
| with: | ||
| distribution: "temurin" | ||
| java-version: "17" | ||
|
|
||
| - name: Install Nextflow ${{ matrix.NXF_VER }} | ||
| uses: nf-core/setup-nextflow@v2 | ||
| with: | ||
| version: "${{ matrix.NXF_VER }}" | ||
|
|
||
| - name: Install nf-test | ||
| uses: nf-core/setup-nf-test@v1 | ||
| with: | ||
| version: ${{ env.NFTEST_VER }} | ||
|
|
||
| - name: Setup apptainer | ||
| if: matrix.profile == 'singularity' | ||
| uses: eWaterCycle/setup-apptainer@main | ||
|
|
||
| - name: Set up Singularity | ||
| if: matrix.profile == 'singularity' | ||
| run: | | ||
| mkdir -p $NXF_SINGULARITY_CACHEDIR | ||
| mkdir -p $NXF_SINGULARITY_LIBRARYDIR | ||
| - name: Set up miniconda | ||
| if: matrix.profile == 'conda' | ||
| uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 | ||
| with: | ||
| miniconda-version: "latest" | ||
| auto-update-conda: true | ||
| channels: conda-forge,bioconda | ||
|
|
||
| - name: Conda setup | ||
| if: matrix.profile == 'conda' | ||
| run: | | ||
| conda clean -a | ||
| conda install -n base conda-libmamba-solver | ||
| conda config --set solver libmamba | ||
| echo $(realpath $CONDA)/condabin >> $GITHUB_PATH | ||
| echo $(realpath python) >> $GITHUB_PATH | ||
| # Set up secrets | ||
| - name: Set up nextflow secrets | ||
| if: env.SENTIEON_LICENSE_BASE64 != null | ||
| run: | | ||
| nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} | ||
| nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} | ||
| SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) | ||
| SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) | ||
| SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") | ||
| SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) | ||
| nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 | ||
| - name: Disk space cleanup | ||
| uses: jlumbroso/[email protected] | ||
|
|
||
| # Test the module | ||
| - name: Run nf-test | ||
| run: | | ||
| nf-test test \ | ||
| --profile=${{ matrix.profile }} \ | ||
| --tag ${{ matrix.tags }} \ | ||
| --tap=test.tap \ | ||
| --verbose | ||
| confirm-pass: | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - pytest | ||
| - nftest | ||
| if: always() | ||
| steps: | ||
| - name: All tests ok | ||
|
|
||