diff --git a/.github/nf-test-tags.yml b/.github/nf-test-tags.yml new file mode 100644 index 0000000000..7e01f45416 --- /dev/null +++ b/.github/nf-test-tags.yml @@ -0,0 +1,54 @@ +exclude: + - tags: "bcftools/annotate" + - tags: "bcftools/concat" + - tags: "bcftools/mpileup" + - tags: "bcftools/sort" + - tags: "bwa/index" + - tags: "bwa/mem" + - tags: "bwamem2/index" + - tags: "bwamem2/mem" + - tags: "cat/cat" + - tags: "cat/fastq" + - tags: "cnvkit/antitarget" + - tags: "cnvkit/batch" + - tags: "cnvkit/reference" + - tags: "deepvariant" + - tags: "dragmap/align" + - tags: "dragmap/hashtable" + - tags: "ensemblvep/download" + - tags: "ensemblvep/vep" + - tags: "fastp" + - tags: "fastqc" + - tags: "fgbio/fastqtobam" + - tags: "freebayes" + - tags: "gatk4/applybqsr" + - tags: "gatk4/baserecalibrator" + - tags: "gatk4/estimatelibrarycomplexity" + - tags: "gatk4/genomicsdbimport" + - tags: "gatk4/haplotypecaller" + - tags: "gatk4/markduplicates" + - tags: "gatk4/mergevcfs" + - tags: "gatk4/mutect2" + - tags: "gatk4spark/applybqsr" + - tags: "gatk4spark/markduplicates" + - tags: "gawk" + - tags: "lofreq/callparallel" + - tags: "mosdepth" + - tags: "multiqc" + - tags: "ngscheckmate/ncm" + - tags: "samblaster" + - tags: "samtools/convert" + - tags: "samtools/mpileup" + - tags: "samtools/stats" + - tags: "snpeff/snpeff" + - tags: "strelka/germline" + - tags: "strelka/somatic" + - tags: "subworkflows/utils_nfvalidation_plugin" + - tags: "tabix/bgziptabix" + - tags: "tabix/tabix" + - tags: "tiddit/sv" + - tags: "untar" + - tags: "pipeline_sarek" +include: + - tags: "pipeline_sarek" + profile: "test,docker" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9999f5092..611f7595e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,10 @@ env: NFT_WORKDIR: "~" NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" + TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" @@ -26,8 +30,9 @@ jobs: fail-fast: false matrix: shard: [1, 2, 3, 4, 5] - NXF_VER: ["latest-stable"] # TODO Test minimum version on release + NXF_VER: ["23.04.0", "latest-everything"] # TODO Test minimum version on release profile: ["docker"] + filter: ["pipeline", "workflow"] # TODO Modules and Functions steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 @@ -52,14 +57,47 @@ jobs: with: version: ${{ env.NFT_VER }} + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + auto-update-conda: true + channels: conda-forge,bioconda,defaults + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@v1.3.1 + - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) run: | nf-test test \ --ci \ + --tap=test.tap \ + --verbose \ --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ --changed-since HEAD^ \ --profile "+${{ matrix.profile }}" \ - --filter pipeline + --filter ${{ matrix.filter }} - name: Publish Test Report uses: mikepenz/action-junit-report@v3 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index bff41935bf..df63fce6f8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -12,7 +12,6 @@ on: - dev env: - NFTEST_VER: "0.9.0" NXF_ANSI_LOG: false TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} @@ -189,170 +188,10 @@ jobs: !/home/ubuntu/pytest_workflow_*/*/work/singularity !${{ github.workspace }}/.singularity - nftest-changes: - name: Check for changes (nf-test) - runs-on: ubuntu-latest - outputs: - tags: ${{ steps.filter.outputs.changes }} - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Combine all tags.yml files - id: get_tags - run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml - - - uses: frouioui/paths-filter@main - id: filter - with: - filters: ".github/tags.yml" - token: "" - - nftest: - name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} - runs-on: ubuntu-latest - needs: nftest-changes - if: needs.nftest-changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"] - profile: ["docker"] - # profile: ["docker", "singularity", "conda"] - TEST_DATA_BASE: - - "test-datasets/data" - NXF_VER: - - "23.04.0" - - "latest-everything" - exclude: - - tags: "bcftools/annotate" - - tags: "bcftools/concat" - - tags: "bcftools/mpileup" - - tags: "bcftools/sort" - - tags: "bwa/index" - - tags: "bwa/mem" - - tags: "bwamem2/index" - - tags: "bwamem2/mem" - - tags: "cat/cat" - - tags: "cat/fastq" - - tags: "cnvkit/antitarget" - - tags: "cnvkit/batch" - - tags: "cnvkit/reference" - - tags: "deepvariant" - - tags: "dragmap/align" - - tags: "dragmap/hashtable" - - tags: "ensemblvep/download" - - tags: "ensemblvep/vep" - - tags: "fastp" - - tags: "fastqc" - - tags: "fgbio/fastqtobam" - - tags: "freebayes" - - tags: "gatk4/applybqsr" - - tags: "gatk4/baserecalibrator" - - tags: "gatk4/estimatelibrarycomplexity" - - tags: "gatk4/genomicsdbimport" - - tags: "gatk4/haplotypecaller" - - tags: "gatk4/markduplicates" - - tags: "gatk4/mergevcfs" - - tags: "gatk4/mutect2" - - tags: "gatk4spark/applybqsr" - - tags: "gatk4spark/markduplicates" - - tags: "gawk" - - tags: "lofreq/callparallel" - - tags: "mosdepth" - - tags: "multiqc" - - tags: "ngscheckmate/ncm" - - tags: "samblaster" - - tags: "samtools/convert" - - tags: "samtools/mpileup" - - tags: "samtools/stats" - - tags: "snpeff/snpeff" - - tags: "strelka/germline" - - tags: "strelka/somatic" - - tags: "subworkflows/utils_nfvalidation_plugin" - - tags: "tabix/bgziptabix" - - tags: "tabix/tabix" - - tags: "tiddit/sv" - - tags: "untar" - - tags: "pipeline_sarek" - include: - - tags: "pipeline_sarek" - profile: "test,docker" - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - uses: actions/setup-java@2dfa2011c5b2a0f1489bf9e433881c92c1631f88 # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Install Nextflow ${{ matrix.NXF_VER }} - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,bioconda - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_LICENSE_BASE64 != null - run: | - nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} - nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} - SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) - SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) - SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) - nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 - - - name: Disk space cleanup - uses: jlumbroso/free-disk-space@v1.3.1 - - # Test the module - - name: Run nf-test - run: | - nf-test test \ - --profile=${{ matrix.profile }} \ - --tag ${{ matrix.tags }} \ - --tap=test.tap \ - --verbose - confirm-pass: runs-on: ubuntu-latest needs: - pytest - - nftest if: always() steps: - name: All tests ok