Merge pull request #15 from UMCUGenetics/develop

v1.1.0

README.md

@@ -14,7 +14,12 @@ curl -s https://get.nextflow.io | bash
 
 #### Running WES workflow
 ```bash
-nextflow run WES.nf -c WES.config --fastq_path <fastq_dir_path> --outdir <output_dir_path> [-profile slurm|mac]
+nextflow run WES.nf -c WES.config --fastq_path <fastq_dir_path> --outdir <output_dir_path> --email <email> [-profile slurm|mac]
+```
+
+#### Running WES Fingerprint workflow
+```bash
+nextflow run WES_Fingerprint.nf -c WES.config --bam_path <bam_dir_path> --outdir <output_dir_path> --email <email> [-profile slurm|mac]
 ```
 
 #### Create Kinship container

WES.config

@@ -48,17 +48,10 @@ process {
         }
     }
 
-    withLabel: BWA_0_7_17_MEM {
+    withLabel: BWA_0_7_17_Mem {
         cpus = 10
         memory = '10G'
-        time = '180m'
-    }
-
-    withLabel: Sambamba_0_7_0_ViewSort {
-        cpus = 10
-        memory = '40G'
-        time = '40m'
-        clusterOptions = "$params.cluster_options --gres=tmpspace:10G"
+        time = '3h'
     }
 
     withLabel: Sambamba_0_7_0_MarkdupMerge {

WES.nf

@@ -4,8 +4,7 @@ nextflow.preview.dsl=2
 include extractFastqPairFromDir from './NextflowModules/Utils/fastq.nf'
 
 // Mapping modules
-include MEM as BWA_MEM from './NextflowModules/BWA/0.7.17/MEM.nf' params(genome:"$params.genome", optional: '-c 100 -M')
-include ViewSort as Sambamba_ViewSort from './NextflowModules/Sambamba/0.7.0/ViewSort.nf'
+include BWAMapping from './NextflowModules/BWA-Mapping/bwa-0.7.17_samtools-1.9/Mapping.nf' params(genome_fasta: "$params.genome", optional: '-c 100 -M')
 include MarkdupMerge as Sambamba_MarkdupMerge from './NextflowModules/Sambamba/0.7.0/Markdup.nf'
 
 // IndelRealignment modules
@@ -15,7 +14,7 @@ include ViewUnmapped as Sambamba_ViewUnmapped from './NextflowModules/Sambamba/0
 include Merge as Sambamba_Merge from './NextflowModules/Sambamba/0.7.0/Merge.nf'
 
 // HaplotypeCaller modules
-include IntervalListTools as PICARD_IntervalListTools from './NextflowModules/Picard/2.22.0/IntervalListTools.nf' params(scatter_count:'500')
+include IntervalListTools as PICARD_IntervalListTools from './NextflowModules/Picard/2.22.0/IntervalListTools.nf' params(scatter_count:"500", optional: "")
 include HaplotypeCaller as GATK_HaplotypeCaller from './NextflowModules/GATK/3.8-1-0-gf15c1c3ef/HaplotypeCaller.nf' params(gatk_path: "$params.gatk_path", genome:"$params.genome", optional: "$params.gatk_hc_options")
 include VariantFiltrationSnpIndel as GATK_VariantFiltration from './NextflowModules/GATK/3.8-1-0-gf15c1c3ef/VariantFiltration.nf' params(
     gatk_path: "$params.gatk_path", genome:"$params.genome", snp_filter: "$params.gatk_snp_filter", snp_cluster: "$params.gatk_snp_cluster", indel_filter: "$params.gatk_indel_filter"
@@ -50,10 +49,9 @@ if (!ped_file.exists()) {
 
 workflow {
     // Mapping
-    BWA_MEM(fastq_files)
-    Sambamba_ViewSort(BWA_MEM.out)
+    BWAMapping(fastq_files)
     Sambamba_MarkdupMerge(
-        Sambamba_ViewSort.out.map{
+        BWAMapping.out.map{
             sample_id, rg_id, bam_file, bai_file -> [sample_id, bam_file]
         }.groupTuple()
     )
@@ -94,14 +92,12 @@ workflow {
     Sambamba_Flagstat(Sambamba_Merge.out)
     GetStatsFromFlagstat(Sambamba_Flagstat.out.collect())
 
-    MultiQC(
-        Channel.empty().mix(
-            FastQC.out.flatten().map{file -> [analysis_id, file]},
-            PICARD_CollectMultipleMetrics.out.flatten().map{file -> [analysis_id, file]},
-            PICARD_EstimateLibraryComplexity.out.map{file -> [analysis_id, file]},
-            PICARD_CollectHsMetrics.out.map{file -> [analysis_id, file]}
-        ).groupTuple()
-    )
+    MultiQC(analysis_id, Channel.empty().mix(
+        FastQC.out.collect(),
+        PICARD_CollectMultipleMetrics.out.collect(),
+        PICARD_EstimateLibraryComplexity.out.collect(),
+        PICARD_CollectHsMetrics.out.collect()
+    ))
 
     TrendAnalysisTool(
         GATK_CombineVariants.out.map{id, vcf_file, idx_file -> [id, vcf_file]}
@@ -111,7 +107,7 @@ workflow {
     )
 
     //SavePedFile
-    SavePedFile() 
+    SavePedFile()
 
     // Repository versions
     VersionLog()
@@ -146,13 +142,13 @@ process ExonCov {
     shell = ['/bin/bash', '-eo', 'pipefail']
 
     input:
-    tuple analysis_id, sample_id, file(bam_file), file(bai_file)
+        tuple(analysis_id, sample_id, path(bam_file), path(bai_file))
 
     script:
-    """
-    source ${params.exoncov_path}/venv/bin/activate
-    python ${params.exoncov_path}/ExonCov.py import_bam --threads ${task.cpus} --overwrite --exon_bed ${params.dxtracks_path}/${params.exoncov_bed} ${analysis_id} ${bam_file}
-    """
+        """
+        source ${params.exoncov_path}/venv/bin/activate
+        python ${params.exoncov_path}/ExonCov.py import_bam --threads ${task.cpus} --overwrite --exon_bed ${params.dxtracks_path}/${params.exoncov_bed} ${analysis_id} ${bam_file}
+        """
 }
 
 process ExomeDepth {
@@ -162,16 +158,16 @@ process ExomeDepth {
     shell = ['/bin/bash', '-eo', 'pipefail']
 
     input:
-    tuple analysis_id, sample_id, file(bam_file), file(bai_file), refset
+        tuple(analysis_id, sample_id, path(bam_file), path(bai_file), refset)
 
     output:
-    tuple sample_id, refset, file("UMCU_${refset}_${sample_id}*.vcf"), file("HC_${refset}_${sample_id}*.vcf"), file("${sample_id}*.xml"), file("UMCU_${refset}_${sample_id}*.log"), file("HC_${refset}_${sample_id}*.log"), file("UMCU_${refset}_${sample_id}*.igv"), file("HC_${refset}_${sample_id}*.igv")
+        tuple(sample_id, refset, path("UMCU_${refset}_${sample_id}*.vcf"), path("HC_${refset}_${sample_id}*.vcf"), path("${sample_id}*.xml"), path("UMCU_${refset}_${sample_id}*.log"), path("HC_${refset}_${sample_id}*.log"), path("UMCU_${refset}_${sample_id}*.igv"), path("HC_${refset}_${sample_id}*.igv"))
 
     script:
-    """
-    source ${params.exomedepth_path}/venv/bin/activate
-    python ${params.exomedepth_path}/run_ExomeDepth.py callcnv ./ ${bam_file} ${analysis_id} ${sample_id} ${refset}
-    """
+        """
+        source ${params.exomedepth_path}/venv/bin/activate
+        python ${params.exomedepth_path}/run_ExomeDepth.py callcnv ./ ${bam_file} ${analysis_id} ${sample_id} ${refset}
+        """
 }
 
 
@@ -186,19 +182,19 @@ process Kinship {
     shell = ['/bin/bash', '-euo', 'pipefail']
 
     input:
-    tuple analysis_id, file(vcf_file), file(vcf_index)
+        tuple(analysis_id, path(vcf_file), path(vcf_index))
 
     output:
-    tuple analysis_id, file("${analysis_id}.kinship"), file("${analysis_id}.kinship_check.out")
+        tuple(analysis_id, path("${analysis_id}.kinship"), path("${analysis_id}.kinship_check.out"))
 
     script:
-    """
-    ${params.vcftools_path}/vcftools --vcf ${vcf_file} --plink
-    ${params.plink_path}/plink --file out --make-bed --noweb
-    ${params.king_path}/king -b plink.bed --kinship
-    cp king.kin0 ${analysis_id}.kinship
-    python ${baseDir}/assets/check_kinship.py ${analysis_id}.kinship ${ped_file} > ${analysis_id}.kinship_check.out
-    """
+        """
+        ${params.vcftools_path}/vcftools --vcf ${vcf_file} --plink
+        ${params.plink_path}/plink --file out --make-bed --noweb
+        ${params.king_path}/king -b plink.bed --kinship
+        cp king.kin0 ${analysis_id}.kinship
+        python ${baseDir}/assets/check_kinship.py ${analysis_id}.kinship ${ped_file} > ${analysis_id}.kinship_check.out
+        """
 }
 
 process GetStatsFromFlagstat {
@@ -208,15 +204,15 @@ process GetStatsFromFlagstat {
     shell = ['/bin/bash', '-euo', 'pipefail']
 
     input:
-    file(flagstat_files: "*")
+        path(flagstat_files)
 
     output:
-    file('run_stats.txt')
+        path('run_stats.txt')
 
     script:
-    """
-    python ${baseDir}/assets/get_stats_from_flagstat.py ${flagstat_files} > run_stats.txt
-    """
+        """
+        python ${baseDir}/assets/get_stats_from_flagstat.py ${flagstat_files} > run_stats.txt
+        """
 }
 
 process CreateHSmetricsSummary {
@@ -226,15 +222,15 @@ process CreateHSmetricsSummary {
     shell = ['/bin/bash', '-euo', 'pipefail']
 
     input:
-    file(hsmetrics_files: "*")
+        path(hsmetrics_files)
 
     output:
-    file('HSMetrics_summary.txt')
+        path('HSMetrics_summary.txt')
 
     script:
-    """
-    python ${baseDir}/assets/create_hsmetrics_summary.py ${hsmetrics_files} > HSMetrics_summary.txt
-    """
+        """
+        python ${baseDir}/assets/create_hsmetrics_summary.py ${hsmetrics_files} > HSMetrics_summary.txt
+        """
 }
 
 process TrendAnalysisTool {
@@ -244,13 +240,13 @@ process TrendAnalysisTool {
     shell = ['/bin/bash', '-eo', 'pipefail']
 
     input:
-    tuple analysis_id, file(input_files: "*")
+        tuple(analysis_id, path(input_files))
 
     script:
-    """
-    source ${params.trend_analysis_path}/venv/bin/activate
-    python ${params.trend_analysis_path}/trend_analysis.py upload processed_data ${analysis_id} .
-    """
+        """
+        source ${params.trend_analysis_path}/venv/bin/activate
+        python ${params.trend_analysis_path}/trend_analysis.py upload processed_data ${analysis_id} .
+        """
 }
 
 process SavePedFile {
@@ -259,12 +255,12 @@ process SavePedFile {
     shell = ['/bin/bash', '-euo', 'pipefail']
 
     output:
-    file("*.ped")
+        path("*.ped")
 
     script:
-    """
-    cp ${ped_file} ./
-    """
+        """
+        cp ${ped_file} ./
+        """
 }
 
 process VersionLog {
@@ -274,23 +270,23 @@ process VersionLog {
     shell = ['/bin/bash', '-eo', 'pipefail']
 
     output:
-    file('repository_version.log')
+        path('repository_version.log')
 
     script:
-    """
-    echo 'DxNextflowWes' > repository_version.log
-    git --git-dir=${workflow.projectDir}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+        """
+        echo 'DxNextflowWes' > repository_version.log
+        git --git-dir=${workflow.projectDir}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
 
-    echo 'Dx_tracks' >> repository_version.log
-    git --git-dir=${params.dxtracks_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+        echo 'Dx_tracks' >> repository_version.log
+        git --git-dir=${params.dxtracks_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
 
-    echo 'ExonCov' >> repository_version.log
-    git --git-dir=${params.exoncov_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+        echo 'ExonCov' >> repository_version.log
+        git --git-dir=${params.exoncov_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
 
-    echo 'ExomeDepth' >> repository_version.log
-    git --git-dir=${params.exomedepth_path}/../.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+        echo 'ExomeDepth' >> repository_version.log
+        git --git-dir=${params.exomedepth_path}/../.git log --pretty=oneline --decorate -n 2 >> repository_version.log
 
-    echo 'TrendAnalysis' >> repository_version.log
-    git --git-dir=${params.trend_analysis_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
-    """
+        echo 'TrendAnalysis' >> repository_version.log
+        git --git-dir=${params.trend_analysis_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+        """
 }

WES_Fingerprint.nf

@@ -0,0 +1,55 @@
+#!/usr/bin/env nextflow
+nextflow.preview.dsl=2
+
+include extractBamFromDir from './NextflowModules/Utils/bam.nf'
+
+// Fingerprint modules
+include UnifiedGenotyper as GATK_UnifiedGenotyper from './NextflowModules/GATK/3.8-1-0-gf15c1c3ef/UnifiedGenotyper.nf' params(gatk_path: "$params.gatk_path", genome:"$params.genome", optional: "--intervals $params.dxtracks_path/$params.fingerprint_target --output_mode EMIT_ALL_SITES")
+
+def bam_files = extractBamFromDir(params.bam_path)
+def analysis_id = params.outdir.split('/')[-1]
+
+workflow {
+    // GATK UnifiedGenotyper (fingerprint)
+    GATK_UnifiedGenotyper(bam_files)
+}
+
+// Workflow completion notification
+workflow.onComplete {
+    // HTML Template
+    def template = new File("$baseDir/assets/workflow_complete.html")
+    def binding = [
+        runName: analysis_id,
+        workflow: workflow
+    ]
+    def engine = new groovy.text.GStringTemplateEngine()
+    def email_html = engine.createTemplate(template).make(binding).toString()
+
+    // Send email
+    if (workflow.success) {
+        def subject = "WES Fingerprint Workflow Successful: ${analysis_id}"
+        sendMail(to: params.email, subject: subject, body: email_html)
+    } else {
+        def subject = "WES Fingerprint Workflow Failed: ${analysis_id}"
+        sendMail(to: params.email, subject: subject, body: email_html)
+    }
+}
+
+process VersionLog {
+    // Custom process to log repository versions
+    tag {"VersionLog ${analysis_id}"}
+    label 'VersionLog'
+    shell = ['/bin/bash', '-eo', 'pipefail']
+
+    output:
+        path('repository_version.log')
+
+    script:
+        """
+        echo 'DxNextflowWes' > repository_version.log
+        git --git-dir=${workflow.projectDir}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+
+        echo 'Dx_tracks' >> repository_version.log
+        git --git-dir=${params.dxtracks_path}/.git log --pretty=oneline --decorate -n 2 >> repository_version.log
+        """
+}

assets/check_kinship.py

@@ -27,8 +27,7 @@ def parse_ped(ped_file):
     return samples
 
 
-def check_kinship(kinship_file, samples):
-    kinship_setting = [0.177, 0.354]
+def check_kinship(kinship_file, samples, kinship_setting):
     kinship_errors = False
     print_kinship('sample_1', 'sample_2', 'kinship', 'related', 'type', 'status')  # header
     for line in kinship_file:
@@ -83,12 +82,16 @@ def print_kinship(sample_1, sample_2, kinship, fam_status, relation_status, kins
     parser.add_argument('ped_file', type=argparse.FileType('r'), help='PED file')
     arguments = parser.parse_args()
 
+    # settings
+    kinship_setting = [0.177, 0.354]
+
     # Parse ped file and check kinship
     samples = parse_ped(arguments.ped_file)
-    kinship_errors = check_kinship(arguments.kinship_file, samples)
+    kinship_errors = check_kinship(arguments.kinship_file, samples, kinship_setting)
 
     # Print summary
     if kinship_errors:
         print("\n# WARNING: Kinship errors found.")
     else:
         print("\n# No kinship errors found.")
+    print("# Used kinship check settings: {0}".format(kinship_setting))

run_nextflow_wes.sh

@@ -40,11 +40,11 @@ module load Java/1.8.0_60
 if [ \$? -eq 0 ]; then
     echo "Nextflow done."
 
-    echo "Running Nextflow clean"
-    /hpc/diaggen/software/tools/nextflow clean -f -k -q
-
     echo "Zip work directory"
-    zip -r -m -q work.zip work
+    find work -type f | egrep "\.(command|exitcode)" | zip -@ -q work.zip
+
+    echo "Remove work directory"
+    rm -r work
 
     echo "Creating md5sum"
     find -type f -not -iname 'md5sum.txt' -exec md5sum {} \; > md5sum.txt