Merge pull request #97 from RoanKanninga/master

get NGS_Demultiplexing working on betabarrel

generate_template.sh

@@ -1,36 +1,28 @@
 #!/bin/bash
-set -e 
-set -u
+set -eu
 
 ##external call with 3 arguments (rawdataName, group and workDir)
 
 module list
+host=$(hostname -s)
 
-ENVIRONMENT_PARAMETERS="parameters_gattaca.csv"
+ENVIRONMENT_PARAMETERS="parameters_${host}.csv"
 RAWDATANAME="${1}"
 WORKDIR="${2}"
 GROUP="${3}"
 WORKFLOW="${EBROOTNGS_DEMULTIPLEXING}/workflow.csv"
 echo "$WORKDIR AND $RAWDATANAME"
 echo "GROUPIE: $GROUP"
 
-if [ -f ".compute.properties" ];
-then
-     rm ".compute.properties"
-fi
-
-mkdir -p "${WORKDIR}/generatedscripts/${RAWDATANAME}/"
+mkdir -p "${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/"
 mkdir -p "${WORKDIR}/rawdata/ngs/${RAWDATANAME}/"
 
-if [ -f "${WORKDIR}/generatedscripts/${RAWDATANAME}/out.csv"  ];
-then
-	rm -rf "${WORKDIR}/generatedscripts/${RAWDATANAME}/out.csv"
-fi
+rm -rf "${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/out.csv"
 
 #
 ###### Dual barcode checker
 #
-sampsheet="${WORKDIR}/generatedscripts/${RAWDATANAME}/${RAWDATANAME}.csv"
+sampsheet="${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/${RAWDATANAME}.csv"
 
 mac2unix "${sampsheet}"
 
@@ -59,21 +51,21 @@ fi
 
 ## Will return or nothing or in case there is a dualbarcode it will create a file
 perl "${EBROOTNGS_DEMULTIPLEXING}/convertParametersGitToMolgenis.pl" "${EBROOTNGS_DEMULTIPLEXING}/parameters.csv" > \
-"${WORKDIR}/generatedscripts/${RAWDATANAME}/out.csv"
+"${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/out.csv"
 
 perl "${EBROOTNGS_DEMULTIPLEXING}/convertParametersGitToMolgenis.pl" "${EBROOTNGS_DEMULTIPLEXING}/${ENVIRONMENT_PARAMETERS}" > \
-"${WORKDIR}/generatedscripts/${RAWDATANAME}/environment_parameters.csv"
+"${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/environment_parameters.csv"
 
 perl "${EBROOTNGS_DEMULTIPLEXING}/convertParametersGitToMolgenis.pl" "${EBROOTNGS_DEMULTIPLEXING}/parameters_${GROUP}.csv" > \
-"${WORKDIR}/generatedscripts/${RAWDATANAME}/parameters_group.csv"
+"${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/parameters_group.csv"
 
 bash "${EBROOTMOLGENISMINCOMPUTE}/molgenis_compute.sh" \
--p "${WORKDIR}/generatedscripts/${RAWDATANAME}/out.csv" \
--p "${WORKDIR}/generatedscripts/${RAWDATANAME}/parameters_group.csv" \
--p "${WORKDIR}/generatedscripts/${RAWDATANAME}/environment_parameters.csv" \
--p "${WORKDIR}/generatedscripts/${RAWDATANAME}/${RAWDATANAME}.csv" \
+-p "${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/out.csv" \
+-p "${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/parameters_group.csv" \
+-p "${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/environment_parameters.csv" \
+-p "${WORKDIR}/generatedscripts/NGS_Demultiplexing/${RAWDATANAME}/${RAWDATANAME}.csv" \
 -w "${WORKFLOW}" \
--rundir "${WORKDIR}/runs/${RAWDATANAME}/jobs" \
+-rundir "${WORKDIR}/runs/NGS_Demultiplexing/${RAWDATANAME}/jobs" \
 -o "dualBarcode=${dualBarcode};\
 demultiplexingversion=$(module list | grep -o -P 'NGS_Demultiplexing(.+)')" \
 -b slurm \

parameters.csv

@@ -3,9 +3,9 @@ stage,module load
 checkStage,module list
 
 ### Tool versions ####
-bcl2fastqVersion,bcl2fastq2/2.20.0-foss-2018b
-perlPlusVersion,PerlPlus/5.30.0-GCCcore-7.3.0-v19.08.1
-interopVersion,interop/1.1.23-foss-2018b
+bcl2fastqVersion,bcl2fastq2/2.20.0-foss-2022a
+perlPlusVersion,PerlPlus/5.34.1-GCCcore-11.3.0-v22.11.1
+interopVersion,interop/1.2.0-foss-2022a
 
 #### Prefixes ####
 filePrefix,${sequencingStartDate}_${sequencer}_${run}_${flowcell}
@@ -14,69 +14,50 @@ runPrefix,${filePrefix}_L${lane}
 ##### GENERAL DIRECTORIES #####
 dataDir,${root}/data/
 rawDataDir,${workDir}/rawdata
-
 ngsDir,${rawDataDir}/ngs/${filePrefix}/
 arrayDir,${rawDataDir}/array 
-
 runID,run_${run}
-generatedScriptsDir,${workDir}/generatedscripts/${filePrefix}/
-runDir,${workDir}/runs/${filePrefix}/
-runJobsDir,${runDir}/jobs/
+generatedScriptsDir,${workDir}/generatedscripts/NGS_Demultiplexing/${filePrefix}/
+runDir,${workDir}/runs/NGS_Demultiplexing/${filePrefix}/
+runJobsDir,${runDir}/jobs/## CreateFinalReport
 runResultsDir,${runDir}/results/
 prepKitsDir,${dataDir}/Prepkits/
-intermediateDir,${workDir}/tmp/${filePrefix}/
-
+intermediateDir,${workDir}/tmp/NGS_Demultiplexing/${filePrefix}/
 bcl2fastqDir,${nextSeqRunDataDir}/${project}
-sampleSheet,${workDir}/generatedscripts/${filePrefix}/${filePrefix}.csv
-MCsampleSheet,${intermediateDir}/status_samples.csv
 
+sampleSheet,${workDir}/generatedscripts/NGS_Demultiplexing/${filePrefix}/${filePrefix}.csv
+MCsampleSheet,${intermediateDir}/status_samples.csv
 sourcesDir,${root}/sources/
-
 adapterStringency,0.9
-
-## CreateFinalReport
 createPerSampleFinalReportPl,CreateFinalReportPerSample.pl
 finalReportResultDir,${nextSeqRunDataDir}/finalreports/
-
 fastqExtension,.fq
 gzipExtension,.gz
 md5sumExtension,.md5
-
 filenameSuffixPE1,1
 filenameSuffixPE2,2
 filenameSuffixDiscardedReads,DISCARDED
-
 fastqFilenameSR,${runPrefix}${fastqExtension}
 fastqFilenamePE1,${runPrefix}_${filenameSuffixPE1}${fastqExtension}
 fastqFilenamePE2,${runPrefix}_${filenameSuffixPE2}${fastqExtension}
-
 compressedFastqFilenameSR,${fastqFilenameSR}${gzipExtension}
 compressedFastqFilenamePE1,${fastqFilenamePE1}${gzipExtension}
 compressedFastqFilenamePE2,${fastqFilenamePE2}${gzipExtension}
-
-
 demultiplexedSampleFastqFilenameSR,${runPrefix}_${barcode}${fastqExtension}
 demultiplexedSampleFastqFilenamePE1,${runPrefix}_${barcode}_${filenameSuffixPE1}${fastqExtension}
 demultiplexedSampleFastqFilenamePE2,${runPrefix}_${barcode}_${filenameSuffixPE2}${fastqExtension}
-
 demultiplexedDiscardedFastqFilenameSR,${runPrefix}_${filenameSuffixDiscardedReads}${fastqExtension}
 demultiplexedDiscardedFastqFilenamePE1,${runPrefix}_${filenameSuffixDiscardedReads}_${filenameSuffixPE1}${fastqExtension}
 demultiplexedDiscardedFastqFilenamePE2,${runPrefix}_${filenameSuffixDiscardedReads}_${filenameSuffixPE2}${fastqExtension}
-
-
 compressedDemultiplexedSampleFastqFilenameSR,${demultiplexedSampleFastqFilenameSR}${gzipExtension}
 compressedDemultiplexedSampleFastqFilenamePE1,${demultiplexedSampleFastqFilenamePE1}${gzipExtension}
 compressedDemultiplexedSampleFastqFilenamePE2,${demultiplexedSampleFastqFilenamePE2}${gzipExtension}
-
 compressedDemultiplexedDiscardedFastqFilenameSR,${demultiplexedDiscardedFastqFilenameSR}${gzipExtension}
 compressedDemultiplexedDiscardedFastqFilenamePE1,${demultiplexedDiscardedFastqFilenamePE1}${gzipExtension}
 compressedDemultiplexedDiscardedFastqFilenamePE2,${demultiplexedDiscardedFastqFilenamePE2}${gzipExtension}
-
-
 demultiplexedSampleFastqChecksumFilenameSR,${demultiplexedSampleFastqFilenameSR}${md5sumExtension}
 demultiplexedSampleFastqChecksumFilenamePE1,${demultiplexedSampleFastqFilenamePE1}${md5sumExtension}
 demultiplexedSampleFastqChecksumFilenamePE2,${demultiplexedSampleFastqFilenamePE2}${md5sumExtension}
-
 demultiplexedDiscardedFastqChecksumFilenameSR,${demultiplexedDiscardedFastqFilenameSR}${md5sumExtension}
 demultiplexedDiscardedFastqChecksumFilenamePE1,${demultiplexedDiscardedFastqFilenamePE1}${md5sumExtension}
 demultiplexedDiscardedFastqChecksumFilenamePE2,${demultiplexedDiscardedFastqFilenamePE2}${md5sumExtension}

parameters_betabarrel.csv

@@ -0,0 +1,4 @@
+tmpName,tmp05
+root,/apps/
+nextSeqDir,/groups/umcg-lab/${tmpName}/sequencers
+nextSeqRunDataDir,${nextSeqDir}/${filePrefix}/

parameters_copperfist.csv

@@ -0,0 +1,4 @@
+tmpName,tmp06
+root,/apps/
+nextSeqDir,/groups/umcg-lab/${tmpName}/sequencers
+nextSeqRunDataDir,${nextSeqDir}/${filePrefix}/

parameters_wingedhelix.csv

@@ -0,0 +1,4 @@
+tmpName,tmp07
+root,/apps/
+nextSeqDir,/groups/umcg-lab/${tmpName}/sequencers
+nextSeqRunDataDir,${nextSeqDir}/${filePrefix}/

protocols/Bcl2FastQ.sh

@@ -10,7 +10,7 @@
 #string runJobsDir
 #string ngsDir
 #string prepKitsDir
-#string ngsUtilsVersion
+#string perlPlusVersion
 #string dualBarcode
 #string runResultsDir
 #string barcodeType
@@ -24,31 +24,13 @@ ${checkStage}
 #
 # Initialize script specific vars.
 #
-if [ ! -d "${runResultsDir}" ]
-then
-	mkdir -p "${runResultsDir}"
-	echo "mkdir ${runResultsDir}"
-fi
-#Make an intermediate and resultsDir
-if [ ! -d "${intermediateDir}" ]
-then
-	mkdir -p "${intermediateDir}"
-fi
+mkdir -vp "${runResultsDir}"
+mkdir -vp "${intermediateDir}"
 
-if [ -d "${intermediateDir}/Reports" ]
-then
-	rm -rf "${intermediateDir}/Reports"
-fi
+rm -rf "${intermediateDir}/Reports"
+rm -rf "${intermediateDir}/Stats"
 
-if [ -d "${intermediateDir}/Stats" ]
-then
-	rm -rf "${intermediateDir}/Stats"
-fi
-
-if [ ! -d "${ngsDir}" ]
-then
-	mkdir -p "${ngsDir}"
-fi
+mkdir -vp "${ngsDir}"
 
 cp "${sampleSheet}" "${runJobsDir}"
 
@@ -61,17 +43,17 @@ echo "tmpIntermediateDir: ${tmpIntermediateDir}"
 
 module load "${demultiplexingversion}"
 
-if [ "${dualBarcode}" == "TRUE" ]
+if [[ "${dualBarcode}" == 'TRUE' ]]
 then
-	echo "dualBarcode modus on"
+	echo 'dualBarcode modus on'
 	perl "${EBROOTNGS_DEMULTIPLEXING}/CreateIlluminaSampleSheet_V3.pl" \
 	-i "${sampleSheet}" \
 	-o "${tmpIntermediateDir}/Illumina_R${run}.csv" \
 	-r "${run}" \
 	-d TRUE \
 	-s "${prepKitsDir}"
 else
-	echo "only one barcode detected"
+	echo 'only one barcode detected'
 	perl "${EBROOTNGS_DEMULTIPLEXING}/CreateIlluminaSampleSheet_V2.pl" \
 	-i "${sampleSheet}" \
 	-o "${tmpIntermediateDir}/Illumina_R${run}.csv" \