Skip to content

Commit

Permalink
add new config,dry-run and enforce profiles
Browse files Browse the repository at this point in the history
  • Loading branch information
@edsu7
edsu7 committed Nov 30, 2023
1 parent 0d82991 commit ab451f4
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 73 deletions.
160 changes: 99 additions & 61 deletions argo-data-submission-wf/main.nf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,11 @@ params.download_mode="local"
params.song_container = "ghcr.io/overture-stack/song-client"
params.song_container_version = "5.0.2"
params.score_container = "ghcr.io/overture-stack/score"
params.score_container_version = "5.9.0"
params.score_container_version = "5.10.0"
params.score_mem = 20
params.score_cpus = 8
params.score_force = false
params.dry_run = false

// sanityChecks
params.song_url=""
Expand All @@ -58,7 +59,7 @@ params.api_token=""
// payloadJsonToTsvs
params.data_directory="NO_FILE1"
params.skip_duplicate_check=false

params.skip_sanity_check=false
// payloadGenSeqExperiment
params.schema_url=""
params.experiment_info_tsv="NO_FILE2"
Expand Down Expand Up @@ -237,6 +238,19 @@ workflow ArgoDataSubmissionWf {
clinical_url
main:

if (!"${workflow.profile}".contains('docker') && !"${workflow.profile}".contains('singularity')){
exit 1, "Error Missing profile. `-profile` must be specified with the engines :`docker` or `singularity`."
}
if (!"${workflow.profile}".contains('rdpc_qa') && !"${workflow.profile}".contains('rdpc_dev') && !"${workflow.profile}".contains('rdpc')){
exit 1, "Error Missing profile. `-profile` must be specified with the rdpc environments : `rdpc_qa`,`rdpc_dev`, or `rdpc`."
}

if (!params.api_token){
if (!params.api_download_token || !params.api_upload_token) {
exit 1, "Error SONG parameters detected but missing token params. `--api_token` or `api_upload_token` and `api_download_token` must be supplied when uploading."
}
}

if (
og_experiment_info_tsv.startsWith("NO_FILE") && \
og_read_group_info_tsv.startsWith("NO_FILE") && \
Expand Down Expand Up @@ -278,29 +292,41 @@ workflow ArgoDataSubmissionWf {
file(data_directory)
)

sanityCheck(
payloadJsonToTsvs.out.experiment_tsv,
api_token,
song_url,
clinical_url,
params.skip_duplicate_check
)

experiment_info_tsv=sanityCheck.out.updated_experiment_info_tsv
read_group_info_tsv=payloadJsonToTsvs.out.read_group_tsv
file_info_tsv=payloadJsonToTsvs.out.file_tsv
if (params.skip_sanity_check){
experiment_info_tsv=file(payloadJsonToTsvs.out.experiment_tsv)
read_group_info_tsv=file(og_read_group_info_tsv)
file_info_tsv=file(og_file_info_tsv)
} else {
sanityCheck(
payloadJsonToTsvs.out.experiment_tsv,
api_token,
song_url,
clinical_url,
params.skip_duplicate_check
)

experiment_info_tsv=sanityCheck.out.updated_experiment_info_tsv
read_group_info_tsv=file(og_read_group_info_tsv)
file_info_tsv=file(og_file_info_tsv)
}
} else {
sanityCheck(
file(og_experiment_info_tsv),
api_token,
song_url,
clinical_url,
params.skip_duplicate_check
)

experiment_info_tsv=sanityCheck.out.updated_experiment_info_tsv
read_group_info_tsv=file(og_read_group_info_tsv)
file_info_tsv=file(og_file_info_tsv)
if (params.skip_sanity_check){
experiment_info_tsv=file(og_experiment_info_tsv)
read_group_info_tsv=file(og_read_group_info_tsv)
file_info_tsv=file(og_file_info_tsv)
} else {
sanityCheck(
file(og_experiment_info_tsv),
api_token,
song_url,
clinical_url,
params.skip_duplicate_check
)

experiment_info_tsv=sanityCheck.out.updated_experiment_info_tsv
read_group_info_tsv=file(og_read_group_info_tsv)
file_info_tsv=file(og_file_info_tsv)
}
}

checkCramReference(
Expand Down Expand Up @@ -362,20 +388,22 @@ workflow ArgoDataSubmissionWf {
skipping_tests
)

uploadWf(
study_id,
valSeq.out.validated_payload,
sequence_files.collect(),
''
)

submissionReceipt(
if (!params.dry_run){
uploadWf(
study_id,
uploadWf.out.analysis_id,
song_url,
params.skip_submission_check,
sequence_files.collect()
)
valSeq.out.validated_payload,
sequence_files.collect(),
''
)

submissionReceipt(
study_id,
uploadWf.out.analysis_id,
song_url,
params.skip_submission_check,
sequence_files.collect()
)
}

} else if (checkCramReference.out.check_status && !ref_genome_fa.startsWith("NO_FILE")){
// If reference genome is provided...
Expand Down Expand Up @@ -403,52 +431,62 @@ workflow ArgoDataSubmissionWf {
sequence_files.collect().concat(cram2bam.out.output_bam.collect()).collect(),
skipping_tests
)

uploadWf(
study_id,
valSeq.out.validated_payload,
not_cram_sequence_files.concat(cram2bam.out.output_bam.collect()).collect(),
''
)

submissionReceipt(
if (!params.dry_run){
uploadWf(
study_id,
uploadWf.out.analysis_id,
song_url,
params.skip_submission_check,
valSeq.out.validated_payload,
not_cram_sequence_files.concat(cram2bam.out.output_bam.collect()).collect(),
)
''
)

submissionReceipt(
study_id,
uploadWf.out.analysis_id,
song_url,
params.skip_submission_check,
not_cram_sequence_files.concat(cram2bam.out.output_bam.collect()).collect()
)
}
}
if (params.cleanup && params.download_mode!='local' && ref_genome_fa.startsWith("NO_FILE")) {
if (params.cleanup && params.download_mode!='local' && ref_genome_fa.startsWith("NO_FILE") && !params.dry_run) {
// only cleanup the sequence files when they are not from local
cleanup(
sequence_files.collect(),
submissionReceipt.out.receipt // wait until upload is done
)
} else if (params.cleanup && params.download_mode!='local' && !ref_genome_fa.startsWith("NO_FILE")){
} else if (params.cleanup && params.download_mode!='local' && !ref_genome_fa.startsWith("NO_FILE") && !params.dry_run){
// only cleanup the sequence files and cram2bam output when they are not from local
cleanup(
sequence_files.collect().concat(cram2bam.out.output_bam.collect()).collect(),
submissionReceipt.out.receipt // wait until upload is done
)
} else if (params.cleanup && params.download_mode=='local' && !ref_genome_fa.startsWith("NO_FILE")){
} else if (params.cleanup && params.download_mode=='local' && !ref_genome_fa.startsWith("NO_FILE") && !params.dry_run){
// only cleanup output from cram2bam on local
cleanup(
cram2bam.out.output_bam.collect(),
submissionReceipt.out.receipt // wait until upload is done
)
}

printOut(
pGenExp.out.payload,
uploadWf.out.analysis_id,
submissionReceipt.out.receipt
)
if (!params.dry_run){
printOut(
pGenExp.out.payload,
uploadWf.out.analysis_id,
submissionReceipt.out.receipt
)
out_payload = pGenExp.out.payload
out_analysis_id = uploadWf.out.analysis_id
out_receipt = submissionReceipt.out.receipt
} else {
out_payload = null
out_analysis_id = null
out_receipt = null
}

emit:
json_file=pGenExp.out.payload
output_analysis_id=uploadWf.out.analysis_id
receipt=submissionReceipt.out.receipt
out_json_file=out_payload
out_output_analysis_id=out_analysis_id
out_receipt=out_receipt
}

// this provides an entry point for this main script, so it can be run directly without clone the repo
Expand Down
24 changes: 12 additions & 12 deletions nextflow.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,11 @@ process {
}

profiles {
collab {
params.song_url = 'https://submission-song.rdpc.cancercollaboratory.org'
params.score_url = 'https://submission-score.rdpc.cancercollaboratory.org'
rdpc {
params.song_url = 'https://submission-song.rdpc.argo.genomeinformatics.org'
params.score_url = 'https://submission-score.rdpc.argo.genomeinformatics.org'
params.clinical_url = 'https://clinical.platform.icgc-argo.org'
params.schema_url = 'https://submission-song.rdpc.cancercollaboratory.org/schemas/sequencing_experiment'
params.schema_url = 'https://submission-song.rdpc.argo.genomeinformatics.org/schemas/sequencing_experiment'
}
docker {
docker.enabled = true
Expand All @@ -73,16 +73,16 @@ profiles {
singularity.autoMounts = true
singularity.engineOptions = '-s'
}
debug_qa {
params.song_url = 'https://submission-song.rdpc-qa.cancercollaboratory.org'
params.score_url = 'https://submission-score.rdpc-qa.cancercollaboratory.org'
rdpc_qa {
params.song_url = 'https://submission-song.rdpc-qa.cumulus.genomeinformatics.org'
params.score_url = 'https://submission-score.rdpc-qa.cumulus.genomeinformatics.org'
params.clinical_url = 'https://clinical.qa.argo.cancercollaboratory.org'
params.schema_url = 'https://submission-song.rdpc-qa.cancercollaboratory.org/schemas/sequencing_experiment'
params.schema_url = 'https://submission-song.rdpc-qa.cumulus.genomeinformatics.org/schemas/sequencing_experiment'
}
debug_dev {
params.song_url = 'https://submission-song.rdpc-dev.cancercollaboratory.org'
params.score_url = 'https://submission-score.rdpc-dev.cancercollaboratory.org'
rdpc_dev {
params.song_url = 'https://submission-song.rdpc-dev.cumulus.genomeinformatics.org'
params.score_url = 'https://submission-score.rdpc-dev.cumulus.genomeinformatics.org'
params.clinical_url = 'https://clinical.dev.argo.cancercollaboratory.org'
params.schema_url = 'https://submission-song.rdpc-dev.cancercollaboratory.org/schemas/sequencing_experiment'
params.schema_url = 'https://submission-song.rdpc-dev.cumulus.genomeinformatics.org/schemas/sequencing_experiment'
}
}

0 comments on commit ab451f4

Please sign in to comment.