From 09efa91d770b332f8cff2b083e940756b6177362 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 15 Aug 2024 14:50:54 -0400 Subject: [PATCH 01/25] Update locidex container registry --- modules/local/locidex/merge/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/locidex/merge/main.nf b/modules/local/locidex/merge/main.nf index 058d3a3..66ba9a7 100644 --- a/modules/local/locidex/merge/main.nf +++ b/modules/local/locidex/merge/main.nf @@ -5,8 +5,8 @@ process LOCIDEX_MERGE { label 'process_medium' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/locidex:0.1.1--pyhdfd78af_0' : - 'quay.io/biocontainers/locidex:0.1.1--pyhdfd78af_0' }" + 'docker.io/mwells14/locidex:0.2.2' : + 'docker.io/mwells14/locidex:0.2.2' }" input: path input_values // [file(sample1), file(sample2), file(sample3), etc...] From 9e820c828c703bff5958cc02f387715768cf9dc8 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 15 Aug 2024 14:52:43 -0400 Subject: [PATCH 02/25] Update input_assure.py to accomodate the revised Locidex mlst.json report format --- assets/samplesheet.csv | 6 +++--- bin/input_assure.py | 29 ++++++++++++++++++----------- conf/test.config | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/assets/samplesheet.csv b/assets/samplesheet.csv index 82842ce..7438d85 100644 --- a/assets/samplesheet.csv +++ b/assets/samplesheet.csv @@ -1,4 +1,4 @@ sample,mlst_alleles -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json +sample1,/root/working_directory/nml-phac/gasclustering/tests/data/reports/sample1.mlst.json +sample2,/root/working_directory/nml-phac/gasclustering/tests/data/reports/sample2.mlst.json +sample3,/root/working_directory/nml-phac/gasclustering/tests/data/reports/sample3.mlst.json diff --git a/bin/input_assure.py b/bin/input_assure.py index d99bf2a..7f5d56e 100755 --- a/bin/input_assure.py +++ b/bin/input_assure.py @@ -19,18 +19,24 @@ def check_inputs(json_file, sample_id, address, output_error_file, output_json_f with open_file(json_file, "rt") as f: json_data = json.load(f) + print(json.dumps(json_data, indent=4)) + + # Extract the profile from the json_data + profile = json_data.get("data", {}).get("profile", {}) + # Check for multiple keys in the JSON file and define error message + keys = list(profile.keys()) + original_key = keys[0] if keys else None + # Define a variable to store the match_status (True or False) - match_status = sample_id in json_data + match_status = sample_id in profile # Initialize the error message error_message = None - # Check for multiple keys in the JSON file and define error message - keys = list(json_data.keys()) - original_key = keys[0] if keys else None - - if len(keys) == 0: - error_message = f"{json_file} is completely empty!" + if not keys: + error_message = ( + f"{json_file} is missing the 'profile' section or is completely empty!" + ) print(error_message) sys.exit(1) elif len(keys) > 1: @@ -38,11 +44,11 @@ def check_inputs(json_file, sample_id, address, output_error_file, output_json_f if not match_status: error_message = f"No key in the MLST JSON file ({json_file}) matches the specified sample ID '{sample_id}'. The first key '{original_key}' has been forcefully changed to '{sample_id}' and all other keys have been removed." # Retain only the specified sample ID - json_data = {sample_id: json_data.pop(original_key)} + json_data["data"]["profile"] = {sample_id: profile.pop(original_key)} else: error_message = f"MLST JSON file ({json_file}) contains multiple keys: {keys}. The MLST JSON file has been modified to retain only the '{sample_id}' entry" - # Remove all keys expect the one matching sample_id - json_data = {sample_id: json_data[sample_id]} + # Retain only the specified sample_id in the profile + json_data["data"]["profile"] = {sample_id: profile[sample_id]} elif not match_status: # Define error message based on meta.address (query or reference) if address == "null": @@ -50,7 +56,8 @@ def check_inputs(json_file, sample_id, address, output_error_file, output_json_f else: error_message = f"Reference {sample_id} ID and JSON key in {json_file} DO NOT MATCH. The '{original_key}' key in {json_file} has been forcefully changed to '{sample_id}': User should manually check input files to ensure correctness." # Update the JSON file with the new sample ID - json_data[sample_id] = json_data.pop(original_key) + json_data["data"]["profile"] = {sample_id: profile.pop(original_key)} + json_data["data"]["sample_name"] = sample_id # Write file containing relevant error messages if error_message: diff --git a/conf/test.config b/conf/test.config index 51549ed..b0679a2 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,7 +20,7 @@ params { max_time = '1.h' // Input data - input = 'https://raw.githubusercontent.com/phac-nml/gasclustering/dev/assets/samplesheet.csv' + input = "${projectDir}/assets/samplesheet.csv" } From 29debe1722bb2d7c8c9dbc47c0adfe554198c34e Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 15 Aug 2024 14:53:34 -0400 Subject: [PATCH 03/25] Update test data (reports and samplesheets) to accomodate the revised Locidex mlst.json report format --- .../case-hamming/sample1.mlst.subtyping.json | 22 +++++++++++++++---- .../case-hamming/sample2.mlst.subtyping.json | 22 +++++++++++++++---- .../case-hamming/sample3.mlst.subtyping.json | 22 +++++++++++++++---- .../sample1.mlst.subtyping.json | 22 +++++++++++++++---- .../sample2.mlst.subtyping.json | 22 +++++++++++++++---- .../sample3-more-missing.mlst.subtyping.json | 22 +++++++++++++++---- .../sample3.mlst.subtyping.json | 22 +++++++++++++++---- tests/data/reports/sample1.mlst.json | 22 +++++++++++++++---- tests/data/reports/sample2.mlst.json | 22 +++++++++++++++---- tests/data/reports/sample3.mlst.json | 22 +++++++++++++++---- .../data/samplesheets/samplesheet-hamming.csv | 6 ++--- .../samplesheets/samplesheet-hash-missing.csv | 6 ++--- .../samplesheet-hash-more-missing.csv | 6 ++--- .../samplesheet-little-metadata.csv | 6 ++--- .../samplesheet-mismatched-ids.csv | 6 ++--- .../samplesheets/samplesheet-no-metadata.csv | 6 ++--- .../samplesheet-partial-mismatched-ids.csv | 6 ++--- tests/data/samplesheets/samplesheet-tabs.csv | 6 ++--- tests/data/samplesheets/samplesheet1.csv | 6 ++--- 19 files changed, 207 insertions(+), 67 deletions(-) diff --git a/tests/data/reports/case-hamming/sample1.mlst.subtyping.json b/tests/data/reports/case-hamming/sample1.mlst.subtyping.json index 01bc774..63a71b4 100644 --- a/tests/data/reports/case-hamming/sample1.mlst.subtyping.json +++ b/tests/data/reports/case-hamming/sample1.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample1": { - "l1": "1", - "l2": "1", - "l3": "1" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample1", + "profile": { + "sample1": { + "l1": "1", + "l2": "1", + "l3": "1" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/case-hamming/sample2.mlst.subtyping.json b/tests/data/reports/case-hamming/sample2.mlst.subtyping.json index 1e8b0b2..704bcfe 100644 --- a/tests/data/reports/case-hamming/sample2.mlst.subtyping.json +++ b/tests/data/reports/case-hamming/sample2.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample2": { - "l1": "1", - "l2": "2", - "l3": "1" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample2", + "profile": { + "sample2": { + "l1": "1", + "l2": "2", + "l3": "1" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/case-hamming/sample3.mlst.subtyping.json b/tests/data/reports/case-hamming/sample3.mlst.subtyping.json index bec5935..7e8d7f8 100644 --- a/tests/data/reports/case-hamming/sample3.mlst.subtyping.json +++ b/tests/data/reports/case-hamming/sample3.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample3": { - "l1": "2", - "l2": "1", - "l3": "2" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample3", + "profile": { + "sample3": { + "l1": "2", + "l2": "1", + "l3": "2" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json b/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json index e559003..eb8c116 100644 --- a/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json +++ b/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample1": { - "l1": "b026324c6904b2a9cb4b88d6d61c81d1", - "l2": "b026324c6904b2a9cb4b88d6d61c81d1", - "l3": "b026324c6904b2a9cb4b88d6d61c81d1" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample1", + "profile": { + "sample1": { + "l1": "b026324c6904b2a9cb4b88d6d61c81d1", + "l2": "b026324c6904b2a9cb4b88d6d61c81d1", + "l3": "b026324c6904b2a9cb4b88d6d61c81d1" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json b/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json index 06148ce..ee75b74 100644 --- a/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json +++ b/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample2": { - "l1": "-", - "l2": "26ab0db90d72e28ad0ba1e22ee510510", - "l3": "b026324c6904b2a9cb4b88d6d61c81d1" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample2", + "profile": { + "sample2": { + "l1": "-", + "l2": "26ab0db90d72e28ad0ba1e22ee510510", + "l3": "b026324c6904b2a9cb4b88d6d61c81d1" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json b/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json index 7546de9..0bf1372 100644 --- a/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json +++ b/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample3": { - "l1": "-", - "l2": "-", - "l3": "26ab0db90d72e28ad0ba1e22ee510510" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample3", + "profile": { + "sample3": { + "l1": "-", + "l2": "-", + "l3": "26ab0db90d72e28ad0ba1e22ee510510" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json b/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json index a0f0259..fe8441d 100644 --- a/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json +++ b/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json @@ -1,7 +1,21 @@ { - "sample3": { - "l1": "-", - "l2": "b026324c6904b2a9cb4b88d6d61c81d1", - "l3": "26ab0db90d72e28ad0ba1e22ee510510" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample3", + "profile": { + "sample3": { + "l1": "-", + "l2": "b026324c6904b2a9cb4b88d6d61c81d1", + "l3": "26ab0db90d72e28ad0ba1e22ee510510" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/sample1.mlst.json b/tests/data/reports/sample1.mlst.json index 01bc774..63a71b4 100644 --- a/tests/data/reports/sample1.mlst.json +++ b/tests/data/reports/sample1.mlst.json @@ -1,7 +1,21 @@ { - "sample1": { - "l1": "1", - "l2": "1", - "l3": "1" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample1", + "profile": { + "sample1": { + "l1": "1", + "l2": "1", + "l3": "1" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/sample2.mlst.json b/tests/data/reports/sample2.mlst.json index 7c0426c..3d9ee23 100644 --- a/tests/data/reports/sample2.mlst.json +++ b/tests/data/reports/sample2.mlst.json @@ -1,7 +1,21 @@ { - "sample2": { - "l1": "1", - "l2": "1", - "l3": "1" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample2", + "profile": { + "sample2": { + "l1": "1", + "l2": "1", + "l3": "1" + } + }, + "seq_data": {} } } diff --git a/tests/data/reports/sample3.mlst.json b/tests/data/reports/sample3.mlst.json index 43ea3c7..d57ee75 100644 --- a/tests/data/reports/sample3.mlst.json +++ b/tests/data/reports/sample3.mlst.json @@ -1,7 +1,21 @@ { - "sample3": { - "l1": "1", - "l2": "1", - "l3": "2" + "db_info": {}, + "parameters": { + "mode": "normal", + "min_match_ident": 100, + "min_match_cov": 100, + "max_ambiguous": 0, + "max_internal_stops": 0 + }, + "data": { + "sample_name": "sample3", + "profile": { + "sample3": { + "l1": "1", + "l2": "1", + "l3": "2" + } + }, + "seq_data": {} } } diff --git a/tests/data/samplesheets/samplesheet-hamming.csv b/tests/data/samplesheets/samplesheet-hamming.csv index d18a69c..14f2469 100644 --- a/tests/data/samplesheets/samplesheet-hamming.csv +++ b/tests/data/samplesheets/samplesheet-hamming.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hamming/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hamming/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample1.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/update/input_assure/reports/case-hamming/sample2.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/update/input_assure/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-hash-missing.csv b/tests/data/samplesheets/samplesheet-hash-missing.csv index 9355c3d..6ad5929 100644 --- a/tests/data/samplesheets/samplesheet-hash-missing.csv +++ b/tests/data/samplesheets/samplesheet-hash-missing.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-hash-more-missing.csv b/tests/data/samplesheets/samplesheet-hash-more-missing.csv index ae223fb..37f051f 100644 --- a/tests/data/samplesheets/samplesheet-hash-more-missing.csv +++ b/tests/data/samplesheets/samplesheet-hash-more-missing.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/profile_dists/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/profile_dists/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/profile_dists/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-little-metadata.csv b/tests/data/samplesheets/samplesheet-little-metadata.csv index 3e721de..052b4b3 100644 --- a/tests/data/samplesheets/samplesheet-little-metadata.csv +++ b/tests/data/samplesheets/samplesheet-little-metadata.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,,,,1.4,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,,,,,,3.8 +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,,,,1.4,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,,,,,,3.8 diff --git a/tests/data/samplesheets/samplesheet-mismatched-ids.csv b/tests/data/samplesheets/samplesheet-mismatched-ids.csv index 632768d..6df7315 100644 --- a/tests/data/samplesheets/samplesheet-mismatched-ids.csv +++ b/tests/data/samplesheets/samplesheet-mismatched-ids.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 -sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 -sampleC,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 +sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 +sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 +sampleC,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 diff --git a/tests/data/samplesheets/samplesheet-no-metadata.csv b/tests/data/samplesheets/samplesheet-no-metadata.csv index 9d67864..5f4d2ca 100644 --- a/tests/data/samplesheets/samplesheet-no-metadata.csv +++ b/tests/data/samplesheets/samplesheet-no-metadata.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv b/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv index d5d42f0..cd0655d 100644 --- a/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv +++ b/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 -sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 +sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 +sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 diff --git a/tests/data/samplesheets/samplesheet-tabs.csv b/tests/data/samplesheets/samplesheet-tabs.csv index 56b4243..4408655 100644 --- a/tests/data/samplesheets/samplesheet-tabs.csv +++ b/tests/data/samplesheets/samplesheet-tabs.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,a b,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,,,,a b,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,,,,,,,,a b +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,a b,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,,,,a b,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,,,,,,,,a b diff --git a/tests/data/samplesheets/samplesheet1.csv b/tests/data/samplesheets/samplesheet1.csv index 3200344..f78cff3 100644 --- a/tests/data/samplesheets/samplesheet1.csv +++ b/tests/data/samplesheets/samplesheet1.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 From 2cb48888af92d86cca9bfba6b533ba3871d938f3 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 15 Aug 2024 14:58:57 -0400 Subject: [PATCH 04/25] Update paths to mlst files in samplesheet.csv --- assets/samplesheet.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/samplesheet.csv b/assets/samplesheet.csv index 7438d85..b283ed7 100644 --- a/assets/samplesheet.csv +++ b/assets/samplesheet.csv @@ -1,4 +1,4 @@ sample,mlst_alleles -sample1,/root/working_directory/nml-phac/gasclustering/tests/data/reports/sample1.mlst.json -sample2,/root/working_directory/nml-phac/gasclustering/tests/data/reports/sample2.mlst.json -sample3,/root/working_directory/nml-phac/gasclustering/tests/data/reports/sample3.mlst.json +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json From cc1444b9ee317a373e356e24cea0d7a70a9c3c1e Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 15 Aug 2024 16:01:20 -0400 Subject: [PATCH 05/25] Fixed error in samplesheet --- tests/data/samplesheets/samplesheet-hamming.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/samplesheets/samplesheet-hamming.csv b/tests/data/samplesheets/samplesheet-hamming.csv index 14f2469..0526692 100644 --- a/tests/data/samplesheets/samplesheet-hamming.csv +++ b/tests/data/samplesheets/samplesheet-hamming.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/update/input_assure/reports/case-hamming/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/update/input_assure/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, From 9a87a2b7670e52925fe11d4b703c710f006b810b Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Fri, 16 Aug 2024 11:01:30 -0400 Subject: [PATCH 06/25] Fixed typo --- tests/data/samplesheets/samplesheet-hamming.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/samplesheets/samplesheet-hamming.csv b/tests/data/samplesheets/samplesheet-hamming.csv index 0526692..df78ac4 100644 --- a/tests/data/samplesheets/samplesheet-hamming.csv +++ b/tests/data/samplesheets/samplesheet-hamming.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample2.mlst.subtyping.json,,,,,,,, sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, From 307aa8265567b94e0610b7690b58c6b01988beab Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 09:55:24 -0400 Subject: [PATCH 07/25] Updated container registry handling --- modules/local/arborview.nf | 3 ++- modules/local/gas/mcluster/main.nf | 2 +- modules/local/locidex/merge/main.nf | 5 +++-- modules/local/profile_dists/main.nf | 3 ++- nextflow.config | 3 +++ 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/local/arborview.nf b/modules/local/arborview.nf index 787d8af..5b74e12 100644 --- a/modules/local/arborview.nf +++ b/modules/local/arborview.nf @@ -10,7 +10,8 @@ process ARBOR_VIEW { container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? "docker.io/python:3.11.6" : - "docker.io/python:3.11.6" }" + task.ext.override_configured_container_registry != false ? 'docker.io/python:3.11.6' : + 'python:3.11.6' }" input: tuple path(tree), path(contextual_data) diff --git a/modules/local/gas/mcluster/main.nf b/modules/local/gas/mcluster/main.nf index 0ba2761..7d832bb 100644 --- a/modules/local/gas/mcluster/main.nf +++ b/modules/local/gas/mcluster/main.nf @@ -6,7 +6,7 @@ process GAS_MCLUSTER{ container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/genomic_address_service%3A0.1.1--pyh7cba7a3_1' : - 'quay.io/biocontainers/genomic_address_service:0.1.1--pyh7cba7a3_1' }" + 'biocontainers/genomic_address_service:0.1.1--pyh7cba7a3_1' }" input: path(dist_matrix) diff --git a/modules/local/locidex/merge/main.nf b/modules/local/locidex/merge/main.nf index 66ba9a7..d8ebb95 100644 --- a/modules/local/locidex/merge/main.nf +++ b/modules/local/locidex/merge/main.nf @@ -5,8 +5,9 @@ process LOCIDEX_MERGE { label 'process_medium' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'docker.io/mwells14/locidex:0.2.2' : - 'docker.io/mwells14/locidex:0.2.2' }" + "docker.io/mwells14/locidex:0.2.2" : + task.ext.override_configured_container_registry != false ? 'docker.io/mwells14/locidex:0.2.2' : + 'mwells14/locidex:0.2.2' }" input: path input_values // [file(sample1), file(sample2), file(sample3), etc...] diff --git a/modules/local/profile_dists/main.nf b/modules/local/profile_dists/main.nf index 734d38d..24b3472 100644 --- a/modules/local/profile_dists/main.nf +++ b/modules/local/profile_dists/main.nf @@ -4,7 +4,8 @@ process PROFILE_DISTS{ container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'docker.io/mwells14/gsp:arborator_1.0.0' : - 'docker.io/mwells14/gsp:arborator_1.0.0' }" + task.ext.override_configured_container_registry != false ? 'docker.io/mwells14/gsp:arborator_1.0.0' : + 'mwells14/gsp:arborator_1.0.0' }" input: path query diff --git a/nextflow.config b/nextflow.config index 1dad6fb..fbd25f5 100644 --- a/nextflow.config +++ b/nextflow.config @@ -178,6 +178,9 @@ docker.registry = 'quay.io' podman.registry = 'quay.io' singularity.registry = 'quay.io' +// Override the default Docker registry when required +process.ext.override_configured_container_registry = true + // Nextflow plugins plugins { id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet From 29d427fa7a2076e3119af7b29badcd110c993f18 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 11:39:41 -0400 Subject: [PATCH 08/25] add user UID/GID to Nextflow Docker run in CI to fix getpwuid() error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5abe80d..22bf0ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,4 +56,4 @@ jobs: - name: Nextflow run with test profile run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker -user $(id -u):$(id -g) From 09cfa2b3cb461c583498bccc13de0a10a457647d Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 12:27:27 -0400 Subject: [PATCH 09/25] add root user to Nextflow Docker run in CI to fix getpwuid() error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22bf0ad..6887f88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,4 +56,4 @@ jobs: - name: Nextflow run with test profile run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker -user $(id -u):$(id -g) + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker -user root From 2639645f103624bdfec1b9871bfaf9a15bf7ff1c Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 13:26:03 -0400 Subject: [PATCH 10/25] Updated container for arborview --- modules/local/arborview.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/local/arborview.nf b/modules/local/arborview.nf index 5b74e12..407468a 100644 --- a/modules/local/arborview.nf +++ b/modules/local/arborview.nf @@ -9,9 +9,8 @@ process ARBOR_VIEW { stageInMode 'copy' // Need to copy in arbor view html container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - "docker.io/python:3.11.6" : - task.ext.override_configured_container_registry != false ? 'docker.io/python:3.11.6' : - 'python:3.11.6' }" + 'https://depot.galaxyproject.org/singularity/python%3A3.12' : + 'biocontainers/python:3.12' }" input: tuple path(tree), path(contextual_data) From 726d2c968859cd87dfa3c697292821798972ccc0 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 13:41:41 -0400 Subject: [PATCH 11/25] Refactor user configuration for locidex module: Remove --user specs from ci.yml and add to modules.conf --- .github/workflows/ci.yml | 2 +- conf/modules.config | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6887f88..5abe80d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,4 +56,4 @@ jobs: - name: Nextflow run with test profile run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker -user root + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results diff --git a/conf/modules.config b/conf/modules.config index 9e7bba6..046e121 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -48,6 +48,7 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename.contains(File.separator) ? task.merged_prefix + filename.split(File.separator)[-1] : task.merged_prefix + filename } ] + containerOptions = '--user $(id -u):$(id -g)' } withName: PROFILE_DISTS { From 8a016b40a008b00053aa9facde782565fd4f86a1 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 14:01:03 -0400 Subject: [PATCH 12/25] Revert to previous user config in ci.yml --- .github/workflows/ci.yml | 2 +- conf/modules.config | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5abe80d..fa95efb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,4 +56,4 @@ jobs: - name: Nextflow run with test profile run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker --user $(id -u):$(id -g) diff --git a/conf/modules.config b/conf/modules.config index 046e121..9e7bba6 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -48,7 +48,6 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename.contains(File.separator) ? task.merged_prefix + filename.split(File.separator)[-1] : task.merged_prefix + filename } ] - containerOptions = '--user $(id -u):$(id -g)' } withName: PROFILE_DISTS { From eac46d87a1b7071f9e3884121546c39af3ca8a3b Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 14:09:12 -0400 Subject: [PATCH 13/25] Update --user in ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa95efb..6ba1c50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,4 +56,4 @@ jobs: - name: Nextflow run with test profile run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker --user $(id -u):$(id -g) + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker --user 1001 From db503bf0ed1facc10c4432e753b4624b3f4eb46f Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Mon, 19 Aug 2024 15:42:13 -0400 Subject: [PATCH 14/25] Removed --user configurations for locidex --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ba1c50..5abe80d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,4 +56,4 @@ jobs: - name: Nextflow run with test profile run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results -with-docker --user 1001 + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results From 1aebb70540eafbb86be70f215bcb9de1b8af7ae3 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Tue, 20 Aug 2024 09:53:06 -0400 Subject: [PATCH 15/25] Updated CHANGELOG.md --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5741b..45ccf8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] - 2024-08-20 + +### Changed + +- Upgraded `locidex/merge` to version `0.2.2` and updated `input_assure` and test data for compatibility with the new `mlst.json` allele file format. + - [PR28](https://github.com/phac-nml/gasclustering/pull/28) +- Aligned container registry handling in configuration files and modules with `phac-nml/pipeline-standards` + - [PR28](https://github.com/phac-nml/gasclustering/pull/28) + ## [0.2.0] - 2024-06-26 ### Added From f3f83cd89320fbacdb0c856bb784668883ed4dfb Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Tue, 20 Aug 2024 14:35:11 -0400 Subject: [PATCH 16/25] Updated release version --- CHANGELOG.md | 2 +- nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45ccf8f..1e39b34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - 2024-08-20 +## [0.3.0] - 2024-08-21 ### Changed diff --git a/nextflow.config b/nextflow.config index fbd25f5..a98e2dd 100644 --- a/nextflow.config +++ b/nextflow.config @@ -229,7 +229,7 @@ manifest { description = """IRIDA Next Genomic Address Service Clustering Pipeline""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '0.2.0' + version = '0.3.0' doi = '' defaultBranch = 'main' } From a23379c8b18557814b2bf69bbf8232e4df57d4b1 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Tue, 20 Aug 2024 14:52:41 -0400 Subject: [PATCH 17/25] Removed print debugging statement in input_assure.py --- bin/input_assure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/input_assure.py b/bin/input_assure.py index 7f5d56e..d4c2a1a 100755 --- a/bin/input_assure.py +++ b/bin/input_assure.py @@ -19,8 +19,6 @@ def check_inputs(json_file, sample_id, address, output_error_file, output_json_f with open_file(json_file, "rt") as f: json_data = json.load(f) - print(json.dumps(json_data, indent=4)) - # Extract the profile from the json_data profile = json_data.get("data", {}).get("profile", {}) # Check for multiple keys in the JSON file and define error message From 6f7f59d09714667e004dc55a70f6f7052ec5c7bd Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Tue, 20 Aug 2024 17:28:18 -0400 Subject: [PATCH 18/25] Replace unsorted list with sorted() for consistent ordering in input_assure.py --- bin/input_assure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/input_assure.py b/bin/input_assure.py index d4c2a1a..e2b7ac1 100755 --- a/bin/input_assure.py +++ b/bin/input_assure.py @@ -22,7 +22,7 @@ def check_inputs(json_file, sample_id, address, output_error_file, output_json_f # Extract the profile from the json_data profile = json_data.get("data", {}).get("profile", {}) # Check for multiple keys in the JSON file and define error message - keys = list(profile.keys()) + keys = sorted(profile.keys()) original_key = keys[0] if keys else None # Define a variable to store the match_status (True or False) From f787028216a2b1bbb8f15b32ea8d89c51399da20 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Wed, 21 Aug 2024 11:55:57 -0400 Subject: [PATCH 19/25] Update locidex to version 0.2.3 --- modules/local/locidex/merge/main.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/local/locidex/merge/main.nf b/modules/local/locidex/merge/main.nf index d8ebb95..103c24f 100644 --- a/modules/local/locidex/merge/main.nf +++ b/modules/local/locidex/merge/main.nf @@ -5,9 +5,9 @@ process LOCIDEX_MERGE { label 'process_medium' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - "docker.io/mwells14/locidex:0.2.2" : - task.ext.override_configured_container_registry != false ? 'docker.io/mwells14/locidex:0.2.2' : - 'mwells14/locidex:0.2.2' }" + "docker.io/mwells14/locidex:0.2.3" : + task.ext.override_configured_container_registry != false ? 'docker.io/mwells14/locidex:0.2.3' : + 'mwells14/locidex:0.2.3' }" input: path input_values // [file(sample1), file(sample2), file(sample3), etc...] From cf5248e00d7514936f2e1e0f96750577200c6c26 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Wed, 21 Aug 2024 16:50:51 -0400 Subject: [PATCH 20/25] Add link to release tag --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e39b34..79ef4ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,3 +37,4 @@ Initial release of the Genomic Address Service Clustering pipeline to be used fo [0.1.0]: https://github.com/phac-nml/gasclustering/releases/tag/0.1.0 [0.2.0]: https://github.com/phac-nml/gasclustering/releases/tag/0.2.0 +[0.3.0]: https://github.com/phac-nml/gasclustering/releases/tag/0.3.0 From e653503ef3f8cde25bdd8ac2c54475687b951e97 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 5 Sep 2024 10:39:51 -0400 Subject: [PATCH 21/25] Update CHANGELOG --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ef4ee..e8e400a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,15 +3,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.0] - 2024-08-21 +## [0.3.0] - 2024-09-05 ### Changed -- Upgraded `locidex/merge` to version `0.2.2` and updated `input_assure` and test data for compatibility with the new `mlst.json` allele file format. +- Upgraded `locidex/merge` to version `0.2.3` and updated `input_assure` and test data for compatibility with the new `mlst.json` allele file format. - [PR28](https://github.com/phac-nml/gasclustering/pull/28) - Aligned container registry handling in configuration files and modules with `phac-nml/pipeline-standards` - [PR28](https://github.com/phac-nml/gasclustering/pull/28) +This pipeline is now compatible only with output generated by [Locidex v0.2.3+](https://github.com/phac-nml/locidex) and [Mikrokondo v0.4.0+](https://github.com/phac-nml/mikrokondo/releases/tag/v0.4.0). + ## [0.2.0] - 2024-06-26 ### Added From dc32ebb45927b917dbe5b5cddb3fcca0750a2077 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 5 Sep 2024 10:40:15 -0400 Subject: [PATCH 22/25] Update test data file paths in samplesheets --- tests/data/samplesheets/samplesheet-hamming.csv | 6 +++--- tests/data/samplesheets/samplesheet-hash-missing.csv | 6 +++--- tests/data/samplesheets/samplesheet-hash-more-missing.csv | 6 +++--- tests/data/samplesheets/samplesheet-little-metadata.csv | 6 +++--- tests/data/samplesheets/samplesheet-mismatched-ids.csv | 6 +++--- tests/data/samplesheets/samplesheet-no-metadata.csv | 6 +++--- .../samplesheets/samplesheet-partial-mismatched-ids.csv | 6 +++--- tests/data/samplesheets/samplesheet-tabs.csv | 6 +++--- tests/data/samplesheets/samplesheet1.csv | 6 +++--- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/data/samplesheets/samplesheet-hamming.csv b/tests/data/samplesheets/samplesheet-hamming.csv index df78ac4..d18a69c 100644 --- a/tests/data/samplesheets/samplesheet-hamming.csv +++ b/tests/data/samplesheets/samplesheet-hamming.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hamming/sample1.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hamming/sample2.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hamming/sample3.mlst.subtyping.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-hash-missing.csv b/tests/data/samplesheets/samplesheet-hash-missing.csv index 6ad5929..9355c3d 100644 --- a/tests/data/samplesheets/samplesheet-hash-missing.csv +++ b/tests/data/samplesheets/samplesheet-hash-missing.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample3.mlst.subtyping.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-hash-more-missing.csv b/tests/data/samplesheets/samplesheet-hash-more-missing.csv index 37f051f..4ee53c9 100644 --- a/tests/data/samplesheets/samplesheet-hash-more-missing.csv +++ b/tests/data/samplesheets/samplesheet-hash-more-missing.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample1.mlst.subtyping.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample2.mlst.subtyping.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/case-hash-missing/sample3-more-missing.mlst.subtyping.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-little-metadata.csv b/tests/data/samplesheets/samplesheet-little-metadata.csv index 052b4b3..3e721de 100644 --- a/tests/data/samplesheets/samplesheet-little-metadata.csv +++ b/tests/data/samplesheets/samplesheet-little-metadata.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,,,,1.4,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,,,,,,3.8 +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,,,,1.4,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,,,,,,3.8 diff --git a/tests/data/samplesheets/samplesheet-mismatched-ids.csv b/tests/data/samplesheets/samplesheet-mismatched-ids.csv index 6df7315..632768d 100644 --- a/tests/data/samplesheets/samplesheet-mismatched-ids.csv +++ b/tests/data/samplesheets/samplesheet-mismatched-ids.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 -sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 -sampleC,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 +sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 +sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 +sampleC,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 diff --git a/tests/data/samplesheets/samplesheet-no-metadata.csv b/tests/data/samplesheets/samplesheet-no-metadata.csv index 5f4d2ca..9d67864 100644 --- a/tests/data/samplesheets/samplesheet-no-metadata.csv +++ b/tests/data/samplesheets/samplesheet-no-metadata.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,,,,,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,,,,,,,, +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,,,,,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,,,,,,,, diff --git a/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv b/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv index cd0655d..d5d42f0 100644 --- a/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv +++ b/tests/data/samplesheets/samplesheet-partial-mismatched-ids.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 -sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 +sampleA,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 +sampleB,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 diff --git a/tests/data/samplesheets/samplesheet-tabs.csv b/tests/data/samplesheets/samplesheet-tabs.csv index 4408655..56b4243 100644 --- a/tests/data/samplesheets/samplesheet-tabs.csv +++ b/tests/data/samplesheets/samplesheet-tabs.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,a b,,,,,,, -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,,,,a b,,,, -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,,,,,,,,a b +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,a b,,,,,,, +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,,,,a b,,,, +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,,,,,,,,a b diff --git a/tests/data/samplesheets/samplesheet1.csv b/tests/data/samplesheets/samplesheet1.csv index f78cff3..3200344 100644 --- a/tests/data/samplesheets/samplesheet1.csv +++ b/tests/data/samplesheets/samplesheet1.csv @@ -1,4 +1,4 @@ sample,mlst_alleles,metadata_1,metadata_2,metadata_3,metadata_4,metadata_5,metadata_6,metadata_7,metadata_8 -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8 +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json,2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8 +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8 From b60e1d3bfbf62cab71191defc64b851db9058e0e Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Thu, 5 Sep 2024 12:47:58 -0400 Subject: [PATCH 23/25] update assest/samplesheet.csv file paths --- assets/samplesheet.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/samplesheet.csv b/assets/samplesheet.csv index b283ed7..82842ce 100644 --- a/assets/samplesheet.csv +++ b/assets/samplesheet.csv @@ -1,4 +1,4 @@ sample,mlst_alleles -sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample1.mlst.json -sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample2.mlst.json -sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/update/input_assure/tests/data/reports/sample3.mlst.json +sample1,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample1.mlst.json +sample2,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample2.mlst.json +sample3,https://raw.githubusercontent.com/phac-nml/gasclustering/dev/tests/data/reports/sample3.mlst.json From 0071ab8dacf22bb36fa0022fdcf84356d20f9866 Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Tue, 10 Sep 2024 11:46:53 -0400 Subject: [PATCH 24/25] Update profile_dist container --- modules/local/profile_dists/main.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/local/profile_dists/main.nf b/modules/local/profile_dists/main.nf index 24b3472..d316e78 100644 --- a/modules/local/profile_dists/main.nf +++ b/modules/local/profile_dists/main.nf @@ -3,9 +3,9 @@ process PROFILE_DISTS{ tag "Pairwise Distance Generation" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'docker.io/mwells14/gsp:arborator_1.0.0' : - task.ext.override_configured_container_registry != false ? 'docker.io/mwells14/gsp:arborator_1.0.0' : - 'mwells14/gsp:arborator_1.0.0' }" + 'docker.io/mwells14/profile_dists:1.0.2' : + task.ext.override_configured_container_registry != false ? 'docker.io/mwells14/profile_dists:1.0.2' : + 'mwells14/profile_dists:1.0.2' }" input: path query From 0c358620a2283334247a03189c20fd1dd5b3e36a Mon Sep 17 00:00:00 2001 From: kylacochrane Date: Tue, 10 Sep 2024 11:50:42 -0400 Subject: [PATCH 25/25] Update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8e400a..9ad7753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.0] - 2024-09-05 +## [0.3.0] - 2024-09-10 ### Changed +- Upgraded `profile_dist` container to version `1.0.2` - Upgraded `locidex/merge` to version `0.2.3` and updated `input_assure` and test data for compatibility with the new `mlst.json` allele file format. - [PR28](https://github.com/phac-nml/gasclustering/pull/28) - Aligned container registry handling in configuration files and modules with `phac-nml/pipeline-standards`