Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Creating log files #76

Merged
merged 6 commits into from
Jul 10, 2024
Merged

Creating log files #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
199db57
copy log files and config files to new dir [skip ci]
rnmitchell Jul 8, 2024
2e84c3e
update test [skip ci]
rnmitchell Jul 8, 2024
23e1cd5
Merge branch 'master' into logs
rnmitchell Jul 8, 2024
3133bae
updated tests
rnmitchell Jul 8, 2024
49be971
updated snp workflow to include log [skip ci]
rnmitchell Jul 8, 2024
1ab9ea1
added test for snps log
rnmitchell Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions lusSTR/tests/test_filters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def test_EFMoutput_format(outputdir, datatype, software, tmp_path):
"--str-type",
datatype,
"--input",
"WD",
str(inputfile),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand the significance of this change throughout this test script.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was erring out with just using the WD. Really, the input doesn't matter because it's skipping the initial convert step anyway for all of these tests.

]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "test_output.csv"))
Expand All @@ -208,7 +208,7 @@ def test_STRmixoutput_format(outputdir, datatype, tmp_path):
"-w",
str_path,
"--input",
"WD",
str(inputfile),
"-o",
"STRmix_Files",
"--str-type",
Expand All @@ -230,7 +230,16 @@ def test_STRmixoutput_customranges(tmp_path):
exp_info_out = data_file("NGS_stutter_test/custom/test_stutter_sequence_info.csv")
obs_out = str(tmp_path / f"WD/test_stutter/Sample1_evidence_ngs.csv")
obs_info_out = str(tmp_path / f"WD/test_stutter/test_stutter_sequence_info.csv")
arglist = ["config", "-w", str_path, "--input", "WD", "-o", "test_stutter", "--custom"]
arglist = [
"config",
"-w",
str_path,
"--input",
str(inputfile),
"-o",
"test_stutter",
"--custom",
]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "test_stutter.csv"))
shutil.copyfile(inputfile, os.path.join(str_path, "test_stutter.txt"))
Expand All @@ -247,7 +256,7 @@ def test_nofilters(tmp_path):
inputfile = data_file("test_stutter.txt")
exp_out = data_file("NGS_stutter_test/Sample1_nofilter.csv")
obs_out = str(tmp_path / "WD/lusstr_output/Sample1_evidence_ngs.csv")
arglist = ["config", "-w", str_path, "--input", "WD", "--nofilter"]
arglist = ["config", "-w", str_path, "--input", str(inputfile), "--nofilter"]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "lusstr_output.csv"))
shutil.copyfile(inputfile, os.path.join(str_path, "lusstr_output.txt"))
Expand All @@ -261,7 +270,7 @@ def test_flags(tmp_path):
inputfile = data_file("test_stutter.txt")
exp_out = data_file("RU_stutter_test/Flagged_Loci.csv")
obs_out = str(tmp_path / "WD/lusstr_output/lusstr_output_Flagged_Loci.csv")
arglist = ["config", "-w", str_path, "--input", "WD"]
arglist = ["config", "-w", str_path, "--input", str(inputfile)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "lusstr_output.csv"))
shutil.copyfile(inputfile, os.path.join(str_path, "lusstr_output.txt"))
Expand All @@ -288,7 +297,7 @@ def test_efm_reference(outputdir, datatype, software, tmp_path):
"-w",
str_path,
"--input",
"WD",
str(inputfile),
"--software",
software,
"--reference",
Expand Down Expand Up @@ -316,7 +325,7 @@ def test_strmix_reference(outputdir, datatype, tmp_path):
"-w",
str_path,
"--input",
"WD",
str(inputfile),
"-o",
"STRmix_Files",
"--reference",
Expand All @@ -336,7 +345,7 @@ def test_D7(tmp_path):
inputfile = data_file("test_D7.txt")
exp_out = data_file("D7_microvariant_flagged.csv")
obs_out = str(tmp_path / "WD/test/test_Flagged_Loci.csv")
arglist = ["config", "-w", str_path, "--input", "WD", "-o", "test"]
arglist = ["config", "-w", str_path, "--input", str(inputfile), "-o", "test"]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "test.csv"))
shutil.copyfile(inputfile, os.path.join(str_path, "test.txt"))
Expand Down Expand Up @@ -392,7 +401,7 @@ def test_forward_strand_orientation(tmp_path):
"-w",
str_path,
"--input",
"WD",
str(inputfile),
"-o",
"STRmix_Files",
"--strand",
Expand All @@ -416,7 +425,7 @@ def test_lusplus_sequence_info(tmp_path):
"-w",
str_path,
"--input",
"WD",
str(inputfile),
"-o",
"LUSPlus",
"--strand",
Expand Down
24 changes: 24 additions & 0 deletions lusSTR/tests/test_snps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,3 +260,27 @@ def test_uas_version5(tmp_path):
all_arglist = ["snps", "all", "-w", str(tmp_path)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(all_arglist))
assert filecmp.cmp(exp_out, obs_out) is True


def test_snp_logs(tmp_path):
inputdb = data_file("snps")
arglist = [
"config",
"-w",
str(tmp_path),
"-o",
"sr",
"--input",
str(inputdb),
"--snps",
"--analysis-software",
"straitrazor",
]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
format_arglist = ["snps", "format", "-w", str(tmp_path)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(format_arglist))
for dir in os.listdir(str(tmp_path / "logs/")):
assert os.path.exists(str(tmp_path / f"logs/{dir}/snp_config.yaml"))
assert os.path.exists(
str(tmp_path / f"logs/{dir}/input/Positive Control Sample Details Report 2315.xlsx")
)
30 changes: 22 additions & 8 deletions lusSTR/tests/test_suite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def test_convert_full_nocombine(
"-o",
infile,
"--input",
"WD",
str(inputfile),
]
else:
arglist = [
Expand All @@ -78,7 +78,7 @@ def test_convert_full_nocombine(
"-o",
infile,
"--input",
"WD",
str(inputfile),
"--powerseq",
]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
Expand Down Expand Up @@ -110,7 +110,7 @@ def test_flanks(tmp_path):
"--analysis-software",
"straitrazor",
"--input",
"WD",
str(inputfile),
]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "testflanks.csv"))
Expand All @@ -135,7 +135,7 @@ def test_convert_combine(input, exp_length, tmp_path):
"--analysis-software",
"straitrazor",
"--input",
"WD",
str(inputfile),
]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile, os.path.join(str_path, "testflanks.csv"))
Expand Down Expand Up @@ -183,7 +183,7 @@ def test_powerseq_flanks(tmp_path):
"--analysis-software",
"straitrazor",
"--input",
"WD",
str(inputfile),
"--powerseq",
]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
Expand All @@ -199,7 +199,7 @@ def test_convert_uas_sexloci(tmp_path):
inputfile_sex = data_file("testformat_uas_sexloci.csv")
exp_sex_out = data_file("testformat_uas_sexloci.txt")
obs_sex_out = str(tmp_path / "WD/testformatuas_sexloci.txt")
arglist = ["config", "-w", str_path, "-o", "testformatuas", "--sex", "--input", "WD"]
arglist = ["config", "-w", str_path, "-o", "testformatuas", "--sex", "--input", str(inputfile)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
shutil.copyfile(inputfile_sex, os.path.join(str_path, "testformatuas_sexloci.csv"))
shutil.copyfile(inputfile, os.path.join(str_path, "testformatuas.csv"))
Expand Down Expand Up @@ -242,7 +242,7 @@ def test_convert_sr_sexloci(input, testoutput, flank_output, kit, tmp_path):
"testformatsr",
"--sex",
"--input",
"WD",
str(inputfile),
"--analysis-software",
"straitrazor",
]
Expand All @@ -255,7 +255,7 @@ def test_convert_sr_sexloci(input, testoutput, flank_output, kit, tmp_path):
"testformatsr",
"--sex",
"--input",
"WD",
str(inputfile),
"--analysis-software",
"straitrazor",
"--powerseq",
Expand Down Expand Up @@ -380,3 +380,17 @@ def test_deletions(tmp_path):
snakemake_arglist = ["strs", "convert", "-w", str(tmp_path)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(snakemake_arglist))
assert filecmp.cmp(exp_output, obs_output) is True


def test_log(tmp_path):
config = str(tmp_path / "config.yaml")
inputfile = data_file("UAS_bulk_input/Positive Control Sample Details Report 2315.xlsx")
arglist = ["config", "-w", str(tmp_path), "--input", str(inputfile)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(arglist))
snakemake_arglist = ["strs", "format", "-w", str(tmp_path)]
lusSTR.cli.main(lusSTR.cli.get_parser().parse_args(snakemake_arglist))
for dir in os.listdir(str(tmp_path / "logs/")):
assert os.path.exists(str(tmp_path / f"logs/{dir}/config.yaml"))
assert os.path.exists(
str(tmp_path / f"logs/{dir}/input/Positive Control Sample Details Report 2315.xlsx")
)
20 changes: 20 additions & 0 deletions lusSTR/workflows/snps.smk
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from datetime import datetime
import lusSTR


Expand All @@ -14,6 +15,18 @@ def format_filename(id, refs):
return f"{id}_snp_reference"


def create_log(log):
now = datetime.now()
dt = now.strftime("%m%d%Y_%H_%M_%S")
shell("mkdir -p logs/{dt}/input/")
shell("cp {log} logs/{dt}/strs.log")
if os.path.isdir(input_name):
shell("cp {input_name}/* logs/{dt}/input/")
else:
shell("cp '{input_name}' logs/{dt}/input/")
shell("cp snp_config.yaml logs/{dt}/")
Comment on lines +18 to +27
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the log argument to this function a path to the Snakemake logfile in $WD/.snakemake/?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes it must be built in.



rule all:
input:
expand("{name}.txt", name=output_name),
Expand Down Expand Up @@ -49,3 +62,10 @@ rule convert:
thresh=config["thresh"]
script:
lusSTR.wrapper("snps_convert")


onsuccess:
create_log(log)

onerror:
create_log(log)
22 changes: 21 additions & 1 deletion lusSTR/workflows/strs.smk
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from datetime import datetime
import glob
import lusSTR
import openpyxl
Expand Down Expand Up @@ -72,6 +73,18 @@ def parse_sample_details(filename):
return sampleID


def create_log(log):
now = datetime.now()
dt = now.strftime("%m%d%Y_%H_%M_%S")
shell("mkdir -p logs/{dt}/input/")
shell("cp {log} logs/{dt}/strs.log")
if os.path.isdir(input_name):
shell("cp {input_name}/* logs/{dt}/input/")
else:
shell("cp '{input_name}' logs/{dt}/input/")
shell("cp config.yaml logs/{dt}/")


def get_output():
if custom:
outname = expand("{name}_custom_range.txt", name=output_name)
Expand Down Expand Up @@ -115,7 +128,7 @@ rule convert:
kit=config["kit"],
custom=config["custom_ranges"]
script:
lusSTR.wrapper("convert")
lusSTR.wrapper("convert")


rule filter:
Expand All @@ -139,4 +152,11 @@ rule filter:
custom=config["custom_ranges"]
script:
lusSTR.wrapper("filter")


onsuccess:
create_log(log)

onerror:
create_log(log)

Loading