From 4a9158ab32c696333c3cf60e63dd37e6cbc69ddc Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Wed, 22 May 2024 20:52:12 +0200 Subject: [PATCH 01/11] Extract function get_abs_path_list --- oemof_b3/tools/testing_pipeline.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index 16a18591..bf2a8615 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -119,22 +119,16 @@ def get_abs_path_list(output_rule_list): Outputs ------- - absolute_path_list : str - Absolute file path + None """ - # Loop over each rule which is tested in the snakemake pipeline - for sublist in output_rule_list: - # Get absolute path - absolute_path_list = [os.path.join(os.getcwd(), entry) for entry in sublist] - - return absolute_path_list + # Get absolute path of rule + return [os.path.abspath(entry) for entry in output_rule_list] def pipeline_file_output_test(delete_switch, output_rule_list): # Loop over each rule which is tested in the snakemake pipeline for sublist in output_rule_list: - # Get absolute path - absolute_path_list = [os.path.join(os.getcwd(), entry) for entry in sublist] + absolute_path_list = get_abs_path_list(sublist) renamed_path = [] for raw_file_path in absolute_path_list: From 8a989087b105d34e39bdae16807a94e22c12783b Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Wed, 22 May 2024 22:52:52 +0200 Subject: [PATCH 02/11] Add function file_name_extension --- oemof_b3/tools/testing_pipeline.py | 46 ++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index bf2a8615..49d4831e 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -107,6 +107,7 @@ def remove_test_data(path): if os.path.isfile(path): os.remove(path) + def get_abs_path_list(output_rule_list): """ This function finds the absolut file path for each rule @@ -125,22 +126,45 @@ def get_abs_path_list(output_rule_list): # Get absolute path of rule return [os.path.abspath(entry) for entry in output_rule_list] + +def file_name_extension(raw_file_path): + """ + This function rearranges the current absolute file path + with the new extension suffix '_original'. + + Inputs + ------- + raw_file_path : str + Absolute path of rule + + Outputs + ------- + renamed_path : str + + """ + renamed_path = [] + + # Get file extension + file_extension = raw_file_path[raw_file_path.rfind(".") + 1:] + # Rename existing user data + renamed_file = rename_path( + raw_file_path, "." + file_extension, "_original." + file_extension + ) + renamed_path.append(renamed_file) + + return renamed_path + + def pipeline_file_output_test(delete_switch, output_rule_list): # Loop over each rule which is tested in the snakemake pipeline for sublist in output_rule_list: + # Get absolute path of sublist absolute_path_list = get_abs_path_list(sublist) - renamed_path = [] for raw_file_path in absolute_path_list: - if os.path.isfile(raw_file_path): - # Get file extension - file_extension = raw_file_path[raw_file_path.rfind(".") + 1 :] - # Rename existing user data - renamed_file = rename_path( - raw_file_path, "." + file_extension, "_original." + file_extension - ) - renamed_path.append(renamed_file) + # Rename file with extension original + renamed_path = file_name_extension(raw_file_path) try: # Run the snakemake rule in this loop @@ -149,9 +173,6 @@ def pipeline_file_output_test(delete_switch, output_rule_list): snakefile="Snakefile", ) - # Check if snakemake rule exited without error (true) - assert output - # Check if the output file was created for raw_file_path in absolute_path_list: assert os.path.exists(raw_file_path) @@ -165,6 +186,7 @@ def pipeline_file_output_test(delete_switch, output_rule_list): # If file had to be renamed revert the changes for renamed_file in renamed_path: if os.path.isfile(renamed_file): + file_extension = renamed_file[renamed_file.rfind(".") + 1:] rename_path( renamed_file, "_original." + file_extension, From 2c1775ee5ce1ee0e95fac4f94e5cc7174f06788a Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 16:20:42 +0200 Subject: [PATCH 03/11] Add functions clean_file and rule_test --- oemof_b3/tools/testing_pipeline.py | 104 +++++++++++++++++++---------- 1 file changed, 70 insertions(+), 34 deletions(-) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index 49d4831e..288352af 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -1,6 +1,7 @@ import os import subprocess import snakemake +import logging def install_with_extra(extra): @@ -155,6 +156,69 @@ def file_name_extension(raw_file_path): return renamed_path +def rule_test(sublist): + """ + This function runs the rule from the output rule sublist. + + Inputs + ------- + sublist : str + Path of rule + + Outputs + ------- + None + + """ + # Run the snakemake rule in this loop + output = snakemake.snakemake( + targets=sublist, + snakefile="Snakefile", + ) + + # Check if snakemake rule exited without error (true) + assert output, f"Snakemake rule failed for targets: {sublist}" + + # Log the success + logging.info(f"Snakemake rule executed successfully for targets: {sublist}") + + +def clean_file(sublist, delete_switch, renamed_path): + """ + This function removes test data files and reverts renamed files. + + Inputs + ------- + sublist : list of str + List of target paths of rules + delete_switch : bool + If True, delete the data created during the test run. + If False, do not delete the data. + renamed_path : list of str + List of renamed target paths + + Outputs + ------- + None + + """ + # Remove the file created for this test + for raw_file_path in sublist: + if os.path.exists(raw_file_path): + if delete_switch or renamed_path: + remove_test_data(raw_file_path) + + # If file had to be renamed revert the changes + for renamed_file in renamed_path: + if os.path.isfile(renamed_file): + file_extension = renamed_file[renamed_file.rfind(".") + 1:] + rename_path( + renamed_file, + "_original." + file_extension, + "." + file_extension, + ) + + def pipeline_file_output_test(delete_switch, output_rule_list): # Loop over each rule which is tested in the snakemake pipeline for sublist in output_rule_list: @@ -167,47 +231,19 @@ def pipeline_file_output_test(delete_switch, output_rule_list): renamed_path = file_name_extension(raw_file_path) try: - # Run the snakemake rule in this loop - output = snakemake.snakemake( - targets=sublist, - snakefile="Snakefile", - ) + # Run the snakemake rule + rule_test(sublist) # Check if the output file was created for raw_file_path in absolute_path_list: assert os.path.exists(raw_file_path) - for raw_file_path in sublist: - # Remove the file created for this test - if os.path.exists(raw_file_path): - if delete_switch or renamed_path: - remove_test_data(raw_file_path) - - # If file had to be renamed revert the changes - for renamed_file in renamed_path: - if os.path.isfile(renamed_file): - file_extension = renamed_file[renamed_file.rfind(".") + 1:] - rename_path( - renamed_file, - "_original." + file_extension, - "." + file_extension, - ) + # Revert file changes + clean_file(sublist, delete_switch, renamed_path) except BaseException: - # Revert changes - for remove_raw_file_path in sublist: - # Remove the file created for this test - if os.path.exists(remove_raw_file_path): - remove_test_data(remove_raw_file_path) - - # If file had to be renamed revert the changes - for renamed_file in renamed_path: - if os.path.isfile(renamed_file): - rename_path( - renamed_file, - "_original." + file_extension, - "." + file_extension, - ) + # Revert file changes + clean_file(sublist, delete_switch, renamed_path) raise AssertionError( f"The workflow {raw_file_path} could not be executed correctly. " From 43455c945f1932e49fbe64f709d17e081ed0b464 Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 16:30:55 +0200 Subject: [PATCH 04/11] Add docstring and comment to function pipeline_file_output_test --- oemof_b3/tools/testing_pipeline.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index 288352af..ebb7be64 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -220,12 +220,32 @@ def clean_file(sublist, delete_switch, renamed_path): def pipeline_file_output_test(delete_switch, output_rule_list): + """ + This function tests the Snakemake pipeline for a list of output rules + and reverts all changes made in the target directory. + + Inputs + ------- + delete_switch : bool + If True, delete the data created during the test run. + If False, do not delete the data. + output_rule_list : list of str + Nested list with sublist containing paths to target files + associated with a specific rule. + + + Outputs + ------- + None + + """ # Loop over each rule which is tested in the snakemake pipeline for sublist in output_rule_list: # Get absolute path of sublist absolute_path_list = get_abs_path_list(sublist) for raw_file_path in absolute_path_list: + # Check if file already exists in directory if os.path.isfile(raw_file_path): # Rename file with extension original renamed_path = file_name_extension(raw_file_path) From 53c5a5a09de2ce2e6347e81165445888d13c437e Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 16:36:02 +0200 Subject: [PATCH 05/11] Add and correct docstring --- oemof_b3/tools/testing_pipeline.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index ebb7be64..5892a4a7 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -105,6 +105,19 @@ def rename_path(file_path, before, after): def remove_test_data(path): + """ + This function removes test data. + + Inputs + ------- + path : str + Path of test data + + Outputs + ------- + None + + """ if os.path.isfile(path): os.remove(path) @@ -233,7 +246,6 @@ def pipeline_file_output_test(delete_switch, output_rule_list): Nested list with sublist containing paths to target files associated with a specific rule. - Outputs ------- None From 573ae4222c1c85142715c2515c932bebde01dbe7 Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 18:34:22 +0200 Subject: [PATCH 06/11] Add docstring to test_pipeline_resources_tables.py --- tests/test_pipeline_resources_tables.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_pipeline_resources_tables.py b/tests/test_pipeline_resources_tables.py index 07e041d0..df5ba9e1 100644 --- a/tests/test_pipeline_resources_tables.py +++ b/tests/test_pipeline_resources_tables.py @@ -1,3 +1,7 @@ +""" +This script checks the snakemake pipeline for target rules in folder _resources +and _tables in the results' folder. +""" import os from oemof_b3.tools.testing_pipeline import ( get_repo_path, From c0cafc572b0dff6df00bd3cad301c6fa742bf288 Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 18:34:54 +0200 Subject: [PATCH 07/11] Adjust function file_name_extension --- oemof_b3/tools/testing_pipeline.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index 5892a4a7..75a43a02 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -156,17 +156,14 @@ def file_name_extension(raw_file_path): renamed_path : str """ - renamed_path = [] - # Get file extension file_extension = raw_file_path[raw_file_path.rfind(".") + 1:] # Rename existing user data renamed_file = rename_path( raw_file_path, "." + file_extension, "_original." + file_extension ) - renamed_path.append(renamed_file) - return renamed_path + return renamed_file def rule_test(sublist): @@ -221,7 +218,7 @@ def clean_file(sublist, delete_switch, renamed_path): if delete_switch or renamed_path: remove_test_data(raw_file_path) - # If file had to be renamed revert the changes + # If file had to be renamed revert the changes for renamed_file in renamed_path: if os.path.isfile(renamed_file): file_extension = renamed_file[renamed_file.rfind(".") + 1:] @@ -256,11 +253,13 @@ def pipeline_file_output_test(delete_switch, output_rule_list): # Get absolute path of sublist absolute_path_list = get_abs_path_list(sublist) + renamed_path = [] for raw_file_path in absolute_path_list: # Check if file already exists in directory if os.path.isfile(raw_file_path): # Rename file with extension original - renamed_path = file_name_extension(raw_file_path) + renamed_file = file_name_extension(raw_file_path) + renamed_path.append(renamed_file) try: # Run the snakemake rule From 19c2a4045105626d07bc0b4b918a06597ea79802 Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 18:37:01 +0200 Subject: [PATCH 08/11] Add docstring to test_pipeline_raw_data.py --- tests/test_pipeline_raw_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_pipeline_raw_data.py b/tests/test_pipeline_raw_data.py index fa7ba2fd..0e2564e8 100644 --- a/tests/test_pipeline_raw_data.py +++ b/tests/test_pipeline_raw_data.py @@ -1,3 +1,7 @@ +""" +This script checks the snakemake pipeline for target rules that create +empty time series and scalars in the folder raw. +""" import os import snakemake import shutil From eb1b1a23dcb082c021c5aa365722ff1434514f5b Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Fri, 24 May 2024 18:39:06 +0200 Subject: [PATCH 09/11] Add docstring for testing_pipeline.py --- oemof_b3/tools/testing_pipeline.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index 75a43a02..d4eeeeaa 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -1,3 +1,7 @@ +""" +This script contains functions to test the files and folders created +through the snakemake pipeline. +""" import os import subprocess import snakemake From d3826d24a7e7922b3c89d76ec8308200832eea24 Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Wed, 29 May 2024 22:36:19 +0200 Subject: [PATCH 10/11] Apply black --- docs/conf.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4717e368..1d05a53d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,43 +12,44 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../')) -sys.path.insert(0, os.path.abspath('../scripts')) -sys.path.insert(0, os.path.abspath('../oemof_b3/config/')) + +sys.path.insert(0, os.path.abspath("../")) +sys.path.insert(0, os.path.abspath("../scripts")) +sys.path.insert(0, os.path.abspath("../oemof_b3/config/")) # -- Project information ----------------------------------------------------- -project = 'oemof-B3' -copyright = '2020, Reiner Lemoine Institut' -author = 'Reiner Lemoine Institut' +project = "oemof-B3" +copyright = "2020, Reiner Lemoine Institut" +author = "Reiner Lemoine Institut" # The full version, including alpha/beta/rc tags -release = '0.0.5dev' +release = "0.0.5dev" # -- General configuration --------------------------------------------------- -master_doc = 'index' +master_doc = "index" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.napoleon', - 'sphinxcontrib.bibtex', - 'sphinx.ext.autosectionlabel' + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinxcontrib.bibtex", + "sphinx.ext.autosectionlabel", ] # specify bibfiles for sphinxcontrib.bibtex -bibtex_bibfiles = ['bibliography.bib'] +bibtex_bibfiles = ["bibliography.bib"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- @@ -65,7 +66,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- Options for Sphinx autodoc ---------------------------------------------- From 861cf09f5181941e6601812d6153611f333c9a6f Mon Sep 17 00:00:00 2001 From: Stefanie Nguyen Date: Wed, 29 May 2024 22:37:04 +0200 Subject: [PATCH 11/11] Apply black and update function pipeline_file_output_test --- oemof_b3/tools/testing_pipeline.py | 64 +++++++++++++++++++----------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index d4eeeeaa..17de6748 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -147,21 +147,21 @@ def get_abs_path_list(output_rule_list): def file_name_extension(raw_file_path): """ - This function rearranges the current absolute file path - with the new extension suffix '_original'. + This function rearranges the current absolute file path + with the new extension suffix '_original'. - Inputs - ------- - raw_file_path : str - Absolute path of rule + Inputs + ------- + raw_file_path : str + Absolute path of rule - Outputs - ------- - renamed_path : str + Outputs + ------- + renamed_path : str """ # Get file extension - file_extension = raw_file_path[raw_file_path.rfind(".") + 1:] + file_extension = raw_file_path[raw_file_path.rfind(".") + 1 :] # Rename existing user data renamed_file = rename_path( raw_file_path, "." + file_extension, "_original." + file_extension @@ -172,16 +172,16 @@ def file_name_extension(raw_file_path): def rule_test(sublist): """ - This function runs the rule from the output rule sublist. + This function runs the rule from the output rule sublist. - Inputs - ------- - sublist : str - Path of rule + Inputs + ------- + sublist : str + Path of rule - Outputs - ------- - None + Outputs + ------- + None """ # Run the snakemake rule in this loop @@ -225,7 +225,7 @@ def clean_file(sublist, delete_switch, renamed_path): # If file had to be renamed revert the changes for renamed_file in renamed_path: if os.path.isfile(renamed_file): - file_extension = renamed_file[renamed_file.rfind(".") + 1:] + file_extension = renamed_file[renamed_file.rfind(".") + 1 :] rename_path( renamed_file, "_original." + file_extension, @@ -259,11 +259,27 @@ def pipeline_file_output_test(delete_switch, output_rule_list): renamed_path = [] for raw_file_path in absolute_path_list: - # Check if file already exists in directory - if os.path.isfile(raw_file_path): - # Rename file with extension original - renamed_file = file_name_extension(raw_file_path) - renamed_path.append(renamed_file) + try: + # Check if file already exists in directory + if os.path.isfile(raw_file_path): + # Rename file with extension original + renamed_file = file_name_extension(raw_file_path) + renamed_path.append(renamed_file) + else: + # Check for the file with the _original suffix + original_file_path = raw_file_path.replace( + os.path.splitext(raw_file_path)[1], + "_original" + os.path.splitext(raw_file_path)[1], + ) + if os.path.exists(original_file_path): + raise FileExistsError( + f"File {original_file_path} already exists." + f"Please rename the file {raw_file_path} first." + ) + + except FileNotFoundError as e: + print(e) + continue try: # Run the snakemake rule