Update Snakemake and adapt for dag

Major changes:
- update Snakemake in environment.yaml to version 8

Minor changes:
- move the input file logging from build_demand_profiles.py to retrieve_databundle_light.py to prevent issues with the creation of the rule graph

envs/environment.yaml

@@ -25,7 +25,7 @@ dependencies:
 - xlrd
 - openpyxl
 - seaborn
-- snakemake-minimal<8
+- snakemake-minimal>=8
 - memory_profiler
 - ruamel.yaml<=0.17.26
 - pytables

scripts/build_demand_profiles.py

@@ -96,7 +96,7 @@ def get_gegis_regions(countries):
     return regions
 
 
-def get_load_paths_gegis(ssp_parentfolder, config):
+def prepare_load_paths_gegis(ssp_parentfolder, config):
     """
     Create load paths for GEGIS outputs.
 
@@ -136,14 +136,32 @@ def get_load_paths_gegis(ssp_parentfolder, config):
         load_paths.append(load_path)
         file_names.append(file_name)
 
+    return load_paths, load_dir, file_names
+
+
+def get_load_paths_gegis(ssp_parentfolder, config):
+    """
+    Return load paths for GEGIS outputs.
+    """
+
+    load_paths, _, _ = prepare_load_paths_gegis(ssp_parentfolder, config)
+
+    return load_paths
+
+
+def log_load_paths_gegis(ssp_parentfolder, config):
+    """
+    Log the load paths for GEGIS outputs.
+    """
+
+    load_paths, load_dir, file_names = prepare_load_paths_gegis(ssp_parentfolder, config)
+
     logger.info(
         f"Demand data folder: {load_dir}, load path is {load_paths}.\n"
         + f"Expected files: "
         + "; ".join(file_names)
     )
 
-    return load_paths
-
 
 def shapes_to_shapes(orig, dest):
     """

scripts/retrieve_databundle_light.py

@@ -95,6 +95,7 @@
     create_logger,
     progress_retrieve,
 )
+from build_demand_profiles import log_load_paths_gegis
 from google_drive_downloader import GoogleDriveDownloader as gdd
 from tqdm import tqdm
 
@@ -820,6 +821,9 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level):
     # TODO Make logging compatible with progressbar (see PR #102, PyPSA-Eur)
     configure_logging(snakemake)
 
+    # log the load paths for GEGIS outputs
+    log_load_paths_gegis("data", snakemake.config)
+    raise ValueError("asdaf")
     rootpath = "."
     tutorial = snakemake.params.tutorial
     countries = snakemake.params.countries