From 0f5f09af8a95b69bc46a7b31cf96e7b97489cc94 Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 21 Mar 2024 01:48:36 +0300 Subject: [PATCH 01/45] Account for the file extension --- scripts/build_demand_profiles.py | 34 +++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 9891623be..5452c6dd6 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -109,14 +109,34 @@ def get_load_paths_gegis(ssp_parentfolder, config): ssp = config.get("load_options")["ssp"] load_paths = [] - for continent in region_load: - load_path = os.path.join( - ssp_parentfolder, - str(ssp), - str(prediction_year), - "era5_" + str(weather_year), - str(continent) + ".nc", + load_dir = os.path.join( + ssp_parentfolder, + str(ssp), + str(prediction_year), + "era5_" + str(weather_year), + ) + regions_in_nc = [ + cnt + for cnt in region_load + if os.path.exists(os.path.join(load_dir, str(cnt) + ".nc")) + ] + regions_in_csv = [ + cnt + for cnt in region_load + if os.path.exists(os.path.join(load_dir, str(cnt) + ".csv")) + ] + + if len(regions_in_nc) == len(region_load): + fl_ext = ".nc" + elif len(regions_in_csv) == len(region_load): + fl_ext = ".csv" + else: + logger.error( + "No demand data file for {set(region_load).difference(regions_in_nc)}" ) + + for continent in region_load: + load_path = os.path.join(load_dir, str(continent) + fl_ext) load_paths.append(load_path) return load_paths From 849c092486cd9461718a182665d757787ec6dafd Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 21 Mar 2024 17:06:18 +0300 Subject: [PATCH 02/45] Add import --- scripts/build_demand_profiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 5452c6dd6..6d51bc8f8 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -42,6 +42,7 @@ """ import logging import os +import os.path from itertools import product import geopandas as gpd From cbe0dcec4a8950303fd6155ef8d12f05147436d0 Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 22 Mar 2024 23:54:04 +0300 Subject: [PATCH 03/45] Fix format of the message --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 6d51bc8f8..28ccc4ad0 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -133,7 +133,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): fl_ext = ".csv" else: logger.error( - "No demand data file for {set(region_load).difference(regions_in_nc)}" + f"No demand data file for {set(region_load).difference(regions_in_nc)}" ) for continent in region_load: From a3b2f8cc775f533fea7e531f0cc515760e766650 Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 22 Mar 2024 23:55:02 +0300 Subject: [PATCH 04/45] Add csv loading --- scripts/build_demand_profiles.py | 38 ++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 28ccc4ad0..9b988e7de 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -197,8 +197,42 @@ def build_demand_profiles( substation_lv_i = n.buses.index[n.buses["substation_lv"]] regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i) load_paths = load_paths - # Merge load .nc files: https://stackoverflow.com/questions/47226429/join-merge-multiple-netcdf-files-using-xarray - gegis_load = xr.open_mfdataset(load_paths, combine="nested") + + if str(load_paths[0]).endswith(".csv"): + df = pd.concat( + [ + pd.read_csv( + pth, + sep=";", + dtype={ + "region_code": str, + "time": str, + "region_name": str, + "Electricity demand": np.float64, + }, + keep_default_na=False, + na_values=["_", ""], + ) + for pth in load_paths + ], + axis=0, + ) + df.time = pd.to_datetime(df.time, format="%Y-%m-%d %H:%M:%S") + load_regions = {c: n for c, n in zip(df.region_code, df.region_name)} + + gegis_load = df.set_index(["region_code", "time"]).to_xarray() + gegis_load = gegis_load.assign_coords( + { + "region_name": ( + "region_code", + [name for (code, name) in load_regions.items()], + ) + } + ) + else: + # Merge load .nc files: https://stackoverflow.com/questions/47226429/join-merge-multiple-netcdf-files-using-xarray + gegis_load = xr.open_mfdataset(load_paths, combine="nested") + gegis_load = gegis_load.to_dataframe().reset_index().set_index("time") # filter load for analysed countries gegis_load = gegis_load.loc[gegis_load.region_code.isin(countries)] From 647003491cf2969c417d0650b717c52d0ab55cf7 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 23 Mar 2024 00:26:22 +0300 Subject: [PATCH 05/45] Refactor path generation --- scripts/build_demand_profiles.py | 38 ++++++++++++++------------------ 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 9b988e7de..6198f79f8 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -116,29 +116,23 @@ def get_load_paths_gegis(ssp_parentfolder, config): str(prediction_year), "era5_" + str(weather_year), ) - regions_in_nc = [ - cnt - for cnt in region_load - if os.path.exists(os.path.join(load_dir, str(cnt) + ".nc")) - ] - regions_in_csv = [ - cnt - for cnt in region_load - if os.path.exists(os.path.join(load_dir, str(cnt) + ".csv")) - ] - - if len(regions_in_nc) == len(region_load): - fl_ext = ".nc" - elif len(regions_in_csv) == len(region_load): - fl_ext = ".csv" - else: - logger.error( - f"No demand data file for {set(region_load).difference(regions_in_nc)}" - ) - for continent in region_load: - load_path = os.path.join(load_dir, str(continent) + fl_ext) - load_paths.append(load_path) + for ext in [".nc", ".csv"]: + avail_regions = [ + cnt + for cnt in region_load + if os.path.exists(os.path.join(load_dir, cnt + ext)) + ] + if len(avail_regions) == len(region_load): + for continent in region_load: + load_path = os.path.join(load_dir, continent + ext) + load_paths.append(load_path) + break + + if len(avail_regions) != len(region_load): + raise AssertionError( + f"No demand data file for {set(region_load).difference(avail_regions)}" + ) return load_paths From 7eb36b410f6e8def33985ff83c76c2a0e5e4fb7e Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 23 Mar 2024 12:36:19 +0300 Subject: [PATCH 06/45] Fix the check of demand availability --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 6198f79f8..3c68f0b9e 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -129,7 +129,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): load_paths.append(load_path) break - if len(avail_regions) != len(region_load): + if len(load_paths) == 0: raise AssertionError( f"No demand data file for {set(region_load).difference(avail_regions)}" ) From 5504b79f3a92a8f5fa02ec79546bb8bc77c04a99 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 23 Mar 2024 12:48:10 +0300 Subject: [PATCH 07/45] Comment-out the load path check --- scripts/build_demand_profiles.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 3c68f0b9e..75b075934 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -129,10 +129,10 @@ def get_load_paths_gegis(ssp_parentfolder, config): load_paths.append(load_path) break - if len(load_paths) == 0: - raise AssertionError( - f"No demand data file for {set(region_load).difference(avail_regions)}" - ) + # if len(load_paths) == 0: + # raise AssertionError( + # f"No demand data file for {set(region_load).difference(avail_regions)}" + # ) return load_paths From c37bf1b73cee75f90b53e01ef7ac9301abeb3e06 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Fri, 12 Apr 2024 00:40:22 +0300 Subject: [PATCH 08/45] Implement Davide's suggestion Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> --- scripts/build_demand_profiles.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 75b075934..d873166d1 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -117,15 +117,10 @@ def get_load_paths_gegis(ssp_parentfolder, config): "era5_" + str(weather_year), ) - for ext in [".nc", ".csv"]: - avail_regions = [ - cnt - for cnt in region_load - if os.path.exists(os.path.join(load_dir, cnt + ext)) - ] - if len(avail_regions) == len(region_load): - for continent in region_load: - load_path = os.path.join(load_dir, continent + ext) + for continent in region_load: + for ext in [".nc", ".csv"]: + load_path = os.path.join(load_dir, continent + ext) + if os.path.exists(load_path): load_paths.append(load_path) break From 397a6d9a73131751c40d797ef5dfdea08f560c10 Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 12 Apr 2024 00:48:28 +0300 Subject: [PATCH 09/45] Add a TODO comment --- scripts/build_demand_profiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index d873166d1..05a78fde9 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -124,6 +124,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): load_paths.append(load_path) break + # TODO Remember to remove after debug # if len(load_paths) == 0: # raise AssertionError( # f"No demand data file for {set(region_load).difference(avail_regions)}" From 061c98574d4a712a4a3021d3f7f6db4377ef1c7e Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 12 Apr 2024 00:49:54 +0300 Subject: [PATCH 10/45] Put loading csv-s into a function --- scripts/build_demand_profiles.py | 73 +++++++++++++++++--------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 05a78fde9..459b880bc 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -148,6 +148,39 @@ def shapes_to_shapes(orig, dest): return transfer +def load_demand_csv(path): + df = pd.concat( + [ + pd.read_csv( + pth, + sep=";", + dtype={ + "region_code": str, + "time": str, + "region_name": str, + "Electricity demand": np.float64, + }, + keep_default_na=False, + na_values=["_", ""], + ) + for pth in path + ], + axis=0, + ) + df.time = pd.to_datetime(df.time, format="%Y-%m-%d %H:%M:%S") + load_regions = {c: n for c, n in zip(df.region_code, df.region_name)} + + gegis_load = df.set_index(["region_code", "time"]).to_xarray() + gegis_load = gegis_load.assign_coords( + { + "region_name": ( + "region_code", + [name for (code, name) in load_regions.items()], + ) + } + ) + + def build_demand_profiles( n, load_paths, @@ -188,40 +221,12 @@ def build_demand_profiles( regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i) load_paths = load_paths - if str(load_paths[0]).endswith(".csv"): - df = pd.concat( - [ - pd.read_csv( - pth, - sep=";", - dtype={ - "region_code": str, - "time": str, - "region_name": str, - "Electricity demand": np.float64, - }, - keep_default_na=False, - na_values=["_", ""], - ) - for pth in load_paths - ], - axis=0, - ) - df.time = pd.to_datetime(df.time, format="%Y-%m-%d %H:%M:%S") - load_regions = {c: n for c, n in zip(df.region_code, df.region_name)} - - gegis_load = df.set_index(["region_code", "time"]).to_xarray() - gegis_load = gegis_load.assign_coords( - { - "region_name": ( - "region_code", - [name for (code, name) in load_regions.items()], - ) - } - ) - else: - # Merge load .nc files: https://stackoverflow.com/questions/47226429/join-merge-multiple-netcdf-files-using-xarray - gegis_load = xr.open_mfdataset(load_paths, combine="nested") + for path in load_paths: + if str(path).endswith(".csv"): + gegis_load = load_demand_csv(path) + else: + # Merge load .nc files: https://stackoverflow.com/questions/47226429/join-merge-multiple-netcdf-files-using-xarray + gegis_load = xr.open_mfdataset(path, combine="nested") gegis_load = gegis_load.to_dataframe().reset_index().set_index("time") # filter load for analysed countries From 9464d5598217a8f0ca333add71dea787865524a8 Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 12 Apr 2024 00:50:51 +0300 Subject: [PATCH 11/45] Use na-safe csv loading --- scripts/build_demand_profiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 459b880bc..219fc0043 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -52,7 +52,7 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, create_logger, read_osm_config +from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config from shapely.prepared import prep from shapely.validation import make_valid @@ -151,7 +151,7 @@ def shapes_to_shapes(orig, dest): def load_demand_csv(path): df = pd.concat( [ - pd.read_csv( + read_csv_nafix( pth, sep=";", dtype={ From 8c27cd5b98548d69a38fa2c3a60778c64b267b7f Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 12 Apr 2024 00:53:03 +0300 Subject: [PATCH 12/45] Fix the function output --- scripts/build_demand_profiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 219fc0043..e37134ea5 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -179,6 +179,7 @@ def load_demand_csv(path): ) } ) + return gegis_load def build_demand_profiles( From 59fe870f09784949e44f56097c3463bf36606795 Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 12 Apr 2024 00:57:41 +0300 Subject: [PATCH 13/45] Add an option to combine nc and csv inputs --- scripts/build_demand_profiles.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index e37134ea5..fcbe41bb5 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -222,13 +222,17 @@ def build_demand_profiles( regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i) load_paths = load_paths + gegis_load_list = [] + for path in load_paths: if str(path).endswith(".csv"): - gegis_load = load_demand_csv(path) + gegis_load_xr = load_demand_csv(path) else: # Merge load .nc files: https://stackoverflow.com/questions/47226429/join-merge-multiple-netcdf-files-using-xarray - gegis_load = xr.open_mfdataset(path, combine="nested") + gegis_load_xr = xr.open_mfdataset(path, combine="nested") + gegis_load_list.append(gegis_load_xr) + gegis_load = xr.merge(gegis_load_list) gegis_load = gegis_load.to_dataframe().reset_index().set_index("time") # filter load for analysed countries gegis_load = gegis_load.loc[gegis_load.region_code.isin(countries)] From 94378e57a1047774fc2181d637c1915978f5a7be Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 30 Apr 2024 00:50:44 +0300 Subject: [PATCH 14/45] Fix breaking the loop --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index fcbe41bb5..53906eb12 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -122,7 +122,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): load_path = os.path.join(load_dir, continent + ext) if os.path.exists(load_path): load_paths.append(load_path) - break + break # TODO Remember to remove after debug # if len(load_paths) == 0: From 87fb0993dea5fc6185f8c317f2291a6b71ceb077 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 8 May 2024 01:47:19 +0300 Subject: [PATCH 15/45] Put the path check back --- scripts/build_demand_profiles.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 15e896764..e3e55d4dc 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -117,16 +117,20 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: for ext in [".nc", ".csv"]: - load_path = os.path.join(load_dir, continent + ext) + load_path = os.path.join(load_dir, str(continent) + str(ext)) if os.path.exists(load_path): load_paths.append(load_path) break + avail_regions = [ + os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths + ] + # TODO Remember to remove after debug - # if len(load_paths) == 0: - # raise AssertionError( - # f"No demand data file for {set(region_load).difference(avail_regions)}" - # ) + if len(load_paths) == 0: + raise AssertionError( + f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." + ) return load_paths From 37c62ce1d6b702d03178c81220ee4a2e19cb59e8 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 8 May 2024 10:58:45 +0300 Subject: [PATCH 16/45] Add a string conversion --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index e3e55d4dc..09ecf9cd7 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -117,7 +117,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: for ext in [".nc", ".csv"]: - load_path = os.path.join(load_dir, str(continent) + str(ext)) + load_path = os.path.join(str(load_dir), str(continent) + str(ext)) if os.path.exists(load_path): load_paths.append(load_path) break From 8650b7cb2bed71f12f273a27eda5118e7087d386 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 8 May 2024 10:59:03 +0300 Subject: [PATCH 17/45] Add a test for paths checked --- scripts/build_demand_profiles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 09ecf9cd7..2d3af8c93 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -108,6 +108,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): ssp = config.get("load_options")["ssp"] load_paths = [] + walk_paths = [] load_dir = os.path.join( ssp_parentfolder, str(ssp), @@ -118,6 +119,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: for ext in [".nc", ".csv"]: load_path = os.path.join(str(load_dir), str(continent) + str(ext)) + walk_paths.append(load_path) if os.path.exists(load_path): load_paths.append(load_path) break @@ -129,7 +131,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): # TODO Remember to remove after debug if len(load_paths) == 0: raise AssertionError( - f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." + f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}. The paths checked {walk_paths}." ) return load_paths From 827deb9a613f279d2e2815fe23243e850ef51549 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 8 May 2024 11:15:19 +0300 Subject: [PATCH 18/45] Replace an error with a warning --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 2d3af8c93..a04504777 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -130,7 +130,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): # TODO Remember to remove after debug if len(load_paths) == 0: - raise AssertionError( + logger.warning( f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}. The paths checked {walk_paths}." ) From 8b031fe998ad93eb6093b52d56355111ba11fe49 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 8 May 2024 13:15:56 +0300 Subject: [PATCH 19/45] Add a diagnostic info --- scripts/build_demand_profiles.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index a04504777..8e0dd406c 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -236,8 +236,12 @@ def build_demand_profiles( gegis_load_xr = xr.open_mfdataset(path, combine="nested") gegis_load_list.append(gegis_load_xr) + logger.info( + f"Merging {gegis_load_list} from paths {load_paths} into the load data frame" + ) gegis_load = xr.merge(gegis_load_list) gegis_load = gegis_load.to_dataframe().reset_index().set_index("time") + # filter load for analysed countries gegis_load = gegis_load.loc[gegis_load.region_code.isin(countries)] From 0aabc66a33b4e42308581ebb98fad3789274dc38 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 8 May 2024 19:12:44 +0300 Subject: [PATCH 20/45] Define load_paths to snakemake params --- Snakefile | 5 +++-- scripts/build_demand_profiles.py | 22 ++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Snakefile b/Snakefile index 5b2de322e..3e4197e1e 100644 --- a/Snakefile +++ b/Snakefile @@ -43,7 +43,6 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -load_data_paths = get_load_paths_gegis("data", config) if config["enable"].get("retrieve_cost_data", True): COSTS = "resources/" + RDIR + "costs.csv" else: @@ -417,10 +416,12 @@ rule build_demand_profiles: snapshots=config["snapshots"], load_options=config["load_options"], countries=config["countries"], + weather_year=config["load_options"]["weather_year"], + prediction_year=config["load_options"]["prediction_year"], + ssp=config["load_options"]["ssp"], input: base_network="networks/" + RDIR + "base.nc", regions="resources/" + RDIR + "bus_regions/regions_onshore.geojson", - load=load_data_paths, #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 8e0dd406c..7479a4d85 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -90,7 +90,9 @@ def get_gegis_regions(countries): return regions -def get_load_paths_gegis(ssp_parentfolder, config): +def get_load_paths_gegis( + ssp_parentfolder, countries, weather_year, prediction_year, ssp +): """ Create load paths for GEGIS outputs. @@ -101,11 +103,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): ------- ["/data/ssp2-2.6/2030/era5_2013/Africa.nc", "/data/ssp2-2.6/2030/era5_2013/Africa.nc"] """ - countries = config.get("countries") region_load = get_gegis_regions(countries) - weather_year = config.get("load_options")["weather_year"] - prediction_year = config.get("load_options")["prediction_year"] - ssp = config.get("load_options")["ssp"] load_paths = [] walk_paths = [] @@ -320,14 +318,26 @@ def upsample(cntry, group): # Snakemake imports: regions = snakemake.input.regions - load_paths = snakemake.input["load"] countries = snakemake.params.countries + + weather_year = snakemake.params.load_options["weather_year"] + prediction_year = snakemake.params.load_options["prediction_year"] + ssp = snakemake.params.load_options["ssp"] + admin_shapes = snakemake.input.gadm_shapes scale = snakemake.params.load_options.get("scale", 1.0) start_date = snakemake.params.snapshots["start"] end_date = snakemake.params.snapshots["end"] out_path = snakemake.output[0] + load_paths = get_load_paths_gegis( + ssp_parentfolder="data", + countries=countries, + weather_year=weather_year, + prediction_year=prediction_year, + ssp=ssp, + ) + build_demand_profiles( n, load_paths, From 065a9cd3fe9caa66f7b54e8dd33a4958fc96f975 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 13 May 2024 11:18:46 +0300 Subject: [PATCH 21/45] Fix structure --- scripts/build_demand_profiles.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 7479a4d85..cee133d6d 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -151,24 +151,7 @@ def shapes_to_shapes(orig, dest): def load_demand_csv(path): - df = pd.concat( - [ - read_csv_nafix( - pth, - sep=";", - dtype={ - "region_code": str, - "time": str, - "region_name": str, - "Electricity demand": np.float64, - }, - keep_default_na=False, - na_values=["_", ""], - ) - for pth in path - ], - axis=0, - ) + df = read_csv_nafix(path, sep=";") df.time = pd.to_datetime(df.time, format="%Y-%m-%d %H:%M:%S") load_regions = {c: n for c, n in zip(df.region_code, df.region_name)} From 8c2c2aa532c3f6c40ef6bd496be70ca86ba5ad47 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 13 May 2024 11:27:04 +0300 Subject: [PATCH 22/45] Fix the info message --- scripts/build_demand_profiles.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index cee133d6d..78ceaa273 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -217,9 +217,7 @@ def build_demand_profiles( gegis_load_xr = xr.open_mfdataset(path, combine="nested") gegis_load_list.append(gegis_load_xr) - logger.info( - f"Merging {gegis_load_list} from paths {load_paths} into the load data frame" - ) + logger.info(f"Merging demand data from paths {load_paths} into the load data frame") gegis_load = xr.merge(gegis_load_list) gegis_load = gegis_load.to_dataframe().reset_index().set_index("time") From cfb5432d7a1027d1c2995083f7caab5a9f02b6dc Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 13 May 2024 11:32:50 +0300 Subject: [PATCH 23/45] Update the error message --- scripts/build_demand_profiles.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 78ceaa273..8fb1a70b8 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -106,7 +106,6 @@ def get_load_paths_gegis( region_load = get_gegis_regions(countries) load_paths = [] - walk_paths = [] load_dir = os.path.join( ssp_parentfolder, str(ssp), @@ -117,7 +116,6 @@ def get_load_paths_gegis( for continent in region_load: for ext in [".nc", ".csv"]: load_path = os.path.join(str(load_dir), str(continent) + str(ext)) - walk_paths.append(load_path) if os.path.exists(load_path): load_paths.append(load_path) break @@ -125,11 +123,9 @@ def get_load_paths_gegis( avail_regions = [ os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths ] - - # TODO Remember to remove after debug if len(load_paths) == 0: logger.warning( - f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}. The paths checked {walk_paths}." + f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." ) return load_paths From 5f383dd25380d4dabc8b9c28afecf29c3ce65766 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 25 May 2024 10:34:42 +0300 Subject: [PATCH 24/45] Revert "Define load_paths to snakemake params" This reverts commit 0aabc66a33b4e42308581ebb98fad3789274dc38. --- Snakefile | 5 ++--- scripts/build_demand_profiles.py | 22 ++++++---------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Snakefile b/Snakefile index 3e4197e1e..5b2de322e 100644 --- a/Snakefile +++ b/Snakefile @@ -43,6 +43,7 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" +load_data_paths = get_load_paths_gegis("data", config) if config["enable"].get("retrieve_cost_data", True): COSTS = "resources/" + RDIR + "costs.csv" else: @@ -416,12 +417,10 @@ rule build_demand_profiles: snapshots=config["snapshots"], load_options=config["load_options"], countries=config["countries"], - weather_year=config["load_options"]["weather_year"], - prediction_year=config["load_options"]["prediction_year"], - ssp=config["load_options"]["ssp"], input: base_network="networks/" + RDIR + "base.nc", regions="resources/" + RDIR + "bus_regions/regions_onshore.geojson", + load=load_data_paths, #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 8fb1a70b8..9c12a37f9 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -90,9 +90,7 @@ def get_gegis_regions(countries): return regions -def get_load_paths_gegis( - ssp_parentfolder, countries, weather_year, prediction_year, ssp -): +def get_load_paths_gegis(ssp_parentfolder, config): """ Create load paths for GEGIS outputs. @@ -103,7 +101,11 @@ def get_load_paths_gegis( ------- ["/data/ssp2-2.6/2030/era5_2013/Africa.nc", "/data/ssp2-2.6/2030/era5_2013/Africa.nc"] """ + countries = config.get("countries") region_load = get_gegis_regions(countries) + weather_year = config.get("load_options")["weather_year"] + prediction_year = config.get("load_options")["prediction_year"] + ssp = config.get("load_options")["ssp"] load_paths = [] load_dir = os.path.join( @@ -295,26 +297,14 @@ def upsample(cntry, group): # Snakemake imports: regions = snakemake.input.regions + load_paths = snakemake.input["load"] countries = snakemake.params.countries - - weather_year = snakemake.params.load_options["weather_year"] - prediction_year = snakemake.params.load_options["prediction_year"] - ssp = snakemake.params.load_options["ssp"] - admin_shapes = snakemake.input.gadm_shapes scale = snakemake.params.load_options.get("scale", 1.0) start_date = snakemake.params.snapshots["start"] end_date = snakemake.params.snapshots["end"] out_path = snakemake.output[0] - load_paths = get_load_paths_gegis( - ssp_parentfolder="data", - countries=countries, - weather_year=weather_year, - prediction_year=prediction_year, - ssp=ssp, - ) - build_demand_profiles( n, load_paths, From 4c5974d7f1ed0bf5a81a78d9a6396f20afa1cefa Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 25 May 2024 10:38:15 +0300 Subject: [PATCH 25/45] Add an existence check --- scripts/build_demand_profiles.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 9c12a37f9..7a18c9ebd 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -90,7 +90,7 @@ def get_gegis_regions(countries): return regions -def get_load_paths_gegis(ssp_parentfolder, config): +def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): """ Create load paths for GEGIS outputs. @@ -115,20 +115,21 @@ def get_load_paths_gegis(ssp_parentfolder, config): "era5_" + str(weather_year), ) - for continent in region_load: - for ext in [".nc", ".csv"]: - load_path = os.path.join(str(load_dir), str(continent) + str(ext)) - if os.path.exists(load_path): - load_paths.append(load_path) - break - - avail_regions = [ - os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths - ] - if len(load_paths) == 0: - logger.warning( - f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." - ) + if check_existence: + for continent in region_load: + for ext in [".nc", ".csv"]: + load_path = os.path.join(str(load_dir), str(continent) + str(ext)) + if os.path.exists(load_path): + load_paths.append(load_path) + break + + avail_regions = [ + os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths + ] + if len(load_paths) == 0: + logger.warning( + f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." + ) return load_paths From e63f6b3c71e04b5fb17bb4d45716cb421dfba984 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 25 May 2024 11:16:23 +0300 Subject: [PATCH 26/45] Return default nc-path definition --- scripts/build_demand_profiles.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 7a18c9ebd..3a57cd469 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -130,6 +130,13 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): logger.warning( f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." ) + else: + for continent in region_load: + load_path = os.path.join( + load_dir, + str(continent) + ".nc", + ) + load_paths.append(load_path) return load_paths From 76228187e13057d9423d29fca9a26abe321483ff Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 25 May 2024 20:23:30 +0300 Subject: [PATCH 27/45] Apply changes to the path definition --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 5b2de322e..fdae07d05 100644 --- a/Snakefile +++ b/Snakefile @@ -420,7 +420,7 @@ rule build_demand_profiles: input: base_network="networks/" + RDIR + "base.nc", regions="resources/" + RDIR + "bus_regions/regions_onshore.geojson", - load=load_data_paths, + load=get_load_paths_gegis("data", config, check_existence=True), #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ From 5b488b29623b20cea1c1153cc1d3a1f060acc40a Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 28 May 2024 23:32:58 +0300 Subject: [PATCH 28/45] Remove a redundant definition --- Snakefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Snakefile b/Snakefile index fdae07d05..fc1ebb81e 100644 --- a/Snakefile +++ b/Snakefile @@ -43,7 +43,6 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -load_data_paths = get_load_paths_gegis("data", config) if config["enable"].get("retrieve_cost_data", True): COSTS = "resources/" + RDIR + "costs.csv" else: From 083a1ac113c60320e15b25c1101f3d7f32075d39 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 28 May 2024 23:33:13 +0300 Subject: [PATCH 29/45] Add type conversion --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 3a57cd469..c5183dac5 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -133,7 +133,7 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): else: for continent in region_load: load_path = os.path.join( - load_dir, + str(load_dir), str(continent) + ".nc", ) load_paths.append(load_path) From b71eeea5e19441f233ed80cfcff5c259097bc0da Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 28 May 2024 23:33:27 +0300 Subject: [PATCH 30/45] Add a (temporary) warning --- scripts/build_demand_profiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index c5183dac5..95c62709d 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -131,6 +131,7 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." ) else: + logger.warning("Existence of the load data is not checked") for continent in region_load: load_path = os.path.join( str(load_dir), From 31eec33b5be9f6b3b3a0a8dce2cc432b2a999318 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 28 May 2024 23:49:45 +0300 Subject: [PATCH 31/45] Add a diagnostic logger info --- scripts/build_demand_profiles.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 95c62709d..cab67833c 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -126,6 +126,10 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): avail_regions = [ os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths ] + + # TODO Remove after debugging + logger.info(f" An assumed load folder {load_dir}, load path is {load_paths}.") + if len(load_paths) == 0: logger.warning( f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." From 9482e2882c6d7ce51791ebfb3b147451ddf33cd0 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 29 May 2024 17:41:13 +0300 Subject: [PATCH 32/45] Revert "Remove a redundant definition" This reverts commit 5b488b29623b20cea1c1153cc1d3a1f060acc40a. --- Snakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Snakefile b/Snakefile index fc1ebb81e..fdae07d05 100644 --- a/Snakefile +++ b/Snakefile @@ -43,6 +43,7 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" +load_data_paths = get_load_paths_gegis("data", config) if config["enable"].get("retrieve_cost_data", True): COSTS = "resources/" + RDIR + "costs.csv" else: From 18df6d3a21c6828ab0aac6a5d1397457de12d9b6 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 29 May 2024 17:50:28 +0300 Subject: [PATCH 33/45] Account for different states of the demand inputs --- Snakefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index fdae07d05..33d11e41d 100644 --- a/Snakefile +++ b/Snakefile @@ -43,7 +43,12 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -load_data_paths = get_load_paths_gegis("data", config) +if os.path.exists("data"): + load_data_paths = get_load_paths_gegis("data", config, check_existence=True) +else: + # demand profiles are not available yet -> no opportunity to use custom load files + load_data_paths = get_load_paths_gegis("data", config, check_existence=False) + if config["enable"].get("retrieve_cost_data", True): COSTS = "resources/" + RDIR + "costs.csv" else: @@ -420,7 +425,7 @@ rule build_demand_profiles: input: base_network="networks/" + RDIR + "base.nc", regions="resources/" + RDIR + "bus_regions/regions_onshore.geojson", - load=get_load_paths_gegis("data", config, check_existence=True), + load=load_data_paths, #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ From 049aaa1b66697654de1c002ffff580e18f65ce52 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 29 May 2024 18:10:59 +0300 Subject: [PATCH 34/45] Modify handling an empty demand case --- Snakefile | 2 +- scripts/build_demand_profiles.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 33d11e41d..b04a3e382 100644 --- a/Snakefile +++ b/Snakefile @@ -43,7 +43,7 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -if os.path.exists("data"): +if os.path.exists(os.path.join("data", config["load_options"]["ssp"])): load_data_paths = get_load_paths_gegis("data", config, check_existence=True) else: # demand profiles are not available yet -> no opportunity to use custom load files diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index cab67833c..a47376dff 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -142,6 +142,8 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): str(continent) + ".nc", ) load_paths.append(load_path) + # TODO Remove after debugging + logger.info(f" An assumed load folder {load_dir}, load path is {load_paths}.") return load_paths From 8e342132838f2d2fe007c0d109764767aea09b7e Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 29 May 2024 18:24:03 +0300 Subject: [PATCH 35/45] Add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index dce812acd..f7f351314 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t * Add an option to merge isolated networks into respective backbone networks by countries. `PR #903 `__ +* Add an option to use csv format for custom demand imports. `PR #995 `__ + **Minor Changes and bug-fixing** * Minor bug-fixing to run the cluster wildcard min `PR #1019 `__ From 511140d94b0f311072b2d5e16d8988af98f7c17e Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 29 May 2024 21:06:32 +0300 Subject: [PATCH 36/45] Remove temporary comments --- scripts/build_demand_profiles.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index a47376dff..8df3bc289 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -135,15 +135,12 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." ) else: - logger.warning("Existence of the load data is not checked") for continent in region_load: load_path = os.path.join( str(load_dir), str(continent) + ".nc", ) load_paths.append(load_path) - # TODO Remove after debugging - logger.info(f" An assumed load folder {load_dir}, load path is {load_paths}.") return load_paths From 25451d71aea7c7603cad30e49d46e1b56872628f Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 29 May 2024 23:38:55 +0300 Subject: [PATCH 37/45] Add documentation --- doc/customization_basic1.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/customization_basic1.rst b/doc/customization_basic1.rst index 3056ba59c..683af810d 100644 --- a/doc/customization_basic1.rst +++ b/doc/customization_basic1.rst @@ -82,6 +82,20 @@ Year-related parameters are also being used when specifying `load_options`: The `weather_year` value corresponds to the weather data which was used to generate the electricity demand profiles for a selected area while `prediction_year` corresponds to the point of a `Shared Socioeconomic Pathways (SSP) `__ trajectory. PyPSA-Earth uses SSP2-2.6 scenario within the Shared Socioeconomic Pathways framework, which is characterized by medium challenges to mitigation and adaptation efforts resulting in a global warming of approximately 2.6°C by the end of the 21st century. The available values for `weather_year` and `prediction_year` can be checked by looking into `pypsa-earth/data/ssp2-2.6` folder. Currently, there are pre-calculated demand data for 2011, 2013, 2018 weather years and for 2030, 2040, 2050, and 2100 scenario prediction years. +Use custom demand data +---------------------- + +It is possible to implement custom demand profiles. It can be done by creating a dedicated custom demand sub-folder in a scenario folder `pypsa-earth/data/ssp2-2.6` and placing there a custom demand file. The name of a custom demand sub-folder should correspond to `weather_year` argument which stands in this case for general identification of a demand input. The name of a demand input file should be a continent name to which belongs a country of initerest. Both csv and nc formats can be used for demand files. + +For example, to `pypsa-earth/data/ssp2-2.6/2013_custom/` + +.. note:: + + For example, to provide custom inputs for Nigeria, you can put the time-series into `Africa.csv` file and place the file into `pypsa-earth/data/ssp2-2.6/2013_custom/` folder. To make it fetched, you'll need to specify `weather_year: 2013_custom` under `load_options`. + +A format of the custom csv demand file should correspond to the csv files supplied with the model: there are `region_code`, `time`, `region_name` and `Electricity demand` columns, while a semicolon is used as a separator. + + Configure `atlite` section -------------------------- From 0584f17254e28c5d22565a35a7c000fad11db251 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 4 Jun 2024 13:15:44 +0300 Subject: [PATCH 38/45] Move existence check into the demand parser --- Snakefile | 6 +----- scripts/build_demand_profiles.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index 93514a25f..561be22ff 100644 --- a/Snakefile +++ b/Snakefile @@ -43,11 +43,7 @@ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -if os.path.exists(os.path.join("data", config["load_options"]["ssp"])): - load_data_paths = get_load_paths_gegis("data", config, check_existence=True) -else: - # demand profiles are not available yet -> no opportunity to use custom load files - load_data_paths = get_load_paths_gegis("data", config, check_existence=False) +load_data_paths = get_load_paths_gegis("data", config) if config["enable"].get("retrieve_cost_data", True): COSTS = "resources/" + RDIR + "costs.csv" diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 8df3bc289..9c26bfe26 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -90,7 +90,7 @@ def get_gegis_regions(countries): return regions -def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): +def get_load_paths_gegis(ssp_parentfolder, config): """ Create load paths for GEGIS outputs. @@ -115,7 +115,7 @@ def get_load_paths_gegis(ssp_parentfolder, config, check_existence=False): "era5_" + str(weather_year), ) - if check_existence: + if os.path.exists(os.path.join("data", ssp)): for continent in region_load: for ext in [".nc", ".csv"]: load_path = os.path.join(str(load_dir), str(continent) + str(ext)) From 00b8d60ed8b322774aa6fa483b7667a98cdd5f54 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 4 Jun 2024 13:22:26 +0300 Subject: [PATCH 39/45] Clean-up implementation --- scripts/build_demand_profiles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 9c26bfe26..bcfe4451a 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -107,6 +107,8 @@ def get_load_paths_gegis(ssp_parentfolder, config): prediction_year = config.get("load_options")["prediction_year"] ssp = config.get("load_options")["ssp"] + scenario_path = os.path.join(ssp_parentfolder, ssp) + load_paths = [] load_dir = os.path.join( ssp_parentfolder, @@ -115,7 +117,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): "era5_" + str(weather_year), ) - if os.path.exists(os.path.join("data", ssp)): + if os.path.exists(scenario_path): for continent in region_load: for ext in [".nc", ".csv"]: load_path = os.path.join(str(load_dir), str(continent) + str(ext)) From 5c8ce7f1ad6597aa2572bbe4cc99d703581f793d Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 4 Jun 2024 14:56:28 +0300 Subject: [PATCH 40/45] Remove an outdate commit --- scripts/build_demand_profiles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index bcfe4451a..f65d8ad75 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -129,7 +129,6 @@ def get_load_paths_gegis(ssp_parentfolder, config): os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths ] - # TODO Remove after debugging logger.info(f" An assumed load folder {load_dir}, load path is {load_paths}.") if len(load_paths) == 0: From 4fadf7456167075ea808049524004687bf0c1342 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Thu, 6 Jun 2024 11:31:46 +0300 Subject: [PATCH 41/45] Implement Davide's suggestion Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> --- scripts/build_demand_profiles.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index f65d8ad75..bed4f5add 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -118,23 +118,25 @@ def get_load_paths_gegis(ssp_parentfolder, config): ) if os.path.exists(scenario_path): + regions_found = [] + file_names = [] for continent in region_load: + sel_ext = ".nc" for ext in [".nc", ".csv"]: load_path = os.path.join(str(load_dir), str(continent) + str(ext)) if os.path.exists(load_path): - load_paths.append(load_path) + sel_ext = ext + regions_found.append(continent) break - - avail_regions = [ - os.path.split(os.path.abspath(pth))[1].split(".nc")[0] for pth in load_paths - ] - - logger.info(f" An assumed load folder {load_dir}, load path is {load_paths}.") - - if len(load_paths) == 0: - logger.warning( - f"No demand data file for {set(region_load).difference(avail_regions)}. An assumed load folder {load_dir}." - ) + file_name = str(continent) + str(sel_ext) + load_path = os.path.join(str(load_dir), file_name) + load_paths.append(load_path) + file_names.append(file_name) + + logger.info( + f"Demand data folder: {load_dir}, load path is {load_paths}.\n" + + f"Expected files: " + "; ".join(file_names) + ) else: for continent in region_load: load_path = os.path.join( From d21bbe6a448f8580b8126dc746c5faaa85d65bad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 08:34:01 +0000 Subject: [PATCH 42/45] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_demand_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index bed4f5add..8ea7d5874 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -132,7 +132,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): load_path = os.path.join(str(load_dir), file_name) load_paths.append(load_path) file_names.append(file_name) - + logger.info( f"Demand data folder: {load_dir}, load path is {load_paths}.\n" + f"Expected files: " + "; ".join(file_names) From c6aa7f16443107a4cbef2f5e56f68bc9ac3dab3c Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 6 Jun 2024 11:42:59 +0300 Subject: [PATCH 43/45] Fix formatting --- scripts/build_demand_profiles.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 8ea7d5874..b8118b874 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -128,14 +128,15 @@ def get_load_paths_gegis(ssp_parentfolder, config): sel_ext = ext regions_found.append(continent) break - file_name = str(continent) + str(sel_ext) - load_path = os.path.join(str(load_dir), file_name) - load_paths.append(load_path) - file_names.append(file_name) + file_name = str(continent) + str(sel_ext) + load_path = os.path.join(str(load_dir), file_name) + load_paths.append(load_path) + file_names.append(file_name) logger.info( - f"Demand data folder: {load_dir}, load path is {load_paths}.\n" + - f"Expected files: " + "; ".join(file_names) + f"Demand data folder: {load_dir}, load path is {load_paths}.\n" + + f"Expected files: " + + "; ".join(file_names) ) else: for continent in region_load: From 14f4bca8685b6679e5f8d70675b70191d9c30730 Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 6 Jun 2024 18:37:38 +0300 Subject: [PATCH 44/45] Remove an outdated check --- scripts/build_demand_profiles.py | 48 +++++++++++++------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index b8118b874..352244d3a 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -117,34 +117,26 @@ def get_load_paths_gegis(ssp_parentfolder, config): "era5_" + str(weather_year), ) - if os.path.exists(scenario_path): - regions_found = [] - file_names = [] - for continent in region_load: - sel_ext = ".nc" - for ext in [".nc", ".csv"]: - load_path = os.path.join(str(load_dir), str(continent) + str(ext)) - if os.path.exists(load_path): - sel_ext = ext - regions_found.append(continent) - break - file_name = str(continent) + str(sel_ext) - load_path = os.path.join(str(load_dir), file_name) - load_paths.append(load_path) - file_names.append(file_name) - - logger.info( - f"Demand data folder: {load_dir}, load path is {load_paths}.\n" - + f"Expected files: " - + "; ".join(file_names) - ) - else: - for continent in region_load: - load_path = os.path.join( - str(load_dir), - str(continent) + ".nc", - ) - load_paths.append(load_path) + regions_found = [] + file_names = [] + for continent in region_load: + sel_ext = ".nc" + for ext in [".nc", ".csv"]: + load_path = os.path.join(str(load_dir), str(continent) + str(ext)) + if os.path.exists(load_path): + sel_ext = ext + regions_found.append(continent) + break + file_name = str(continent) + str(sel_ext) + load_path = os.path.join(str(load_dir), file_name) + load_paths.append(load_path) + file_names.append(file_name) + + logger.info( + f"Demand data folder: {load_dir}, load path is {load_paths}.\n" + + f"Expected files: " + + "; ".join(file_names) + ) return load_paths From ef4dc691787fce4af8b092cd137e20087613a76f Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 8 Jun 2024 20:59:34 +0300 Subject: [PATCH 45/45] Remove a not-used variable --- scripts/build_demand_profiles.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 352244d3a..b4893d112 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -117,7 +117,6 @@ def get_load_paths_gegis(ssp_parentfolder, config): "era5_" + str(weather_year), ) - regions_found = [] file_names = [] for continent in region_load: sel_ext = ".nc" @@ -125,7 +124,6 @@ def get_load_paths_gegis(ssp_parentfolder, config): load_path = os.path.join(str(load_dir), str(continent) + str(ext)) if os.path.exists(load_path): sel_ext = ext - regions_found.append(continent) break file_name = str(continent) + str(sel_ext) load_path = os.path.join(str(load_dir), file_name)