From dab0582427787c847ad83806710fde0f29029265 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 30 Nov 2023 00:19:31 +0100 Subject: [PATCH 1/4] Revise default data and cutout bundles --- configs/bundle_config.yaml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 2fa1c74f9..6325ad690 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -170,13 +170,9 @@ databundles: category: common destination: "data" urls: - zenodo: https://sandbox.zenodo.org/record/1183583/files/data_v0.1.zip?download=1 - gdrive: https://drive.google.com/file/d/1jN5iV_iaRa5lc1XOaEtletKPUgPNOVuf/view + zenodo: https://sandbox.zenodo.org/records/4114/files/bundle_data_earth.zip?download=1 + gdrive: https://drive.google.com/file/d/1zU4VRRBDVIVqoykhK9lnOAr60unVm_x2/view?usp=drive_link output: - - data/costs.csv - - data/hydro_capacities.csv - - data/custom_powerplants.csv - - data/eia_hydro_annual_generation.csv - data/eez/eez_v11.gpkg - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif @@ -186,9 +182,6 @@ databundles: - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc - data/ssp2-2.6/2030/era5_2013/Oceania.nc - - data/hydrobasins/hybas_world_lev04_v1c.shp - - data/hydrobasins/hybas_world_lev05_v1c.shp - - data/hydrobasins/hybas_world_lev06_v1c.shp # resources bundle containing the resources folder for Africa only bundle_natura_earth: @@ -321,7 +314,7 @@ databundles: category: cutouts destination: "cutouts" urls: - zenodo: https://sandbox.zenodo.org/record/1223907/files/cutouts_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/records/4114/files/bundle_cutouts_africa.zip?download=1 gdrive: https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV/view?usp=drive_link output: [cutouts/cutout-2013-era5.nc] disable_by_opt: From 9894b6c91e7163753701ac3128c15aeea6765453 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 30 Nov 2023 00:39:42 +0100 Subject: [PATCH 2/4] Improve logging in retrieve_databundle --- scripts/retrieve_databundle_light.py | 29 +++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 580ba20f7..f28331080 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -80,8 +80,6 @@ - ``cutouts``: input data unzipped into the cutouts folder """ -import glob -import logging import os import re from zipfile import ZipFile @@ -352,12 +350,11 @@ def download_and_unpack( os.remove(file_path) logger.info(f"Downloaded resource '{resource}' from cloud '{url}'.") + return True except: logger.warning(f"Failed download resource '{resource}' from cloud '{url}'.") return False - return True - def download_and_unzip_direct(config, rootpath, hot_run=True, disable_progress=False): """ @@ -445,11 +442,13 @@ def download_and_unzip_hydrobasins( level_code = snakemake.config["renewable"]["hydro"]["hydrobasins_level"] level_code = "{:02d}".format(int(level_code)) + all_downloaded = True + for rg in suffix_list: url = url_templ + "hybas_" + rg + "_lev" + level_code + "_v1c.zip" file_path = os.path.join(config["destination"], os.path.basename(url)) - download_and_unpack( + all_downloaded &= download_and_unpack( url=url, file_path=file_path, resource=resource, @@ -460,6 +459,8 @@ def download_and_unzip_hydrobasins( disable_progress=disable_progress, ) + return all_downloaded + def download_and_unzip_post(config, rootpath, hot_run=True, disable_progress=False): """ @@ -742,13 +743,12 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): ] gpdf_list = [None] * len(files_to_merge) - logger.info("Reading hydrobasins files \n\r") + logger.info("Merging hydrobasins files into: " + output_fl) for i, f_name in tqdm(enumerate(files_to_merge)): gpdf_list[i] = gpd.read_file(os.path.join(basins_path, f_name)) fl_merged = gpd.GeoDataFrame(pd.concat(gpdf_list)).drop_duplicates( subset="HYBAS_ID", ignore_index=True ) - logger.info("Merging single files into:\n\t" + output_fl) fl_merged.to_file(output_fl, driver="ESRI Shapefile") @@ -791,6 +791,9 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): logger.info("Bundles to be downloaded:\n\t" + "\n\t".join(bundles_to_download)) + # initialize downloaded and missing bundles + downloaded_bundles = [] + # download the selected bundles for b_name in bundles_to_download: host_list = config_bundles[b_name]["urls"] @@ -811,6 +814,7 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): logger.warning(f"Error in downloading bundle {b_name} - host {host}") if downloaded_bundle: + downloaded_bundles.append(b_name) break if not downloaded_bundle: @@ -826,7 +830,14 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): config_bundles[hydrobasin_bundles[0]], hydrobasins_level ) + # log the downloaded and missing bundles logger.info( - "Bundle successfully loaded and unzipped:\n\t" - + "\n\t".join(bundles_to_download) + "Bundle successfully loaded and unzipped:\n\t" + "\n\t".join(downloaded_bundles) ) + + missing_bundles = set(bundles_to_download) - set(downloaded_bundles) + if missing_bundles: + logger.warning( + "The following bundles could not be downloaded:\n\t" + + "\n\t".join(list(missing_bundles)) + ) From e768026e3522826016d18f4459f67f62263d1dfd Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 30 Nov 2023 10:56:49 +0100 Subject: [PATCH 3/4] add landcover output file in tutorial bundle --- configs/bundle_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 6325ad690..ff980f978 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -149,6 +149,7 @@ databundles: - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc - data/ssp2-2.6/2030/era5_2013/Oceania.nc + - data/landcover/world_protected_area/protected_areas_tutorial.shp # global data for hydrobasins bundle_hydrobasins: From b5204b1dd9fc9fe35e33e287076cd826d1b76e31 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 30 Nov 2023 11:36:24 +0100 Subject: [PATCH 4/4] Add release_notes --- doc/release_notes.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 121c8fd6d..8f5bc803e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -12,7 +12,7 @@ Upcoming release Please add descriptive release notes like in `PyPSA-Eur `__. E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does. -**New Features and major Changes** +**New Features and Major Changes** * Keep all traceback in logs. `PR #898 `__ @@ -22,6 +22,9 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t * Add functionality to load shapefiles for hydrobasins directly from the data source directly `PR #919 `__ +**Minor Changes and bug-fixing** + +* Revise databundles and improve logging in retrieve_databundle `PR #928 `__ PyPSA-Earth 0.2.3 =================