Merge pull request #928 from davide-f/revise_bundles

Revise bundles and improve logging

configs/bundle_config.yaml

@@ -149,6 +149,7 @@ databundles:
     - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc
     - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc
     - data/ssp2-2.6/2030/era5_2013/Oceania.nc
+    - data/landcover/world_protected_area/protected_areas_tutorial.shp
 
   # global data for hydrobasins
   bundle_hydrobasins:
@@ -170,13 +171,9 @@ databundles:
     category: common
     destination: "data"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/data_v0.1.zip?download=1
-      gdrive: https://drive.google.com/file/d/1jN5iV_iaRa5lc1XOaEtletKPUgPNOVuf/view
+      zenodo: https://sandbox.zenodo.org/records/4114/files/bundle_data_earth.zip?download=1
+      gdrive: https://drive.google.com/file/d/1zU4VRRBDVIVqoykhK9lnOAr60unVm_x2/view?usp=drive_link
     output:
-    - data/costs.csv
-    - data/hydro_capacities.csv
-    - data/custom_powerplants.csv
-    - data/eia_hydro_annual_generation.csv
     - data/eez/eez_v11.gpkg
     - data/gebco/GEBCO_2021_TID.nc
     - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif
@@ -186,9 +183,6 @@ databundles:
     - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc
     - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc
     - data/ssp2-2.6/2030/era5_2013/Oceania.nc
-    - data/hydrobasins/hybas_world_lev04_v1c.shp
-    - data/hydrobasins/hybas_world_lev05_v1c.shp
-    - data/hydrobasins/hybas_world_lev06_v1c.shp
 
   # resources bundle containing the resources folder for Africa only
   bundle_natura_earth:
@@ -321,7 +315,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/cutouts_africa.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/4114/files/bundle_cutouts_africa.zip?download=1
       gdrive: https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV/view?usp=drive_link
     output: [cutouts/cutout-2013-era5.nc]
     disable_by_opt:

doc/release_notes.rst

@@ -12,7 +12,7 @@ Upcoming release
 Please add descriptive release notes like in `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur/blob/master/doc/release_notes.rst>`__.
 E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does.
 
-**New Features and major Changes**
+**New Features and Major Changes**
 
 * Keep all traceback in logs. `PR #898 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/898>`__
 
@@ -22,6 +22,9 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t
 
 * Add functionality to load shapefiles for hydrobasins directly from the data source directly `PR #919 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/919>`__
 
+**Minor Changes and bug-fixing**
+
+* Revise databundles and improve logging in retrieve_databundle `PR #928 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/928>`__
 
 PyPSA-Earth 0.2.3
 =================

scripts/retrieve_databundle_light.py

@@ -80,8 +80,6 @@
 - ``cutouts``: input data unzipped into the cutouts folder
 
 """
-import glob
-import logging
 import os
 import re
 from zipfile import ZipFile
@@ -352,12 +350,11 @@ def download_and_unpack(
 
                 os.remove(file_path)
             logger.info(f"Downloaded resource '{resource}' from cloud '{url}'.")
+            return True
         except:
             logger.warning(f"Failed download resource '{resource}' from cloud '{url}'.")
             return False
 
-    return True
-
 
 def download_and_unzip_direct(config, rootpath, hot_run=True, disable_progress=False):
     """
@@ -445,11 +442,13 @@ def download_and_unzip_hydrobasins(
     level_code = snakemake.config["renewable"]["hydro"]["hydrobasins_level"]
     level_code = "{:02d}".format(int(level_code))
 
+    all_downloaded = True
+
     for rg in suffix_list:
         url = url_templ + "hybas_" + rg + "_lev" + level_code + "_v1c.zip"
         file_path = os.path.join(config["destination"], os.path.basename(url))
 
-        download_and_unpack(
+        all_downloaded &= download_and_unpack(
             url=url,
             file_path=file_path,
             resource=resource,
@@ -460,6 +459,8 @@ def download_and_unzip_hydrobasins(
             disable_progress=disable_progress,
         )
 
+    return all_downloaded
+
 
 def download_and_unzip_post(config, rootpath, hot_run=True, disable_progress=False):
     """
@@ -742,13 +743,12 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level):
     ]
 
     gpdf_list = [None] * len(files_to_merge)
-    logger.info("Reading hydrobasins files \n\r")
+    logger.info("Merging hydrobasins files into: " + output_fl)
     for i, f_name in tqdm(enumerate(files_to_merge)):
         gpdf_list[i] = gpd.read_file(os.path.join(basins_path, f_name))
     fl_merged = gpd.GeoDataFrame(pd.concat(gpdf_list)).drop_duplicates(
         subset="HYBAS_ID", ignore_index=True
     )
-    logger.info("Merging single files into:\n\t" + output_fl)
     fl_merged.to_file(output_fl, driver="ESRI Shapefile")
 
 
@@ -791,6 +791,9 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level):
 
     logger.info("Bundles to be downloaded:\n\t" + "\n\t".join(bundles_to_download))
 
+    # initialize downloaded and missing bundles
+    downloaded_bundles = []
+
     # download the selected bundles
     for b_name in bundles_to_download:
         host_list = config_bundles[b_name]["urls"]
@@ -811,6 +814,7 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level):
                 logger.warning(f"Error in downloading bundle {b_name} - host {host}")
 
             if downloaded_bundle:
+                downloaded_bundles.append(b_name)
                 break
 
         if not downloaded_bundle:
@@ -826,7 +830,14 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level):
             config_bundles[hydrobasin_bundles[0]], hydrobasins_level
         )
 
+    # log the downloaded and missing bundles
     logger.info(
-        "Bundle successfully loaded and unzipped:\n\t"
-        + "\n\t".join(bundles_to_download)
+        "Bundle successfully loaded and unzipped:\n\t" + "\n\t".join(downloaded_bundles)
     )
+
+    missing_bundles = set(bundles_to_download) - set(downloaded_bundles)
+    if missing_bundles:
+        logger.warning(
+            "The following bundles could not be downloaded:\n\t"
+            + "\n\t".join(list(missing_bundles))
+        )