From 9ef492b8a032ca5aa17e2501ff6c5facddd31afb Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 00:47:32 +0100 Subject: [PATCH 01/39] obsolete Centroids.set_meta_to_lat_lon use hdf5 centroids test data assign region_id to geodataframe, not centroids --- climada_petals/hazard/emulator/geo.py | 1 - climada_petals/hazard/relative_cropyield.py | 3 +-- climada_petals/hazard/tc_rainfield.py | 4 ---- climada_petals/hazard/tc_surge_bathtub.py | 3 --- climada_petals/hazard/test/test_tc_rainfield.py | 8 ++++---- climada_petals/hazard/wildfire.py | 7 ------- 6 files changed, 5 insertions(+), 21 deletions(-) diff --git a/climada_petals/hazard/emulator/geo.py b/climada_petals/hazard/emulator/geo.py index 23ddf030c..28ef9062a 100644 --- a/climada_petals/hazard/emulator/geo.py +++ b/climada_petals/hazard/emulator/geo.py @@ -103,7 +103,6 @@ def centroids(self, latlon=None, res_as=360): """ if latlon is None: centroids = Centroids.from_base_grid(res_as=res_as) - centroids.set_meta_to_lat_lon() else: centroids = Centroids.from_lat_lon(*latlon) msk = shapely.vectorized.contains(self.shape, centroids.lon, centroids.lat) diff --git a/climada_petals/hazard/relative_cropyield.py b/climada_petals/hazard/relative_cropyield.py index e5002e55c..e42d2d6a2 100644 --- a/climada_petals/hazard/relative_cropyield.py +++ b/climada_petals/hazard/relative_cropyield.py @@ -246,8 +246,7 @@ def from_isimip_netcdf(cls, input_dir=None, filename=None, bbox=None, haz.units = 't / y / ha' haz.date = np.array(dt.str_to_date( [event_ + '-01-01' for event_ in haz.event_name])) - haz.centroids.set_meta_to_lat_lon() - haz.centroids.region_id = ( + haz.centroids.gdf['region_id'] = ( coord.coord_on_land(haz.centroids.lat, haz.centroids.lon)).astype(dtype=int) haz.check() return haz diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index ec774dddf..6cb11dc79 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -399,10 +399,6 @@ def from_tracks( num_tracks = tracks.size if centroids is None: centroids = Centroids.from_base_grid(res_as=360, land=True) - - if not centroids.coord.size: - centroids.set_meta_to_lat_lon() - if ignore_distance_to_coast: # Select centroids with lat <= max_latitude [idx_centr_filter] = (np.abs(centroids.lat) <= max_latitude).nonzero() diff --git a/climada_petals/hazard/tc_surge_bathtub.py b/climada_petals/hazard/tc_surge_bathtub.py index bfa30779c..309bcc843 100644 --- a/climada_petals/hazard/tc_surge_bathtub.py +++ b/climada_petals/hazard/tc_surge_bathtub.py @@ -72,9 +72,6 @@ def from_tc_winds(wind_haz, topo_path, inland_decay_rate=0.2, add_sea_level_rise """ centroids = copy.deepcopy(wind_haz.centroids) - if not centroids.coord.size: - centroids.set_meta_to_lat_lon() - # Select wind-affected centroids which are inside MAX_DIST_COAST and |lat| < 61 if not centroids.dist_coast.size or np.all(centroids.dist_coast >= 0): centroids.set_dist_coast(signed=True, precomputed=True) diff --git a/climada_petals/hazard/test/test_tc_rainfield.py b/climada_petals/hazard/test/test_tc_rainfield.py index 895362db3..1d515d307 100644 --- a/climada_petals/hazard/test/test_tc_rainfield.py +++ b/climada_petals/hazard/test/test_tc_rainfield.py @@ -45,12 +45,12 @@ def getTestData(): client = Client() - centr_ds = client.get_dataset_info(name='test_tc_rainfield', status='test_dataset') - _, [centr_test_mat, track, track_short, haz_mat] = client.download_dataset(centr_ds) - return Centroids.from_mat(centr_test_mat), track, track_short, haz_mat + centr_ds = client.get_dataset_info(name='tc_rainfield_test', status='test_dataset') + _, [centr_test_mat, track, track_short, haz_hdf5] = client.download_dataset(centr_ds) + return Centroids.from_hdf5(centr_test_mat), track, track_short, haz_hdf5 -CENTR_TEST_BRB, TEST_TRACK, TEST_TRACK_SHORT, HAZ_TEST_MAT = getTestData() +CENTR_TEST_BRB, TEST_TRACK, TEST_TRACK_SHORT, HAZ_TEST_HDF5 = getTestData() def tcrain_examples(): diff --git a/climada_petals/hazard/wildfire.py b/climada_petals/hazard/wildfire.py index cca16d246..074b7ea8f 100644 --- a/climada_petals/hazard/wildfire.py +++ b/climada_petals/hazard/wildfire.py @@ -168,9 +168,6 @@ def from_hist_fire_FIRMS(cls, df_firms, centr_res_factor=1.0, centroids=None): res_data = haz._firms_resolution(df_firms) if not centroids: centroids = haz._firms_centroids_creation(df_firms, res_data, centr_res_factor) - else: - if not centroids.lat.any(): - centroids.set_meta_to_lat_lon() res_centr = haz._centroids_resolution(centroids) # fire identification @@ -259,9 +256,6 @@ def from_hist_fire_seasons_FIRMS(cls, df_firms, centr_res_factor=1.0, res_data = haz._firms_resolution(df_firms) if not centroids: centroids = haz._firms_centroids_creation(df_firms, res_data, centr_res_factor) - else: - if not centroids.coord.size: - centroids.set_meta_to_lat_lon() # define hemisphere if hemisphere is None: @@ -664,7 +658,6 @@ def _firms_centroids_creation(df_firms, res_data, centr_res_factor): centroids = Centroids.from_pnt_bounds((df_firms['longitude'].min(), \ df_firms['latitude'].min(), df_firms['longitude'].max(), \ df_firms['latitude'].max()), res=res_data/centr_res_factor) - centroids.set_meta_to_lat_lon() centroids.set_area_approx() centroids.set_on_land() centroids.empty_geometry_points() From b083f5d65c7147a696ef5681228510d1e0b1d5bf Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 00:51:38 +0100 Subject: [PATCH 02/39] obsolete Centroids.set_meta_to_lat_lon --- climada_petals/hazard/low_flow.py | 1 - climada_petals/hazard/relative_cropyield.py | 2 -- climada_petals/hazard/river_flood.py | 5 ----- 3 files changed, 8 deletions(-) diff --git a/climada_petals/hazard/low_flow.py b/climada_petals/hazard/low_flow.py index ec686d7f3..066f87631 100644 --- a/climada_petals/hazard/low_flow.py +++ b/climada_petals/hazard/low_flow.py @@ -645,7 +645,6 @@ def _init_centroids(dis_xarray, centr_res_factor=1): (dis_xarray.lon.values.min(), dis_xarray.lat.values.min(), dis_xarray.lon.values.max(), dis_xarray.lat.values.max()), res=res_data / centr_res_factor) - centroids.set_meta_to_lat_lon() centroids.set_area_approx() centroids.set_on_land() centroids.empty_geometry_points() diff --git a/climada_petals/hazard/relative_cropyield.py b/climada_petals/hazard/relative_cropyield.py index e42d2d6a2..66de8a11f 100644 --- a/climada_petals/hazard/relative_cropyield.py +++ b/climada_petals/hazard/relative_cropyield.py @@ -370,8 +370,6 @@ def plot_time_series(self, event=None): else: event = [str(n) for n in range(event[0], event[1] + 1)] - self.centroids.set_meta_to_lat_lon() - # definition of plot extents len_lat = abs(self.centroids.lat[0] - self.centroids.lat[-1]) * (2.5 / 13.5) len_lon = abs(self.centroids.lon[0] - self.centroids.lon[-1]) * (5 / 26) diff --git a/climada_petals/hazard/river_flood.py b/climada_petals/hazard/river_flood.py index 35f7e5587..588e15638 100644 --- a/climada_petals/hazard/river_flood.py +++ b/climada_petals/hazard/river_flood.py @@ -166,14 +166,12 @@ def from_nc(cls, dph_path=None, frc_path=None, origin=False, files_fraction=[frc_path], band=bands.tolist(), geometry=cntry_geom.geoms) - # self.centroids.set_meta_to_lat_lon() else: cntry_geom = u_coord.get_land_geometry(countries) haz = cls.from_raster(files_intensity=[dph_path], files_fraction=[frc_path], band=bands.tolist(), geometry=cntry_geom.geoms) - # self.centroids.set_meta_to_lat_lon() elif shape: shapes = gpd.read_file(shape) @@ -195,7 +193,6 @@ def from_nc(cls, dph_path=None, frc_path=None, origin=False, haz = cls.from_raster(files_intensity=[dph_path], files_fraction=[frc_path], band=bands.tolist()) - # self.centroids.set_meta_to_lat_lon() else: # use given centroids # if centroids.meta or grid_is_regular(centroids)[0]: @@ -206,8 +203,6 @@ def from_nc(cls, dph_path=None, frc_path=None, origin=False, # (transform) # reprojection change resampling""" # else: - if centroids.meta: - centroids.set_meta_to_lat_lon() metafrc, fraction = u_coord.read_raster(frc_path, band=bands.tolist()) metaint, intensity = u_coord.read_raster(dph_path, band=bands.tolist()) x_i = ((centroids.lon - metafrc['transform'][2]) / From 35a0d437460b5ede84c1abc3e002e02a9f86c83f Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 09:20:44 +0100 Subject: [PATCH 03/39] jenkinsfile alternative reference environment --- script/jenkins/branches/Jenkinsfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 5303137bb..6993b9e28 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -25,7 +25,23 @@ pipeline { source activate petals_env rm -rf tests_xml/ rm -rf coverage/ - make unit_test''' + + CORENV=~/jobs/petals_branches/core_env + BRANCH=`git name-rev --name-only HEAD | cut -f 3- -d /` + if [ -f $CORENV/$BRANCH ]; then + python -m venv --system-site-packages tvenv + source tvenv/bin/activate + + pip install -e `cat $CORENV/$BRANCH` + fi + + make unit_test + + if [ -f $CORENV/$BRANCH ]; then + deactivate + rm -r tvenv + fi + ''' } } From d267b14927a13b86efb9a0afbe64a3e6b277fe2b Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 09:37:39 +0100 Subject: [PATCH 04/39] test --- script/jenkins/branches/Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 6993b9e28..0358fa20b 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -11,7 +11,8 @@ pipeline { export PATH=$PATH:$CONDAPATH source activate petals_env rm -f pylint.log - pylint -ry climada_petals | tee pylint.log''' + #pylint -ry climada_petals | tee pylint.log + ''' discoverGitReferenceBuild referenceJob: 'petals_branches/develop' recordIssues tools: [pyLint(pattern: 'pylint.log')] @@ -28,6 +29,7 @@ pipeline { CORENV=~/jobs/petals_branches/core_env BRANCH=`git name-rev --name-only HEAD | cut -f 3- -d /` + echo $CORENV/$BRANCH if [ -f $CORENV/$BRANCH ]; then python -m venv --system-site-packages tvenv source tvenv/bin/activate @@ -35,7 +37,7 @@ pipeline { pip install -e `cat $CORENV/$BRANCH` fi - make unit_test + #make unit_test if [ -f $CORENV/$BRANCH ]; then deactivate From 0c4b73cd1a57da6cce6c80022226c11ce46b8713 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 09:42:23 +0100 Subject: [PATCH 05/39] test --- script/jenkins/branches/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 0358fa20b..deb8091e3 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -41,7 +41,7 @@ pipeline { if [ -f $CORENV/$BRANCH ]; then deactivate - rm -r tvenv + #rm -r tvenv fi ''' } From 8f24efc5a854b47b7cae509af336c373e46d9d5e Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 10:00:46 +0100 Subject: [PATCH 06/39] test --- script/jenkins/branches/Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index deb8091e3..70224be8b 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { export PATH=$PATH:$CONDAPATH source activate petals_env rm -f pylint.log - #pylint -ry climada_petals | tee pylint.log + pylint -ry climada_petals | tee pylint.log ''' discoverGitReferenceBuild referenceJob: 'petals_branches/develop' @@ -29,7 +29,6 @@ pipeline { CORENV=~/jobs/petals_branches/core_env BRANCH=`git name-rev --name-only HEAD | cut -f 3- -d /` - echo $CORENV/$BRANCH if [ -f $CORENV/$BRANCH ]; then python -m venv --system-site-packages tvenv source tvenv/bin/activate @@ -37,7 +36,7 @@ pipeline { pip install -e `cat $CORENV/$BRANCH` fi - #make unit_test + make unit_test if [ -f $CORENV/$BRANCH ]; then deactivate From b50c54e91c848143cbab2388a95c32becf1b9a8c Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 10:32:57 +0100 Subject: [PATCH 07/39] test --- script/jenkins/branches/Jenkinsfile | 23 +---------------------- script/jenkins/branches/run_unittest.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 22 deletions(-) create mode 100644 script/jenkins/branches/run_unittest.sh diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 70224be8b..36ac7c1f4 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -21,28 +21,7 @@ pipeline { stage('unit_test') { steps { - sh '''#!/bin/bash - export PATH=$PATH:$CONDAPATH - source activate petals_env - rm -rf tests_xml/ - rm -rf coverage/ - - CORENV=~/jobs/petals_branches/core_env - BRANCH=`git name-rev --name-only HEAD | cut -f 3- -d /` - if [ -f $CORENV/$BRANCH ]; then - python -m venv --system-site-packages tvenv - source tvenv/bin/activate - - pip install -e `cat $CORENV/$BRANCH` - fi - - make unit_test - - if [ -f $CORENV/$BRANCH ]; then - deactivate - #rm -r tvenv - fi - ''' + sh script/jenkins/branches/run_unittest.sh } } diff --git a/script/jenkins/branches/run_unittest.sh b/script/jenkins/branches/run_unittest.sh new file mode 100644 index 000000000..10850d1ce --- /dev/null +++ b/script/jenkins/branches/run_unittest.sh @@ -0,0 +1,21 @@ +#!/bin/bash +export PATH=$PATH:$CONDAPATH +source activate petals_env +rm -rf tests_xml/ +rm -rf coverage/ + +CORENV=~/jobs/petals_branches/core_env +BRANCH=`git name-rev --name-only HEAD | cut -f 3- -d /` +if [ -f $CORENV/$BRANCH ]; then + python -m venv --system-site-packages tvenv + source tvenv/bin/activate + + pip install -e `cat $CORENV/$BRANCH` +fi + +make unit_test + +if [ -f $CORENV/$BRANCH ]; then + deactivate + #rm -r tvenv +fi \ No newline at end of file From 9725657f45d8bc1d6f76486e2105cc3fac95cec0 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 12:44:31 +0100 Subject: [PATCH 08/39] make wildfire pass tests again --- climada_petals/hazard/wildfire.py | 9 ++++++--- script/jenkins/branches/run_lint.sh | 7 +++++++ script/jenkins/branches/run_unittest.sh | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 script/jenkins/branches/run_lint.sh diff --git a/climada_petals/hazard/wildfire.py b/climada_petals/hazard/wildfire.py index 074b7ea8f..dc10561c8 100644 --- a/climada_petals/hazard/wildfire.py +++ b/climada_petals/hazard/wildfire.py @@ -677,9 +677,12 @@ def _centroids_resolution(centroids): res_centr : float grid resolution of centroids """ - if centroids.meta: - res_centr = abs(centroids.meta['transform'][4]), \ - centroids.meta['transform'][0] + # TODO: centroids as gdf + # proper implementation. this one is a mere petch to make it run in climada 5.0 + # centroids.get_meta() does not check for being a raster, as centroids.meta did + if centroids.get_meta(): + res_centr = abs(centroids.get_meta()['transform'][4]), \ + centroids.get_meta()['transform'][0] else: res_centr, _ = u_coord.get_resolution(centroids.lat, centroids.lon) if abs(abs(res_centr[0]) - abs(res_centr[1])) > 1.0e-6: diff --git a/script/jenkins/branches/run_lint.sh b/script/jenkins/branches/run_lint.sh new file mode 100644 index 000000000..c8306b6e7 --- /dev/null +++ b/script/jenkins/branches/run_lint.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export PATH=$PATH:$CONDAPATH +source activate petals_env + +rm -f pylint.log + +pylint -ry climada_petals | tee pylint.log diff --git a/script/jenkins/branches/run_unittest.sh b/script/jenkins/branches/run_unittest.sh index 10850d1ce..a35eabc82 100644 --- a/script/jenkins/branches/run_unittest.sh +++ b/script/jenkins/branches/run_unittest.sh @@ -18,4 +18,4 @@ make unit_test if [ -f $CORENV/$BRANCH ]; then deactivate #rm -r tvenv -fi \ No newline at end of file +fi From c4beb8c1ebfe3b087cc711f8fab0fd0a224ec5b2 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 13:17:03 +0100 Subject: [PATCH 09/39] test_wildfire: adapt test to new centroids --- climada_petals/hazard/test/test_wildfire.py | 27 +++++++++++---------- script/jenkins/branches/Jenkinsfile | 10 ++------ script/jenkins/branches/run_unittest.sh | 1 + 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/climada_petals/hazard/test/test_wildfire.py b/climada_petals/hazard/test/test_wildfire.py index c4ae6bacb..a12adaa6e 100644 --- a/climada_petals/hazard/test/test_wildfire.py +++ b/climada_petals/hazard/test/test_wildfire.py @@ -274,20 +274,21 @@ def test_centroids_pass(self): wf = WildFire() firms = wf._clean_firms_df(TEST_FIRMS) centroids = wf._firms_centroids_creation(firms, 0.375/ONE_LAT_KM, 1/2) - self.assertEqual(centroids.meta['width'], 144) - self.assertEqual(centroids.meta['height'], 138) - self.assertAlmostEqual(centroids.meta['transform'][0], 0.006749460043196544) - self.assertAlmostEqual(centroids.meta['transform'][1], 0.0) - self.assertTrue(centroids.meta['transform'][2]<= firms.longitude.min()) - self.assertAlmostEqual(centroids.meta['transform'][3], 0.0) - self.assertAlmostEqual(centroids.meta['transform'][4], -centroids.meta['transform'][0]) - self.assertTrue(centroids.meta['transform'][5] >= firms.latitude.max()) - self.assertTrue(firms.latitude.max() <= centroids.total_bounds[3]) - self.assertTrue(firms.latitude.min() >= centroids.total_bounds[1]) - self.assertTrue(firms.longitude.max() <= centroids.total_bounds[2]) - self.assertTrue(firms.longitude.min() >= centroids.total_bounds[0]) + centroids_meta = centroids.get_meta() + self.assertEqual(centroids_meta['width'], 144) + self.assertEqual(centroids_meta['height'], 138) + self.assertAlmostEqual(centroids_meta['transform'][0], 0.006749460043196544) + self.assertAlmostEqual(centroids_meta['transform'][1], 0.0) + self.assertLessEqual(centroids_meta['transform'][2], firms.longitude.min()) + self.assertAlmostEqual(centroids_meta['transform'][3], 0.0) + self.assertAlmostEqual(centroids_meta['transform'][4], -centroids_meta['transform'][0]) + self.assertGreaterEqual(centroids_meta['transform'][5], firms.latitude.max()) + self.assertLessEqual(firms.latitude.max(), centroids.total_bounds[3]) + self.assertGreaterEqual(firms.latitude.min(), centroids.total_bounds[1]) + self.assertLessEqual(firms.longitude.max(), centroids.total_bounds[2]) + self.assertGreaterEqual(firms.longitude.min(), centroids.total_bounds[0]) self.assertTrue(centroids.lat.size) - self.assertTrue(centroids.area_pixel.size) + self.assertTrue(centroids.get_area_pixel().size) self.assertTrue(centroids.on_land.size) def test_firms_resolution_pass(self): diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 36ac7c1f4..a6793eeef 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -7,13 +7,7 @@ pipeline { stage('lint') { steps { - sh '''#!/bin/bash - export PATH=$PATH:$CONDAPATH - source activate petals_env - rm -f pylint.log - pylint -ry climada_petals | tee pylint.log - ''' - + sh 'bash script/jeknins/branches/run_lint.sh' discoverGitReferenceBuild referenceJob: 'petals_branches/develop' recordIssues tools: [pyLint(pattern: 'pylint.log')] } @@ -21,7 +15,7 @@ pipeline { stage('unit_test') { steps { - sh script/jenkins/branches/run_unittest.sh + sh 'bash script/jenkins/branches/run_unittest.sh' } } diff --git a/script/jenkins/branches/run_unittest.sh b/script/jenkins/branches/run_unittest.sh index a35eabc82..e2c23df47 100644 --- a/script/jenkins/branches/run_unittest.sh +++ b/script/jenkins/branches/run_unittest.sh @@ -1,5 +1,6 @@ #!/bin/bash export PATH=$PATH:$CONDAPATH + source activate petals_env rm -rf tests_xml/ rm -rf coverage/ From 23af185d8871498c5a65f66c24ce0e9e2091b987 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 13:43:13 +0100 Subject: [PATCH 10/39] river_flood: blindly adapt to new centroids --- climada_petals/hazard/river_flood.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/climada_petals/hazard/river_flood.py b/climada_petals/hazard/river_flood.py index 588e15638..97c3568e1 100644 --- a/climada_petals/hazard/river_flood.py +++ b/climada_petals/hazard/river_flood.py @@ -137,22 +137,22 @@ def from_nc(cls, dph_path=None, frc_path=None, origin=False, if ISINatIDGrid: dest_centroids = RiverFlood._select_exact_area(countries, reg)[0] - meta_centroids = copy.copy(dest_centroids) - meta_centroids.set_lat_lon_to_meta() + centroids_meta = dest_centroids.get_meta() haz = cls.from_raster(files_intensity=[dph_path], files_fraction=[frc_path], band=bands.tolist(), - transform=meta_centroids.meta['transform'], - width=meta_centroids.meta['width'], - height=meta_centroids.meta['height'], + transform=centroids_meta['transform'], + width=centroids_meta['width'], + height=centroids_meta['height'], resampling=Resampling.nearest) - x_i = ((dest_centroids.lon - haz.centroids.meta['transform'][2]) / - haz.centroids.meta['transform'][0]).astype(int) - y_i = ((dest_centroids.lat - haz.centroids.meta['transform'][5]) / - haz.centroids.meta['transform'][4]).astype(int) + haz_centroids_meta = haz.centroids.get_meta() + x_i = ((dest_centroids.lon - haz_centroids_meta['transform'][2]) / + haz_centroids_meta['transform'][0]).astype(int) + y_i = ((dest_centroids.lat - haz_centroids_meta['transform'][5]) / + haz_centroids_meta['transform'][4]).astype(int) - fraction = haz.fraction[:, y_i * haz.centroids.meta['width'] + x_i] - intensity = haz.intensity[:, y_i * haz.centroids.meta['width'] + x_i] + fraction = haz.fraction[:, y_i * haz_centroids_meta['width'] + x_i] + intensity = haz.intensity[:, y_i * haz_centroids_meta['width'] + x_i] haz.centroids = dest_centroids haz.intensity = sp.sparse.csr_matrix(intensity) @@ -316,7 +316,7 @@ def set_flooded_area(self, save_centr=False): MemoryError """ self.centroids.set_area_pixel() - area_centr = self.centroids.area_pixel + area_centr = self.centroids.get_area_pixel() event_years = np.array([dt.date.fromordinal(self.date[i]).year for i in range(len(self.date))]) years = np.unique(event_years) From ee173c3ea8ffe75d024e00038533e16f218a240e Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 14:17:33 +0100 Subject: [PATCH 11/39] climada_petals.hazard.TCRainField: adapt to new centroids --- climada_petals/hazard/tc_rainfield.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index 6cb11dc79..b86b4cca6 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -404,10 +404,8 @@ def from_tracks( [idx_centr_filter] = (np.abs(centroids.lat) <= max_latitude).nonzero() else: # Select centroids which are inside max_dist_inland_km and lat <= max_latitude - if not centroids.dist_coast.size: - centroids.set_dist_coast() [idx_centr_filter] = ( - (centroids.dist_coast <= max_dist_inland_km * 1000) + (centroids.get_dist_coast() <= max_dist_inland_km * 1000) & (np.abs(centroids.lat) <= max_latitude) ).nonzero() From 1be80a8f427fe4998e8991ef58dae43b4f38a4f1 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 14:36:44 +0100 Subject: [PATCH 12/39] hazard.low_flow: centroids resolution in any case determined with util.coordinates.get_resolution --- climada_petals/hazard/low_flow.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/climada_petals/hazard/low_flow.py b/climada_petals/hazard/low_flow.py index 066f87631..a017ee5f1 100644 --- a/climada_petals/hazard/low_flow.py +++ b/climada_petals/hazard/low_flow.py @@ -539,11 +539,7 @@ def _centroids_resolution(centroids): ------- float """ - if centroids.meta: - res_centr = abs(centroids.meta['transform'][4]), \ - centroids.meta['transform'][0] - else: - res_centr = np.abs(u_coord.get_resolution(centroids.lat, centroids.lon)) + res_centr = np.abs(u_coord.get_resolution(centroids.lat, centroids.lon)) if np.abs(res_centr[0] - res_centr[1]) > 1.0e-6: LOGGER.warning('Centroids do not represent regular pixels %s.', str(res_centr)) return (res_centr[0] + res_centr[1]) / 2 From bf47977c47f4ca33bf7330da069e706cb67f79f2 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 14:38:16 +0100 Subject: [PATCH 13/39] hazard.wildfire: centroids resolution in any case determined with util.coordinates.get_resolution --- climada_petals/hazard/wildfire.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/climada_petals/hazard/wildfire.py b/climada_petals/hazard/wildfire.py index dc10561c8..71e12ae25 100644 --- a/climada_petals/hazard/wildfire.py +++ b/climada_petals/hazard/wildfire.py @@ -677,14 +677,7 @@ def _centroids_resolution(centroids): res_centr : float grid resolution of centroids """ - # TODO: centroids as gdf - # proper implementation. this one is a mere petch to make it run in climada 5.0 - # centroids.get_meta() does not check for being a raster, as centroids.meta did - if centroids.get_meta(): - res_centr = abs(centroids.get_meta()['transform'][4]), \ - centroids.get_meta()['transform'][0] - else: - res_centr, _ = u_coord.get_resolution(centroids.lat, centroids.lon) + res_centr, _ = u_coord.get_resolution(centroids.lat, centroids.lon) if abs(abs(res_centr[0]) - abs(res_centr[1])) > 1.0e-6: raise ValueError('Centroids are not a regular raster') return res_centr[0] From 656920d2fd30c28fc705aad1a96117f74d2db20c Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 14:40:19 +0100 Subject: [PATCH 14/39] Jenkinsfile: fix typo --- script/jenkins/branches/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index a6793eeef..5616ae5a6 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -7,7 +7,7 @@ pipeline { stage('lint') { steps { - sh 'bash script/jeknins/branches/run_lint.sh' + sh 'bash script/jenkins/branches/run_lint.sh' discoverGitReferenceBuild referenceJob: 'petals_branches/develop' recordIssues tools: [pyLint(pattern: 'pylint.log')] } From 95d117733afb3e685ef3d541cbcaba74190dc43f Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 15:05:15 +0100 Subject: [PATCH 15/39] pylint --- climada_petals/hazard/river_flood.py | 1 - 1 file changed, 1 deletion(-) diff --git a/climada_petals/hazard/river_flood.py b/climada_petals/hazard/river_flood.py index 97c3568e1..006d0c8c0 100644 --- a/climada_petals/hazard/river_flood.py +++ b/climada_petals/hazard/river_flood.py @@ -23,7 +23,6 @@ import logging import datetime as dt -import copy from collections.abc import Iterable from pathlib import Path From d216be2c409f2e81f5cad444fb7b3a0f8349d80c Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 15:55:19 +0100 Subject: [PATCH 16/39] hazard.tc_surge_bathtub: adapt to new centroids --- climada_petals/hazard/tc_surge_bathtub.py | 23 ++++++------------- .../hazard/test/test_tc_surge_bathtub.py | 8 +++---- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/climada_petals/hazard/tc_surge_bathtub.py b/climada_petals/hazard/tc_surge_bathtub.py index 309bcc843..bb43b7737 100644 --- a/climada_petals/hazard/tc_surge_bathtub.py +++ b/climada_petals/hazard/tc_surge_bathtub.py @@ -73,11 +73,10 @@ def from_tc_winds(wind_haz, topo_path, inland_decay_rate=0.2, add_sea_level_rise centroids = copy.deepcopy(wind_haz.centroids) # Select wind-affected centroids which are inside MAX_DIST_COAST and |lat| < 61 - if not centroids.dist_coast.size or np.all(centroids.dist_coast >= 0): - centroids.set_dist_coast(signed=True, precomputed=True) + centroids_dist_coast = centroids.get_dist_coast(signed=True, precomputed=True) coastal_msk = (wind_haz.intensity > 0).sum(axis=0).A1 > 0 - coastal_msk &= (centroids.dist_coast < 0) - coastal_msk &= (centroids.dist_coast >= -MAX_DIST_COAST * 1000) + coastal_msk &= (centroids_dist_coast < 0) + coastal_msk &= (centroids_dist_coast >= -MAX_DIST_COAST * 1000) coastal_msk &= (np.abs(centroids.lat) <= MAX_LATITUDE) # Load elevation at coastal centroids @@ -114,7 +113,7 @@ def from_tc_winds(wind_haz, topo_path, inland_decay_rate=0.2, add_sea_level_rise if inland_decay_rate != 0: # Add decay according to distance from coast - dist_coast_km = np.abs(centroids.dist_coast[coastal_idx]) / 1000 + dist_coast_km = np.abs(centroids_dist_coast[coastal_idx]) / 1000 coastal_centroids_h += inland_decay_rate * dist_coast_km coastal_centroids_h -= add_sea_level_rise @@ -165,12 +164,9 @@ def _fraction_on_land(centroids, topo_path): """ bounds = np.array(centroids.total_bounds) shape = [0, 0] - if centroids.meta: - shape = centroids.shape - cen_trans = centroids.meta['transform'] - else: - shape[0], shape[1], cen_trans = u_coord.pts_to_raster_meta( - bounds, min(u_coord.get_resolution(centroids.lat, centroids.lon))) + shape[0], shape[1], cen_trans = u_coord.pts_to_raster_meta( + points_bounds=bounds, + res=min(u_coord.get_resolution(centroids.lat, centroids.lon))) read_raster_buffer = 0.5 * max(np.abs(cen_trans[0]), np.abs(cen_trans[4])) bounds += read_raster_buffer * np.array([-1., -1., 1., 1.]) @@ -188,9 +184,4 @@ def _fraction_on_land(centroids, topo_path): resampling=rasterio.warp.Resampling.average, src_nodata=dem_nodata, dst_nodata=0.0) - if not centroids.meta: - x_i = ((centroids.lon - cen_trans[2]) / cen_trans[0]).astype(int) - y_i = ((centroids.lat - cen_trans[5]) / cen_trans[4]).astype(int) - fractions = fractions[y_i, x_i] - return fractions.reshape(-1) diff --git a/climada_petals/hazard/test/test_tc_surge_bathtub.py b/climada_petals/hazard/test/test_tc_surge_bathtub.py index a56574181..6bc11c2cd 100644 --- a/climada_petals/hazard/test/test_tc_surge_bathtub.py +++ b/climada_petals/hazard/test/test_tc_surge_bathtub.py @@ -49,8 +49,7 @@ def __init__(self, bounds, res_deg, slope=0.006): self.shape = (lat.size, lon.size) self.transform = rasterio.Affine(res_deg, 0, bounds[0], 0, -res_deg, bounds[3]) centroids = Centroids.from_lat_lon(*[ar.ravel() for ar in np.meshgrid(lon, lat)][::-1]) - centroids.set_dist_coast(signed=True, precomputed=True) - self.dist_coast = centroids.dist_coast + self.dist_coast = centroids.get_dist_coast(signed=True, precomputed=True) self.slope = slope def __enter__(self): @@ -95,14 +94,13 @@ def test_fraction_on_land(self): shape = (lat.size, lon.size) lon, lat = [ar.ravel() for ar in np.meshgrid(lon, lat)] centroids = Centroids.from_lat_lon(lat, lon) - centroids.set_dist_coast(signed=True, precomputed=True) - + dem_bounds = (bounds[0] - 1, bounds[1] - 1, bounds[2] + 1, bounds[3] + 1) dem_res = 3 / (60 * 60) with tmp_artifical_topo(dem_bounds, dem_res) as topo_path: fraction = _fraction_on_land(centroids, topo_path) fraction = fraction.reshape(shape) - dist_coast = centroids.dist_coast.reshape(shape) + dist_coast = centroids.get_dist_coast(signed=True, precomputed=True).reshape(shape) # check valid range and order of magnitude self.assertTrue(np.all((fraction >= 0) & (fraction <= 1))) From 9257bf0efe9c314fc23a740b803b399abebfde9b Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 23 Feb 2024 19:38:02 +0100 Subject: [PATCH 17/39] hazard.wildfire.from_hist_fire: fix util.get_resolution returns (x,y) pair not ((x,y), z) --- climada_petals/hazard/wildfire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/climada_petals/hazard/wildfire.py b/climada_petals/hazard/wildfire.py index 71e12ae25..5d5fc9f46 100644 --- a/climada_petals/hazard/wildfire.py +++ b/climada_petals/hazard/wildfire.py @@ -677,7 +677,7 @@ def _centroids_resolution(centroids): res_centr : float grid resolution of centroids """ - res_centr, _ = u_coord.get_resolution(centroids.lat, centroids.lon) + res_centr = u_coord.get_resolution(centroids.lat, centroids.lon) if abs(abs(res_centr[0]) - abs(res_centr[1])) > 1.0e-6: raise ValueError('Centroids are not a regular raster') return res_centr[0] From 6f0d924c83a1152c8e15d9e8b1e1f1258696190e Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 15 Mar 2024 13:55:08 +0100 Subject: [PATCH 18/39] wildfire._centroids_resolution: make sure it's >= 0 --- climada_petals/hazard/wildfire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/climada_petals/hazard/wildfire.py b/climada_petals/hazard/wildfire.py index 5d5fc9f46..a74720119 100644 --- a/climada_petals/hazard/wildfire.py +++ b/climada_petals/hazard/wildfire.py @@ -680,7 +680,7 @@ def _centroids_resolution(centroids): res_centr = u_coord.get_resolution(centroids.lat, centroids.lon) if abs(abs(res_centr[0]) - abs(res_centr[1])) > 1.0e-6: raise ValueError('Centroids are not a regular raster') - return res_centr[0] + return abs(res_centr[0]) def _firms_cons_days(self, df_firms): """ Compute clusters of consecutive days (temporal clusters). From a76aaa6ea97fe42028309f462bb34510182c4c26 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 15 Mar 2024 14:46:02 +0100 Subject: [PATCH 19/39] petals: eliminate Centroids.from_base_grid calls --- climada_petals/hazard/emulator/geo.py | 4 +++- climada_petals/hazard/tc_rainfield.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/climada_petals/hazard/emulator/geo.py b/climada_petals/hazard/emulator/geo.py index 28ef9062a..f4dbcb52a 100644 --- a/climada_petals/hazard/emulator/geo.py +++ b/climada_petals/hazard/emulator/geo.py @@ -27,6 +27,7 @@ from shapely.geometry import Polygon from climada.hazard import Centroids +from climada.util.constants import NATEARTH_CENTROIDS import climada.util.coordinates as u_coord import climada_petals.hazard.emulator.const as const @@ -102,7 +103,8 @@ def centroids(self, latlon=None, res_as=360): centroids : climada.hazard.Centroids object """ if latlon is None: - centroids = Centroids.from_base_grid(res_as=res_as) + # default centroids: Natural Earth data at given resolution + centroids = Centroids.from_hdf5(NATEARTH_CENTROIDS[res_as]) else: centroids = Centroids.from_lat_lon(*latlon) msk = shapely.vectorized.contains(self.shape, centroids.lon, centroids.lat) diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index b86b4cca6..43e5adb8f 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -398,7 +398,12 @@ def from_tracks( """ num_tracks = tracks.size if centroids is None: - centroids = Centroids.from_base_grid(res_as=360, land=True) + # default centroids: Natural Earth data at resolution 360 + centroids = Centroids.from_hdf5(u_const.NATEARTH_CENTROIDS[360]) + # only consider regions on land, without Antarctica + land_reg_ids = list(range(1, 1000)) + land_reg_ids.remove(10) # Antarctica + centroids = centroids.select(reg_id=land_reg_ids) if ignore_distance_to_coast: # Select centroids with lat <= max_latitude [idx_centr_filter] = (np.abs(centroids.lat) <= max_latitude).nonzero() From eac32d9acbcf33735c497a47fff616c94767e119 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 15 Mar 2024 15:42:43 +0100 Subject: [PATCH 20/39] Makefiile: run python -m [module] instead of [executable] --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f70dde017..96e70f97d 100644 --- a/Makefile +++ b/Makefile @@ -16,15 +16,15 @@ help: ## Use one of the following instructions: .PHONY : lint lint : ## Static code analysis with Pylint - pylint -ry climada_petals > pylint.log || true + python -m pylint -ry climada_petals > pylint.log || true .PHONY : unit_test unit_test : ## Unit tests execution with coverage and xml reports - pytest $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/ + python -m pytest $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/ .PHONY : install_test install_test : ## Test installation was successful - pytest $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \ + python -m pytest $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \ climada.engine.test.test_impact .PHONY : data_test @@ -37,11 +37,11 @@ notebook_test : ## Test notebooks in doc/tutorial .PHONY : integ_test integ_test : ## Integration tests execution with xml reports - pytest $(PYTEST_ARGS) climada_petals/test/ + python -m pytest $(PYTEST_ARGS) climada_petals/test/ .PHONY : test test : ## Unit and integration tests execution with coverage and xml reports - pytest $(PYTEST_ARGS) climada_petals/ + python -m pytest $(PYTEST_ARGS) climada_petals/ .PHONY : ci-clean ci-clean : From 2c49b1f70f1b41c801454d72056625ae39b3e7a9 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 15 Mar 2024 17:30:33 +0100 Subject: [PATCH 21/39] TCRain.from_tracks: increase performance by using precomputed dist to coast --- climada_petals/hazard/tc_rainfield.py | 2 +- .../hazard/test/test_tc_rainfield.py | 20 ------ .../test/test_tc_rainfield_integr.py | 68 +++++++++++++++++++ 3 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 climada_petals/test/test_tc_rainfield_integr.py diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index 43e5adb8f..da3f7812e 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -410,7 +410,7 @@ def from_tracks( else: # Select centroids which are inside max_dist_inland_km and lat <= max_latitude [idx_centr_filter] = ( - (centroids.get_dist_coast() <= max_dist_inland_km * 1000) + (centroids.get_dist_coast(precomputed=True) <= max_dist_inland_km * 1000) & (np.abs(centroids.lat) <= max_latitude) ).nonzero() diff --git a/climada_petals/hazard/test/test_tc_rainfield.py b/climada_petals/hazard/test/test_tc_rainfield.py index 1d515d307..33f7f5e91 100644 --- a/climada_petals/hazard/test/test_tc_rainfield.py +++ b/climada_petals/hazard/test/test_tc_rainfield.py @@ -20,7 +20,6 @@ """ import datetime as dt -from pathlib import Path import unittest import numpy as np @@ -203,25 +202,6 @@ def test_compute_rain_pass(self): self.assertAlmostEqual(rainfall[0, 130], 39.584947656) self.assertAlmostEqual(rainfall[0, 200], 73.792450959) - def test_rainfield_diff_time_steps(self): - """Check that the results do not depend too much on the track's time step sizes.""" - tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) - - train_org = TCRain.from_tracks(tc_track) - - tc_track.equal_timestep(time_step_h=1) - train_1h = TCRain.from_tracks(tc_track) - - tc_track.equal_timestep(time_step_h=0.5) - train_05h = TCRain.from_tracks(tc_track) - - for train in [train_1h, train_05h]: - np.testing.assert_allclose( - train_org.intensity.sum(), - train.intensity.sum(), - rtol=1e-1, - ) - def test_r_from_t_same_level(self): """Test the derivative of _r_from_t_same_level""" for tetens_coeffs in ["Alduchov1996", "Buck1981", "Bolton1980", "Murray1967"]: diff --git a/climada_petals/test/test_tc_rainfield_integr.py b/climada_petals/test/test_tc_rainfield_integr.py new file mode 100644 index 000000000..ff70b1c66 --- /dev/null +++ b/climada_petals/test/test_tc_rainfield_integr.py @@ -0,0 +1,68 @@ +""" +This file is part of CLIMADA. + +Copyright (C) 2017 ETH Zurich, CLIMADA contributors listed in AUTHORS. + +CLIMADA is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free +Software Foundation, version 3. + +CLIMADA is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with CLIMADA. If not, see . + +--- + +Test TCRain class +""" + +import unittest + +import numpy as np + +from climada import CONFIG +from climada.hazard import TCTracks +from climada.util.api_client import Client + +from climada_petals.hazard.tc_rainfield import TCRain + + +def getTestData(): + client = Client() + centr_ds = client.get_dataset_info(name='tc_rainfield_test', status='test_dataset') + _, [centr_test_mat, track, track_short, haz_hdf5] = client.download_dataset(centr_ds) + return track + + +TEST_TRACK = getTestData() + + +class TestModel(unittest.TestCase): + """Test modelling of rainfall""" + + def test_rainfield_diff_time_steps(self): + """Check that the results do not depend too much on the track's time step sizes.""" + tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) + + train_org = TCRain.from_tracks(tc_track) + + tc_track.equal_timestep(time_step_h=1) + train_1h = TCRain.from_tracks(tc_track) + + tc_track.equal_timestep(time_step_h=0.5) + train_05h = TCRain.from_tracks(tc_track) + + for train in [train_1h, train_05h]: + np.testing.assert_allclose( + train_org.intensity.sum(), + train.intensity.sum(), + rtol=1e-1, + ) + + +if __name__ == "__main__": + TESTS.addTests(unittest.TestLoader().loadTestsFromTestCase(TestModel)) + unittest.TextTestRunner(verbosity=2).run(TESTS) From 7d41cc75bc402bd6da571ba80d9b67f4fe2b493a Mon Sep 17 00:00:00 2001 From: Thomas Vogt Date: Mon, 18 Mar 2024 09:30:28 +0100 Subject: [PATCH 22/39] tc_rainfield: require centroids arg --- climada_petals/hazard/tc_rainfield.py | 11 +-- .../hazard/test/test_tc_rainfield.py | 32 +++++++-- .../test/test_tc_rainfield_integr.py | 68 ------------------- doc/tutorial/climada_hazard_TCRain.ipynb | 6 +- 4 files changed, 31 insertions(+), 86 deletions(-) delete mode 100644 climada_petals/test/test_tc_rainfield_integr.py diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index da3f7812e..33c95472e 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -229,7 +229,7 @@ def set_from_tracks(self, *args, **kwargs): def from_tracks( cls, tracks: TCTracks, - centroids: Optional[Centroids] = None, + centroids: Centroids, pool: Optional[pathos.pools.ProcessPool] = None, model: str = 'R-CLIPER', model_kwargs: Optional[dict] = None, @@ -297,7 +297,7 @@ def from_tracks( ---------- tracks : climada.hazard.TCTracks Tracks of storm events. - centroids : Centroids, optional + centroids : Centroids Centroids where to model TC. Default: global centroids at 360 arc-seconds resolution. pool : pathos.pool, optional Pool that will be used for parallel computation of rain fields. Default: None @@ -397,13 +397,6 @@ def from_tracks( TCRain """ num_tracks = tracks.size - if centroids is None: - # default centroids: Natural Earth data at resolution 360 - centroids = Centroids.from_hdf5(u_const.NATEARTH_CENTROIDS[360]) - # only consider regions on land, without Antarctica - land_reg_ids = list(range(1, 1000)) - land_reg_ids.remove(10) # Antarctica - centroids = centroids.select(reg_id=land_reg_ids) if ignore_distance_to_coast: # Select centroids with lat <= max_latitude [idx_centr_filter] = (np.abs(centroids.lat) <= max_latitude).nonzero() diff --git a/climada_petals/hazard/test/test_tc_rainfield.py b/climada_petals/hazard/test/test_tc_rainfield.py index 33f7f5e91..d4da35da5 100644 --- a/climada_petals/hazard/test/test_tc_rainfield.py +++ b/climada_petals/hazard/test/test_tc_rainfield.py @@ -66,7 +66,7 @@ def test_set_one_pass(self): """Test from_tracks constructor with a single track.""" tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) tc_track.equal_timestep() - tc_haz = TCRain.from_tracks(tc_track, centroids=CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB) self.assertEqual(tc_haz.haz_type, 'TR') self.assertEqual(tc_haz.units, 'mm') @@ -94,7 +94,7 @@ def test_tcr(self): tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) tc_track.equal_timestep() - tc_haz = TCRain.from_tracks(tc_track, model="TCR", centroids=CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB, model="TCR") self.assertTrue(isinstance(tc_haz.intensity, sparse.csr_matrix)) self.assertEqual(tc_haz.intensity.shape, (1, 296)) self.assertEqual(tc_haz.intensity.nonzero()[0].size, 296) @@ -105,7 +105,7 @@ def test_tcr(self): # This increases the results by more than 70% because the default value for saturation # specific humidity corresponds to a temperature of only ~267 K. tc_track.data[0]["t600"] = xr.full_like(tc_track.data[0]["central_pressure"], 275.0) - tc_haz = TCRain.from_tracks(tc_track, model="TCR", centroids=CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB, model="TCR") self.assertTrue(isinstance(tc_haz.intensity, sparse.csr_matrix)) self.assertEqual(tc_haz.intensity.shape, (1, 296)) self.assertEqual(tc_haz.intensity.nonzero()[0].size, 296) @@ -121,7 +121,7 @@ def test_cross_antimeridian(self): # Cyclone YASA (2020) passed directly over Fiji tr = TCTracks.from_ibtracs_netcdf(storm_id=["2020346S13168"]) - inten = TCRain.from_tracks(tr, centroids=cen).intensity.toarray()[0, :] + inten = TCRain.from_tracks(tr, cen).intensity.toarray()[0, :] # Centroids 1 and 2 are identical, they just use a different normalization for lon. This # should not affect the result at all: @@ -134,7 +134,7 @@ def test_cross_antimeridian(self): def test_from_file_pass(self): """Test from_tracks constructor with one input.""" tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK_SHORT) - tc_haz = TCRain.from_tracks(tc_track, centroids=CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB) tc_haz.check() self.assertEqual(tc_haz.haz_type, 'TR') @@ -158,7 +158,7 @@ def test_from_file_pass(self): def test_two_files_pass(self): """Test from_tracks constructor with two ibtracs.""" tc_track = TCTracks.from_processed_ibtracs_csv([TEST_TRACK_SHORT, TEST_TRACK_SHORT]) - tc_haz = TCRain.from_tracks(tc_track, centroids=CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB) tc_haz.remove_duplicates() tc_haz.check() @@ -202,6 +202,26 @@ def test_compute_rain_pass(self): self.assertAlmostEqual(rainfall[0, 130], 39.584947656) self.assertAlmostEqual(rainfall[0, 200], 73.792450959) + def test_rainfield_diff_time_steps(self): + """Check that the results do not depend too much on the track's time step sizes.""" + tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) + centroids = CENTR_TEST_BRB + + train_org = TCRain.from_tracks(tc_track, centroids) + + tc_track.equal_timestep(time_step_h=1) + train_1h = TCRain.from_tracks(tc_track, centroids) + + tc_track.equal_timestep(time_step_h=0.5) + train_05h = TCRain.from_tracks(tc_track, centroids) + + for train in [train_1h, train_05h]: + np.testing.assert_allclose( + train_org.intensity.sum(), + train.intensity.sum(), + rtol=1e-1, + ) + def test_r_from_t_same_level(self): """Test the derivative of _r_from_t_same_level""" for tetens_coeffs in ["Alduchov1996", "Buck1981", "Bolton1980", "Murray1967"]: diff --git a/climada_petals/test/test_tc_rainfield_integr.py b/climada_petals/test/test_tc_rainfield_integr.py deleted file mode 100644 index ff70b1c66..000000000 --- a/climada_petals/test/test_tc_rainfield_integr.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -This file is part of CLIMADA. - -Copyright (C) 2017 ETH Zurich, CLIMADA contributors listed in AUTHORS. - -CLIMADA is free software: you can redistribute it and/or modify it under the -terms of the GNU General Public License as published by the Free -Software Foundation, version 3. - -CLIMADA is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with CLIMADA. If not, see . - ---- - -Test TCRain class -""" - -import unittest - -import numpy as np - -from climada import CONFIG -from climada.hazard import TCTracks -from climada.util.api_client import Client - -from climada_petals.hazard.tc_rainfield import TCRain - - -def getTestData(): - client = Client() - centr_ds = client.get_dataset_info(name='tc_rainfield_test', status='test_dataset') - _, [centr_test_mat, track, track_short, haz_hdf5] = client.download_dataset(centr_ds) - return track - - -TEST_TRACK = getTestData() - - -class TestModel(unittest.TestCase): - """Test modelling of rainfall""" - - def test_rainfield_diff_time_steps(self): - """Check that the results do not depend too much on the track's time step sizes.""" - tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) - - train_org = TCRain.from_tracks(tc_track) - - tc_track.equal_timestep(time_step_h=1) - train_1h = TCRain.from_tracks(tc_track) - - tc_track.equal_timestep(time_step_h=0.5) - train_05h = TCRain.from_tracks(tc_track) - - for train in [train_1h, train_05h]: - np.testing.assert_allclose( - train_org.intensity.sum(), - train.intensity.sum(), - rtol=1e-1, - ) - - -if __name__ == "__main__": - TESTS.addTests(unittest.TestLoader().loadTestsFromTestCase(TestModel)) - unittest.TextTestRunner(verbosity=2).run(TESTS) diff --git a/doc/tutorial/climada_hazard_TCRain.ipynb b/doc/tutorial/climada_hazard_TCRain.ipynb index 89f499bd7..7ef1d2a28 100644 --- a/doc/tutorial/climada_hazard_TCRain.ipynb +++ b/doc/tutorial/climada_hazard_TCRain.ipynb @@ -265,7 +265,7 @@ "source": [ "from climada_petals.hazard import TCRain\n", "\n", - "tr_bang = TCRain.from_tracks(tracks, centroids=cent_bang, model=\"R-CLIPER\")\n", + "tr_bang = TCRain.from_tracks(tracks, cent_bang, model=\"R-CLIPER\")\n", "ax = tr_bang.plot_intensity(1)\n", "ax.set_title('Rain SIDR 2007 (from R-CLIPER)')\n", "ax = tr_bang.plot_intensity(2)\n", @@ -342,7 +342,7 @@ } ], "source": [ - "tr_bang = TCRain.from_tracks(tracks, centroids=cent_bang, model=\"TCR\")\n", + "tr_bang = TCRain.from_tracks(tracks, cent_bang, model=\"TCR\")\n", "ax = tr_bang.plot_intensity(1)\n", "ax.set_title('Rain SIDR 2007 (from TCR)')\n", "ax = tr_bang.plot_intensity(2)\n", @@ -429,7 +429,7 @@ "tracks = TCTracks.from_hdf5(tcrain_examples)\n", "tracks.equal_timestep(0.5)\n", "\n", - "tr_bang = TCRain.from_tracks(tracks, centroids=cent_bang, model=\"TCR\")\n", + "tr_bang = TCRain.from_tracks(tracks, cent_bang, model=\"TCR\")\n", "ax = tr_bang.plot_intensity(1)\n", "ax.set_title('Rain SIDR 2007 (from TCR, with all required inputs)')\n", "ax = tr_bang.plot_intensity(2)\n", From 8e7c6f69f39a0b8e572cf3aebd9d0b8b97ee5c19 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Mon, 18 Mar 2024 09:42:18 +0100 Subject: [PATCH 23/39] hazard.landslide: use Centroids.from_meta instead of creating a meta attribute of centroids --- climada_petals/hazard/landslide.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/climada_petals/hazard/landslide.py b/climada_petals/hazard/landslide.py index fb00c936a..a1d846064 100644 --- a/climada_petals/hazard/landslide.py +++ b/climada_petals/hazard/landslide.py @@ -299,8 +299,9 @@ def from_prob(cls, bbox, path_sourcefile, corr_fact=10e6, n_years=500, haz = cls() # raster with occurrence probs - haz.centroids.meta, prob_matrix = \ + meta, prob_matrix = \ u_coord.read_raster(path_sourcefile, geometry=[shapely.geometry.box(*bbox, ccw=True)]) + haz.centroids = Centroids.from_meta(meta) prob_matrix = prob_matrix.squeeze()/corr_fact # sample events from probabilities @@ -314,11 +315,6 @@ def from_prob(cls, bbox, path_sourcefile, corr_fact=10e6, n_years=500, haz.event_name = np.array(range(n_years)) haz.event_id = np.array(range(n_years)) - if not haz.centroids.meta['crs'].is_epsg_code: - haz.centroids.meta['crs'] = haz.centroids.meta['crs' - ].from_user_input(DEF_CRS) - haz.centroids.set_geometry_points() - haz.check() return haz From 792dab418c847a0a48c8ba6ede9b121967cb7623 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Wed, 20 Mar 2024 12:29:41 +0100 Subject: [PATCH 24/39] TCSurgeBathtub tutorial: Centroids doesn't check() anymore --- doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb b/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb index d831354a7..0309d166f 100644 --- a/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb +++ b/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb @@ -144,7 +144,6 @@ "min_lat, max_lat, min_lon, max_lon = 20, 27, 88.5, 92.5\n", "cent_bang = Centroids.from_pnt_bounds((min_lon, min_lat, max_lon, max_lat), res=0.015)\n", "cent_bang.set_dist_coast(signed=True, precomputed=True)\n", - "cent_bang.check()\n", "\n", "tc_bang = TropCyclone.from_tracks(tr_usa, centroids=cent_bang)" ] From 1aefefeb6517aae1638e04189fd8c86caeb93de4 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Thu, 21 Mar 2024 11:39:08 +0100 Subject: [PATCH 25/39] hazard_emulator tutorial: adapt to refactored centroids --- doc/tutorial/climada_hazard_emulator.ipynb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/tutorial/climada_hazard_emulator.ipynb b/doc/tutorial/climada_hazard_emulator.ipynb index f6effd08e..2919517c8 100644 --- a/doc/tutorial/climada_hazard_emulator.ipynb +++ b/doc/tutorial/climada_hazard_emulator.ipynb @@ -92,9 +92,8 @@ "source": [ "import warnings; warnings.filterwarnings('ignore')\n", "import matplotlib.pyplot as plt\n", - "import pyproj\n", "plt.rcParams['figure.dpi'] = 120\n", - "hazard.centroids.geometry = hazard.centroids.geometry.to_crs(pyproj.CRS(\"EPSG:4326\"))\n", + "hazard.centroids.to_crs(\"EPSG:4326\", inplace=True)\n", "hazard.centroids.plot(c=hazard.intensity[16637,:].toarray().ravel(), s=0.1);" ] }, @@ -194,8 +193,8 @@ ], "source": [ "import shapely\n", - "hazard.centroids.region_id = shapely.vectorized.contains(\n", - " region.shape, hazard.centroids.lon, hazard.centroids.lat) & hazard.centroids.on_land.astype(bool)\n", + "hazard.centroids.gdf['region_id'] = shapely.vectorized.contains(\n", + " region.shape, hazard.centroids.lon, hazard.centroids.lat) & hazard.centroids.gdf.on_land.astype(bool)\n", "hazard_EP = hazard.select(reg_id=1)\n", "# Plot one event as an example:\n", "hazard_EP.centroids.plot(c=hazard_EP.intensity[31577,:].toarray().ravel(), s=3);" @@ -763,8 +762,8 @@ ], "source": [ "observed = TropCyclone.from_hdf5(EMULATOR_DATA_DIR.joinpath(\"hazard_360as_ibtracs_1950-2019.hdf5\"))\n", - "observed.centroids.region_id = shapely.vectorized.contains(\n", - " region.shape, observed.centroids.lon, observed.centroids.lat) & observed.centroids.on_land.astype(bool)\n", + "observed.centroids.gdf['region_id'] = shapely.vectorized.contains(\n", + " region.shape, observed.centroids.lon, observed.centroids.lat) & observed.centroids.gdf.on_land.astype(bool)\n", "observed_EP = observed.select(reg_id=1)" ] }, From 9819929e306ef4b3c370f1031fbbe1b9e21e74b4 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Apr 2024 14:57:48 +0200 Subject: [PATCH 26/39] TC tutorials: adaptations to centroids refactoring --- doc/tutorial/climada_hazard_TCRain.ipynb | 15 ++------------- doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/doc/tutorial/climada_hazard_TCRain.ipynb b/doc/tutorial/climada_hazard_TCRain.ipynb index 7ef1d2a28..fb9eb80d5 100644 --- a/doc/tutorial/climada_hazard_TCRain.ipynb +++ b/doc/tutorial/climada_hazard_TCRain.ipynb @@ -185,25 +185,14 @@ "shell.execute_reply.started": "2023-07-17T13:15:39.163503Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-07-17 15:15:39,168 - climada.util.coordinates - INFO - Sampling from SYSTEM_DIR/GMT_intermediate_coast_distance_01d.tif\n" - ] - } - ], "source": [ "from climada.hazard import Centroids\n", "\n", "# define centroids and restrict to points on land\n", "min_lat, max_lat, min_lon, max_lon = 21.0, 24.5, 88.5, 92.5\n", "cent_bang = Centroids.from_pnt_bounds((min_lon, min_lat, max_lon, max_lat), res=0.015)\n", - "cent_bang.set_dist_coast(signed=True, precomputed=True)\n", - "cent_bang.dist_coast *= -1.0\n", - "cent_bang.dist_coast[cent_bang.dist_coast < 0] = 9000e3\n", - "cent_bang.check()" + "cent_bang.set_on_land()\n", + "cent_bang = cent_bang.select(sel_cen=cent_bang.on_land)" ] }, { diff --git a/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb b/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb index 0309d166f..692225c5a 100644 --- a/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb +++ b/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb @@ -209,8 +209,8 @@ ], "source": [ "# plot elevation of the raster\n", - "ts_bang.centroids.set_elevation(topo_path)\n", - "ts_bang.centroids.plot(c=ts_bang.centroids.elevation, vmin=0, vmax=10)" + "elevation = ts_bang.centroids.get_elevation(topo_path)\n", + "ts_bang.centroids.plot(c=elevation, vmin=0, vmax=10)" ] }, { From 8fe45be3944e4fd86a6cc7b9452f79b613512aa3 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Thu, 11 Apr 2024 20:39:37 +0200 Subject: [PATCH 27/39] fix tests (make some pass really!) --- climada_petals/hazard/test/test_wildfire.py | 4 +++- climada_petals/test/test_cropyield_integr.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/climada_petals/hazard/test/test_wildfire.py b/climada_petals/hazard/test/test_wildfire.py index a12adaa6e..974e51a03 100644 --- a/climada_petals/hazard/test/test_wildfire.py +++ b/climada_petals/hazard/test/test_wildfire.py @@ -284,7 +284,9 @@ def test_centroids_pass(self): self.assertAlmostEqual(centroids_meta['transform'][4], -centroids_meta['transform'][0]) self.assertGreaterEqual(centroids_meta['transform'][5], firms.latitude.max()) self.assertLessEqual(firms.latitude.max(), centroids.total_bounds[3]) - self.assertGreaterEqual(firms.latitude.min(), centroids.total_bounds[1]) + # TODO: check whether allowing for a tolerance is valid here + TOLERANCE = 0.005 + self.assertGreaterEqual(firms.latitude.min(), centroids.total_bounds[1] - TOLERANCE) self.assertLessEqual(firms.longitude.max(), centroids.total_bounds[2]) self.assertGreaterEqual(firms.longitude.min(), centroids.total_bounds[0]) self.assertTrue(centroids.lat.size) diff --git a/climada_petals/test/test_cropyield_integr.py b/climada_petals/test/test_cropyield_integr.py index 5dd11b478..a36e5a5f6 100644 --- a/climada_petals/test/test_cropyield_integr.py +++ b/climada_petals/test/test_cropyield_integr.py @@ -45,7 +45,7 @@ def test_EU(self): fn_str_var=FN_STR_DEMO) hist_mean = haz.calc_mean(yearrange_mean=(2001, 2005)) haz_new = rel_yield_to_int(haz, hist_mean) - haz_new.centroids.set_region_id() + haz_new.centroids.set_region_id(overwrite=True) exp = CropProduction.from_isimip_netcdf(input_dir=INPUT_DIR, filename=FILENAME_LU, hist_mean=FILENAME_MEAN, bbox=bbox, yearrange=(2001, 2005), scenario='flexible', unit='t/y', From d6b3a5cadc2bd068fc1a2f1af3cab4256301f24f Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Mon, 15 Apr 2024 10:43:46 +0200 Subject: [PATCH 28/39] fix docs --- .../climada_petals.entity.exposures.openstreetmap.rst | 11 ----------- .../climada_petals.entity.exposures.rst | 8 ++++++++ doc/tutorial/climada_hazard_TCRain.ipynb | 1 + 3 files changed, 9 insertions(+), 11 deletions(-) delete mode 100644 doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst diff --git a/doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst b/doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst deleted file mode 100644 index 122a4d955..000000000 --- a/doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst +++ /dev/null @@ -1,11 +0,0 @@ -climada\_petals\.entity\.exposures\.openstreetmap package -========================================================= - -climada\_petals\.entity\.exposures\.openstreetmap\.osm\_dataloader module -------------------------------------------------------------------------- - -.. automodule:: climada_petals.entity.exposures.openstreetmap.osm_dataloader - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/climada_petals/climada_petals.entity.exposures.rst b/doc/climada_petals/climada_petals.entity.exposures.rst index 96040f93e..013b81445 100644 --- a/doc/climada_petals/climada_petals.entity.exposures.rst +++ b/doc/climada_petals/climada_petals.entity.exposures.rst @@ -29,6 +29,14 @@ climada\_petals\.entity\.exposures\.gdp\_asset module :undoc-members: :show-inheritance: +climada\_petals\.entity\.exposures\.osm\_dataloader module +---------------------------------------------------------- + +.. automodule:: climada_petals.entity.exposures.osm_dataloader + :members: + :undoc-members: + :show-inheritance: + climada\_petals\.entity\.exposures\.spam\_agrar module ------------------------------------------------------ diff --git a/doc/tutorial/climada_hazard_TCRain.ipynb b/doc/tutorial/climada_hazard_TCRain.ipynb index fb9eb80d5..725b756a5 100644 --- a/doc/tutorial/climada_hazard_TCRain.ipynb +++ b/doc/tutorial/climada_hazard_TCRain.ipynb @@ -185,6 +185,7 @@ "shell.execute_reply.started": "2023-07-17T13:15:39.163503Z" } }, + "outputs": [], "source": [ "from climada.hazard import Centroids\n", "\n", From 58bd3a7dcfc1c960f73546acc037261cbf61d067 Mon Sep 17 00:00:00 2001 From: Thomas Vogt Date: Mon, 15 Apr 2024 11:19:37 +0200 Subject: [PATCH 29/39] Get rid of deprecated set_dist_coast --- climada_petals/hazard/test/test_tc_rainfield.py | 1 - climada_petals/hazard/test/test_tc_surge_bathtub.py | 2 -- doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb | 1 - 3 files changed, 4 deletions(-) diff --git a/climada_petals/hazard/test/test_tc_rainfield.py b/climada_petals/hazard/test/test_tc_rainfield.py index d4da35da5..e9bc45a2b 100644 --- a/climada_petals/hazard/test/test_tc_rainfield.py +++ b/climada_petals/hazard/test/test_tc_rainfield.py @@ -116,7 +116,6 @@ def test_cross_antimeridian(self): # Two locations on the island Taveuni (Fiji), one west and one east of 180° longitude. # We list the second point twice, with different lon-normalization: cen = Centroids.from_lat_lon([-16.95, -16.8, -16.8], [179.9, 180.1, -179.9]) - cen.set_dist_coast(precomputed=True) # Cyclone YASA (2020) passed directly over Fiji tr = TCTracks.from_ibtracs_netcdf(storm_id=["2020346S13168"]) diff --git a/climada_petals/hazard/test/test_tc_surge_bathtub.py b/climada_petals/hazard/test/test_tc_surge_bathtub.py index 6bc11c2cd..a99cf1af4 100644 --- a/climada_petals/hazard/test/test_tc_surge_bathtub.py +++ b/climada_petals/hazard/test/test_tc_surge_bathtub.py @@ -153,7 +153,6 @@ def test_surge_from_track(self): shape = (lat.size, lon.size) lon, lat = [ar.ravel() for ar in np.meshgrid(lon, lat)] centroids = Centroids.from_lat_lon(lat, lon) - centroids.set_dist_coast(signed=True, precomputed=True) wind_haz = TropCyclone.from_tracks(tc_track, centroids=centroids) @@ -180,7 +179,6 @@ def test_cross_antimeridian(self): # Two locations on the island Taveuni (Fiji), one west and one east of 180° longitude. # We list the second point twice, with different lon-normalization: cen = Centroids.from_lat_lon([-16.95, -16.8, -16.8], [179.9, 180.1, -179.9]) - cen.set_dist_coast(precomputed=True) # Cyclone YASA (2020) passed directly over Fiji tr = TCTracks.from_ibtracs_netcdf(storm_id=["2020346S13168"]) diff --git a/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb b/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb index 692225c5a..fe08e9836 100644 --- a/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb +++ b/doc/tutorial/climada_hazard_TCSurgeBathtub.ipynb @@ -143,7 +143,6 @@ "# define centroids raster\n", "min_lat, max_lat, min_lon, max_lon = 20, 27, 88.5, 92.5\n", "cent_bang = Centroids.from_pnt_bounds((min_lon, min_lat, max_lon, max_lat), res=0.015)\n", - "cent_bang.set_dist_coast(signed=True, precomputed=True)\n", "\n", "tc_bang = TropCyclone.from_tracks(tr_usa, centroids=cent_bang)" ] From d438c72a6761cdb6c205f9423cf22d1d1c767594 Mon Sep 17 00:00:00 2001 From: Thomas Vogt Date: Mon, 15 Apr 2024 11:36:17 +0200 Subject: [PATCH 30/39] get_dist_coast: avoid deprecated precomputed kwarg --- climada_petals/hazard/tc_rainfield.py | 2 +- climada_petals/hazard/tc_surge_bathtub.py | 2 +- climada_petals/hazard/test/test_tc_surge_bathtub.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index 33c95472e..1d4ebbfa5 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -403,7 +403,7 @@ def from_tracks( else: # Select centroids which are inside max_dist_inland_km and lat <= max_latitude [idx_centr_filter] = ( - (centroids.get_dist_coast(precomputed=True) <= max_dist_inland_km * 1000) + (centroids.get_dist_coast() <= max_dist_inland_km * 1000) & (np.abs(centroids.lat) <= max_latitude) ).nonzero() diff --git a/climada_petals/hazard/tc_surge_bathtub.py b/climada_petals/hazard/tc_surge_bathtub.py index bb43b7737..ac0002ac9 100644 --- a/climada_petals/hazard/tc_surge_bathtub.py +++ b/climada_petals/hazard/tc_surge_bathtub.py @@ -73,7 +73,7 @@ def from_tc_winds(wind_haz, topo_path, inland_decay_rate=0.2, add_sea_level_rise centroids = copy.deepcopy(wind_haz.centroids) # Select wind-affected centroids which are inside MAX_DIST_COAST and |lat| < 61 - centroids_dist_coast = centroids.get_dist_coast(signed=True, precomputed=True) + centroids_dist_coast = centroids.get_dist_coast(signed=True) coastal_msk = (wind_haz.intensity > 0).sum(axis=0).A1 > 0 coastal_msk &= (centroids_dist_coast < 0) coastal_msk &= (centroids_dist_coast >= -MAX_DIST_COAST * 1000) diff --git a/climada_petals/hazard/test/test_tc_surge_bathtub.py b/climada_petals/hazard/test/test_tc_surge_bathtub.py index a99cf1af4..cca6b37fa 100644 --- a/climada_petals/hazard/test/test_tc_surge_bathtub.py +++ b/climada_petals/hazard/test/test_tc_surge_bathtub.py @@ -49,7 +49,7 @@ def __init__(self, bounds, res_deg, slope=0.006): self.shape = (lat.size, lon.size) self.transform = rasterio.Affine(res_deg, 0, bounds[0], 0, -res_deg, bounds[3]) centroids = Centroids.from_lat_lon(*[ar.ravel() for ar in np.meshgrid(lon, lat)][::-1]) - self.dist_coast = centroids.get_dist_coast(signed=True, precomputed=True) + self.dist_coast = centroids.get_dist_coast(signed=True) self.slope = slope def __enter__(self): @@ -100,7 +100,7 @@ def test_fraction_on_land(self): with tmp_artifical_topo(dem_bounds, dem_res) as topo_path: fraction = _fraction_on_land(centroids, topo_path) fraction = fraction.reshape(shape) - dist_coast = centroids.get_dist_coast(signed=True, precomputed=True).reshape(shape) + dist_coast = centroids.get_dist_coast(signed=True).reshape(shape) # check valid range and order of magnitude self.assertTrue(np.all((fraction >= 0) & (fraction <= 1))) From 5f08ce0a49b07b432c3e42a32ca36e0513838f33 Mon Sep 17 00:00:00 2001 From: Lukas Riedel Date: Fri, 19 Apr 2024 11:15:41 +0200 Subject: [PATCH 31/39] Add option to change pytest command in Makefile Adapt the Jenkins script accordingly. --- Makefile | 11 +++++++---- script/jenkins/branches/run_unittest.sh | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8aa293f09..2fef0e2a5 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ # test, coverage and lint ### +# Default pytest command +PYTEST_CMD := pytest + PYTEST_JUNIT_ARGS = --junitxml=tests_xml/tests.xml PYTEST_COV_ARGS = \ @@ -20,11 +23,11 @@ lint : ## Static code analysis with Pylint .PHONY : unit_test unit_test : ## Unit tests execution with coverage and xml reports - python -m pytest $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/ + $(PYTEST_CMD) $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/ .PHONY : install_test install_test : ## Test installation was successful - python -m pytest $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \ + $(PYTEST_CMD) $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \ climada.engine.test.test_impact .PHONY : data_test @@ -37,11 +40,11 @@ notebook_test : ## Test notebooks in doc/tutorial .PHONY : integ_test integ_test : ## Integration tests execution with xml reports - python -m pytest $(PYTEST_ARGS) climada_petals/test/ + $(PYTEST_CMD) $(PYTEST_ARGS) climada_petals/test/ .PHONY : test test : ## Unit and integration tests execution with coverage and xml reports - python -m pytest $(PYTEST_ARGS) climada_petals/ + $(PYTEST_CMD) $(PYTEST_ARGS) climada_petals/ .PHONY : ci-clean ci-clean : diff --git a/script/jenkins/branches/run_unittest.sh b/script/jenkins/branches/run_unittest.sh index e2c23df47..bf55ad3a9 100644 --- a/script/jenkins/branches/run_unittest.sh +++ b/script/jenkins/branches/run_unittest.sh @@ -14,7 +14,7 @@ if [ -f $CORENV/$BRANCH ]; then pip install -e `cat $CORENV/$BRANCH` fi -make unit_test +make unit_test PYTEST_CMD="python -m pytest" if [ -f $CORENV/$BRANCH ]; then deactivate From c44c45b46d0dd4f3d8ab1abf96d0cc3c67b1a1e5 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 3 May 2024 16:43:41 +0200 Subject: [PATCH 32/39] hazard.tc_rainfield: re-establish a default centroids for `from_tracks` --- climada_petals/hazard/tc_rainfield.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/climada_petals/hazard/tc_rainfield.py b/climada_petals/hazard/tc_rainfield.py index 1d4ebbfa5..d825efb25 100644 --- a/climada_petals/hazard/tc_rainfield.py +++ b/climada_petals/hazard/tc_rainfield.py @@ -229,7 +229,7 @@ def set_from_tracks(self, *args, **kwargs): def from_tracks( cls, tracks: TCTracks, - centroids: Centroids, + centroids: Centroids = None, pool: Optional[pathos.pools.ProcessPool] = None, model: str = 'R-CLIPER', model_kwargs: Optional[dict] = None, @@ -297,8 +297,9 @@ def from_tracks( ---------- tracks : climada.hazard.TCTracks Tracks of storm events. - centroids : Centroids - Centroids where to model TC. Default: global centroids at 360 arc-seconds resolution. + centroids : Centroids, optional + Centroids where to model TC. Default: centroids at 360 arc-seconds resolution within + tracks' bounds. pool : pathos.pool, optional Pool that will be used for parallel computation of rain fields. Default: None model : str, optional @@ -397,6 +398,9 @@ def from_tracks( TCRain """ num_tracks = tracks.size + if centroids is None: + centroids = Centroids.from_pnt_bounds(tracks.get_bounds(), res=0.1) + if ignore_distance_to_coast: # Select centroids with lat <= max_latitude [idx_centr_filter] = (np.abs(centroids.lat) <= max_latitude).nonzero() From 0ada400af78e50440bda7ec30468278adde12e69 Mon Sep 17 00:00:00 2001 From: Emanuel Schmid <51439563+emanuel-schmid@users.noreply.github.com> Date: Fri, 3 May 2024 17:30:42 +0200 Subject: [PATCH 33/39] undo centroids as strictly positional argument in from_tc_tracks --- climada_petals/hazard/test/test_tc_rainfield.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/climada_petals/hazard/test/test_tc_rainfield.py b/climada_petals/hazard/test/test_tc_rainfield.py index e9bc45a2b..022325ee5 100644 --- a/climada_petals/hazard/test/test_tc_rainfield.py +++ b/climada_petals/hazard/test/test_tc_rainfield.py @@ -66,7 +66,7 @@ def test_set_one_pass(self): """Test from_tracks constructor with a single track.""" tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) tc_track.equal_timestep() - tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, centroids=CENTR_TEST_BRB) self.assertEqual(tc_haz.haz_type, 'TR') self.assertEqual(tc_haz.units, 'mm') @@ -94,7 +94,7 @@ def test_tcr(self): tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) tc_track.equal_timestep() - tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB, model="TCR") + tc_haz = TCRain.from_tracks(tc_track, model="TCR", centroids=CENTR_TEST_BRB) self.assertTrue(isinstance(tc_haz.intensity, sparse.csr_matrix)) self.assertEqual(tc_haz.intensity.shape, (1, 296)) self.assertEqual(tc_haz.intensity.nonzero()[0].size, 296) @@ -105,7 +105,7 @@ def test_tcr(self): # This increases the results by more than 70% because the default value for saturation # specific humidity corresponds to a temperature of only ~267 K. tc_track.data[0]["t600"] = xr.full_like(tc_track.data[0]["central_pressure"], 275.0) - tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB, model="TCR") + tc_haz = TCRain.from_tracks(tc_track, model="TCR", centroids=CENTR_TEST_BRB) self.assertTrue(isinstance(tc_haz.intensity, sparse.csr_matrix)) self.assertEqual(tc_haz.intensity.shape, (1, 296)) self.assertEqual(tc_haz.intensity.nonzero()[0].size, 296) @@ -120,7 +120,7 @@ def test_cross_antimeridian(self): # Cyclone YASA (2020) passed directly over Fiji tr = TCTracks.from_ibtracs_netcdf(storm_id=["2020346S13168"]) - inten = TCRain.from_tracks(tr, cen).intensity.toarray()[0, :] + inten = TCRain.from_tracks(tr, centroids=cen).intensity.toarray()[0, :] # Centroids 1 and 2 are identical, they just use a different normalization for lon. This # should not affect the result at all: @@ -133,7 +133,7 @@ def test_cross_antimeridian(self): def test_from_file_pass(self): """Test from_tracks constructor with one input.""" tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK_SHORT) - tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, centroids=CENTR_TEST_BRB) tc_haz.check() self.assertEqual(tc_haz.haz_type, 'TR') @@ -157,7 +157,7 @@ def test_from_file_pass(self): def test_two_files_pass(self): """Test from_tracks constructor with two ibtracs.""" tc_track = TCTracks.from_processed_ibtracs_csv([TEST_TRACK_SHORT, TEST_TRACK_SHORT]) - tc_haz = TCRain.from_tracks(tc_track, CENTR_TEST_BRB) + tc_haz = TCRain.from_tracks(tc_track, centroids=CENTR_TEST_BRB) tc_haz.remove_duplicates() tc_haz.check() From 8624309e37175334ed93c0c080235051e2cccf07 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Sat, 4 May 2024 01:05:03 +0200 Subject: [PATCH 34/39] test_tc_rainfiield undo obsolete changes in test --- climada_petals/hazard/test/test_tc_rainfield.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/climada_petals/hazard/test/test_tc_rainfield.py b/climada_petals/hazard/test/test_tc_rainfield.py index e9bc45a2b..74b3a96a2 100644 --- a/climada_petals/hazard/test/test_tc_rainfield.py +++ b/climada_petals/hazard/test/test_tc_rainfield.py @@ -45,8 +45,8 @@ def getTestData(): client = Client() centr_ds = client.get_dataset_info(name='tc_rainfield_test', status='test_dataset') - _, [centr_test_mat, track, track_short, haz_hdf5] = client.download_dataset(centr_ds) - return Centroids.from_hdf5(centr_test_mat), track, track_short, haz_hdf5 + _, [centr_test_hdf5, track, track_short, haz_hdf5] = client.download_dataset(centr_ds) + return Centroids.from_hdf5(centr_test_hdf5), track, track_short, haz_hdf5 CENTR_TEST_BRB, TEST_TRACK, TEST_TRACK_SHORT, HAZ_TEST_HDF5 = getTestData() @@ -204,15 +204,14 @@ def test_compute_rain_pass(self): def test_rainfield_diff_time_steps(self): """Check that the results do not depend too much on the track's time step sizes.""" tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK) - centroids = CENTR_TEST_BRB - train_org = TCRain.from_tracks(tc_track, centroids) + train_org = TCRain.from_tracks(tc_track) tc_track.equal_timestep(time_step_h=1) - train_1h = TCRain.from_tracks(tc_track, centroids) + train_1h = TCRain.from_tracks(tc_track) tc_track.equal_timestep(time_step_h=0.5) - train_05h = TCRain.from_tracks(tc_track, centroids) + train_05h = TCRain.from_tracks(tc_track) for train in [train_1h, train_05h]: np.testing.assert_allclose( From 9f48d3fdae8d11d4a4c61212ff1a469d9cf1ef69 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Mon, 6 May 2024 09:27:09 +0200 Subject: [PATCH 35/39] jenkins: experimenting with coverage threshold --- script/jenkins/branches/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 8c38fa34a..61ff70375 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -28,9 +28,9 @@ pipeline { junit 'tests_xml/*.xml' recordCoverage( qualityGates: [[ - baseline: 'MODIFIED_FILES', + baseline: 'PROJECT_DELTA', metric: 'LINE', - threshold: 60.0 + threshold: 0.0 ]], tools: [[ parser: 'COBERTURA', From 999eb9ab1ef70babfd5d3d1e9a5607e8ae245b82 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Mon, 6 May 2024 13:50:51 +0200 Subject: [PATCH 36/39] reverse changes not directly related to this PR --- Makefile | 13 ++++------- ..._petals.entity.exposures.openstreetmap.rst | 11 +++++++++ .../climada_petals.entity.exposures.rst | 8 ------- requirements/env_climada.yml | 4 +++- requirements/env_docs.yml | 12 ++++++---- script/jenkins/branches/Jenkinsfile | 23 +++++++++++++++---- script/jenkins/branches/run_lint.sh | 7 ------ script/jenkins/branches/run_unittest.sh | 22 ------------------ script/jenkins/install_env.sh | 4 +++- setup.py | 6 ++--- 10 files changed, 52 insertions(+), 58 deletions(-) create mode 100644 doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst delete mode 100644 script/jenkins/branches/run_lint.sh delete mode 100644 script/jenkins/branches/run_unittest.sh diff --git a/Makefile b/Makefile index 2fef0e2a5..6253c9e25 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,6 @@ # test, coverage and lint ### -# Default pytest command -PYTEST_CMD := pytest - PYTEST_JUNIT_ARGS = --junitxml=tests_xml/tests.xml PYTEST_COV_ARGS = \ @@ -19,15 +16,15 @@ help: ## Use one of the following instructions: .PHONY : lint lint : ## Static code analysis with Pylint - python -m pylint -ry climada_petals > pylint.log || true + pylint -ry climada_petals > pylint.log || true .PHONY : unit_test unit_test : ## Unit tests execution with coverage and xml reports - $(PYTEST_CMD) $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/ + pytest $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/ .PHONY : install_test install_test : ## Test installation was successful - $(PYTEST_CMD) $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \ + pytest $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \ climada.engine.test.test_impact .PHONY : data_test @@ -40,11 +37,11 @@ notebook_test : ## Test notebooks in doc/tutorial .PHONY : integ_test integ_test : ## Integration tests execution with xml reports - $(PYTEST_CMD) $(PYTEST_ARGS) climada_petals/test/ + pytest $(PYTEST_ARGS) climada_petals/test/ .PHONY : test test : ## Unit and integration tests execution with coverage and xml reports - $(PYTEST_CMD) $(PYTEST_ARGS) climada_petals/ + pytest $(PYTEST_ARGS) climada_petals/ .PHONY : ci-clean ci-clean : diff --git a/doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst b/doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst new file mode 100644 index 000000000..122a4d955 --- /dev/null +++ b/doc/climada_petals/climada_petals.entity.exposures.openstreetmap.rst @@ -0,0 +1,11 @@ +climada\_petals\.entity\.exposures\.openstreetmap package +========================================================= + +climada\_petals\.entity\.exposures\.openstreetmap\.osm\_dataloader module +------------------------------------------------------------------------- + +.. automodule:: climada_petals.entity.exposures.openstreetmap.osm_dataloader + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/climada_petals/climada_petals.entity.exposures.rst b/doc/climada_petals/climada_petals.entity.exposures.rst index 013b81445..96040f93e 100644 --- a/doc/climada_petals/climada_petals.entity.exposures.rst +++ b/doc/climada_petals/climada_petals.entity.exposures.rst @@ -29,14 +29,6 @@ climada\_petals\.entity\.exposures\.gdp\_asset module :undoc-members: :show-inheritance: -climada\_petals\.entity\.exposures\.osm\_dataloader module ----------------------------------------------------------- - -.. automodule:: climada_petals.entity.exposures.osm_dataloader - :members: - :undoc-members: - :show-inheritance: - climada\_petals\.entity\.exposures\.spam\_agrar module ------------------------------------------------------ diff --git a/requirements/env_climada.yml b/requirements/env_climada.yml index 677b8c86e..2b726f543 100644 --- a/requirements/env_climada.yml +++ b/requirements/env_climada.yml @@ -4,9 +4,11 @@ channels: - nodefaults dependencies: - cdsapi>=0.6 + - meson-python>=0.15,<0.16 # as of 2024-04-22 something seems to be wrong with 0.16 + - osm-flex>=1.1 - overpy>=0.7 - pymrio>=0.5 - - ruamel.yaml>=0.18 - rioxarray>=0.13 + - ruamel.yaml>=0.18 - scikit-image>=0.22 - xesmf>=0.8 diff --git a/requirements/env_docs.yml b/requirements/env_docs.yml index 9ca054ef7..d436b5e79 100644 --- a/requirements/env_docs.yml +++ b/requirements/env_docs.yml @@ -4,6 +4,14 @@ channels: - nodefaults dependencies: - cdsapi>=0.6 + - meson-python=0.15 + - osm-flex>=1.1 + - overpy>=0.7 + - pymrio>=0.5 + - rioxarray>=0.13 + - ruamel.yaml>=0.18 + - scikit-image>=0.22 + - xesmf>=0.8 - climada - ipykernel - ipython @@ -11,11 +19,7 @@ dependencies: - nb_conda_kernels - pandoc - pip - - pymrio>=0.5 - - ruamel.yaml>=0.18 - - scikit-image>=0.21 - sphinx>=2.0 - - xesmf>=0.8 - pip: - coverage>=4.5 - descartes diff --git a/script/jenkins/branches/Jenkinsfile b/script/jenkins/branches/Jenkinsfile index 61ff70375..94df05dd4 100644 --- a/script/jenkins/branches/Jenkinsfile +++ b/script/jenkins/branches/Jenkinsfile @@ -7,7 +7,12 @@ pipeline { stage('lint') { steps { - sh 'bash script/jenkins/branches/run_lint.sh' + sh '''#!/bin/bash + export PATH=$PATH:$CONDAPATH + source activate petals_env + rm -f pylint.log + pylint -ry climada_petals | tee pylint.log''' + discoverGitReferenceBuild referenceJob: 'petals_branches/develop' recordIssues tools: [pyLint(pattern: 'pylint.log')] } @@ -15,7 +20,17 @@ pipeline { stage('unit_test') { steps { - sh 'bash script/jenkins/branches/run_unittest.sh' + sh '''#!/bin/bash + export PATH=$PATH:$CONDAPATH + source activate + + # first install xemsf, then activate environment, so .../envs/.../etc/conda/activate.d/esmpy.sh is executed + # otherwise the environment varibalbe ESMFMKFILE is not initialized + mamba install -n petals_env -c conda-forge -y xesmf ruamel.yaml cdsapi + conda activate petals_env + rm -rf tests_xml/ + rm -rf coverage/ + make unit_test''' } } @@ -28,9 +43,9 @@ pipeline { junit 'tests_xml/*.xml' recordCoverage( qualityGates: [[ - baseline: 'PROJECT_DELTA', + baseline: 'MODIFIED_FILES', metric: 'LINE', - threshold: 0.0 + threshold: 60.0 ]], tools: [[ parser: 'COBERTURA', diff --git a/script/jenkins/branches/run_lint.sh b/script/jenkins/branches/run_lint.sh deleted file mode 100644 index c8306b6e7..000000000 --- a/script/jenkins/branches/run_lint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -export PATH=$PATH:$CONDAPATH -source activate petals_env - -rm -f pylint.log - -pylint -ry climada_petals | tee pylint.log diff --git a/script/jenkins/branches/run_unittest.sh b/script/jenkins/branches/run_unittest.sh deleted file mode 100644 index bf55ad3a9..000000000 --- a/script/jenkins/branches/run_unittest.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -export PATH=$PATH:$CONDAPATH - -source activate petals_env -rm -rf tests_xml/ -rm -rf coverage/ - -CORENV=~/jobs/petals_branches/core_env -BRANCH=`git name-rev --name-only HEAD | cut -f 3- -d /` -if [ -f $CORENV/$BRANCH ]; then - python -m venv --system-site-packages tvenv - source tvenv/bin/activate - - pip install -e `cat $CORENV/$BRANCH` -fi - -make unit_test PYTEST_CMD="python -m pytest" - -if [ -f $CORENV/$BRANCH ]; then - deactivate - #rm -r tvenv -fi diff --git a/script/jenkins/install_env.sh b/script/jenkins/install_env.sh index 37f7ea8ae..77fed559a 100644 --- a/script/jenkins/install_env.sh +++ b/script/jenkins/install_env.sh @@ -6,7 +6,9 @@ mamba env update -n petals_env -f ~/jobs/climada_install_env/workspace/requireme mamba env update -n petals_env -f requirements/env_climada.yml source activate petals_env -python -m pip install -e ~/jobs/climada_install_env/workspace/[test] +#python -m pip install -e ~/jobs/climada_install_env/workspace/[test] +#TODO: after mergin PR-122 remove line below and uncomment line above +python -m pip install -e ~/jobs/petals_branches/core_env/develop-freeze/[test] python -m pip install -e "." make install_test diff --git a/setup.py b/setup.py index 61a1fc341..8fd2bbffe 100644 --- a/setup.py +++ b/setup.py @@ -39,12 +39,12 @@ install_requires=[ 'climada>=4.1', 'cdsapi', + 'osm-flex', + "pymrio", + 'rioxarray', 'ruamel.yaml', 'scikit-image', 'xesmf', - "pymrio", - 'rioxarray', - 'osm-flex>=1.1.1', ], packages=find_namespace_packages(include=['climada_petals*']), From 5fe87c59b9828aa21ac6fef1a0b4eb311b56d83e Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Mon, 6 May 2024 14:39:20 +0200 Subject: [PATCH 37/39] reset core installation to develop branch --- script/jenkins/install_env.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/script/jenkins/install_env.sh b/script/jenkins/install_env.sh index 77fed559a..37f7ea8ae 100644 --- a/script/jenkins/install_env.sh +++ b/script/jenkins/install_env.sh @@ -6,9 +6,7 @@ mamba env update -n petals_env -f ~/jobs/climada_install_env/workspace/requireme mamba env update -n petals_env -f requirements/env_climada.yml source activate petals_env -#python -m pip install -e ~/jobs/climada_install_env/workspace/[test] -#TODO: after mergin PR-122 remove line below and uncomment line above -python -m pip install -e ~/jobs/petals_branches/core_env/develop-freeze/[test] +python -m pip install -e ~/jobs/climada_install_env/workspace/[test] python -m pip install -e "." make install_test From 116e2e83d79c292735f65a74a7bc65c518c842aa Mon Sep 17 00:00:00 2001 From: Chahan Kropf Date: Mon, 6 May 2024 14:57:42 +0200 Subject: [PATCH 38/39] Fix pixel center shift --- climada_petals/hazard/test/test_wildfire.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/climada_petals/hazard/test/test_wildfire.py b/climada_petals/hazard/test/test_wildfire.py index 974e51a03..1ad875d3e 100644 --- a/climada_petals/hazard/test/test_wildfire.py +++ b/climada_petals/hazard/test/test_wildfire.py @@ -273,7 +273,9 @@ def test_centroids_pass(self): """ Test _firms_centroids_creation """ wf = WildFire() firms = wf._clean_firms_df(TEST_FIRMS) - centroids = wf._firms_centroids_creation(firms, 0.375/ONE_LAT_KM, 1/2) + res_data = 0.375/ONE_LAT_KM + centr_res_factor = 1/2 + centroids = wf._firms_centroids_creation(firms, res_data, centr_res_factor) centroids_meta = centroids.get_meta() self.assertEqual(centroids_meta['width'], 144) self.assertEqual(centroids_meta['height'], 138) @@ -283,12 +285,12 @@ def test_centroids_pass(self): self.assertAlmostEqual(centroids_meta['transform'][3], 0.0) self.assertAlmostEqual(centroids_meta['transform'][4], -centroids_meta['transform'][0]) self.assertGreaterEqual(centroids_meta['transform'][5], firms.latitude.max()) - self.assertLessEqual(firms.latitude.max(), centroids.total_bounds[3]) - # TODO: check whether allowing for a tolerance is valid here - TOLERANCE = 0.005 - self.assertGreaterEqual(firms.latitude.min(), centroids.total_bounds[1] - TOLERANCE) - self.assertLessEqual(firms.longitude.max(), centroids.total_bounds[2]) - self.assertGreaterEqual(firms.longitude.min(), centroids.total_bounds[0]) + # Meta pixel coordinates are shifted by half resolution to the pixel center lat/lon + pixel_center_shift = (res_data / centr_res_factor) / 2 + self.assertLessEqual(firms.latitude.max(), centroids.total_bounds[3] + pixel_center_shift) + self.assertGreaterEqual(firms.latitude.min(), centroids.total_bounds[1] - pixel_center_shift) + self.assertLessEqual(firms.longitude.max(), centroids.total_bounds[2] + pixel_center_shift) + self.assertGreaterEqual(firms.longitude.min(), centroids.total_bounds[0] - pixel_center_shift) self.assertTrue(centroids.lat.size) self.assertTrue(centroids.get_area_pixel().size) self.assertTrue(centroids.on_land.size) From 16d5710960aa1e94cabd53e8c5f826710ff90928 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Wed, 8 May 2024 16:41:32 +0200 Subject: [PATCH 39/39] changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0203e129..7bdc2ce61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ Code freeze date: YYYY-MM-DD ### Changed +- Adaptations to refactoring of the `climada.hazard.Centroids` class, to be compatible with `climada>=5.0` [#122](https://github.com/CLIMADA-project/climada_petals/pull/122) + ### Fixed - Fix `climada.hazard.tc_rainfield` for TC tracks crossing the antimeridian [#105](https://github.com/CLIMADA-project/climada_petals/pull/105)