From 824c00c660ac681a83876669c651db7050c69f5d Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 21 Feb 2024 15:10:59 -0500 Subject: [PATCH 01/30] xclim 0.48 compatibility --- environment-dev.yml | 6 +++--- environment.yml | 6 +++--- pyproject.toml | 6 +++--- tests/test_catutils.py | 4 ++-- tests/test_diagnostics.py | 4 ++-- tests/test_indicators.py | 18 +++++++++--------- xscen/CVs/frequency_to_timedelta.json | 6 +++--- xscen/CVs/frequency_to_xrfreq.json | 6 +++--- xscen/CVs/xrfreq_to_frequency.json | 7 +++++-- xscen/CVs/xrfreq_to_timedelta.json | 5 ++++- xscen/indicators.py | 4 ++-- 11 files changed, 39 insertions(+), 33 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 6e4e8d79..e8b28c1c 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,7 +2,7 @@ name: xscen-dev channels: - conda-forge dependencies: - - python >=3.9 + - python >=3.9,<3.13 # Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py! # Also consider updating the list in xs.utils.show_versions if you add a new package. # Main packages @@ -21,7 +21,7 @@ dependencies: - netCDF4 - numcodecs - numpy - - pandas >=2.0,<2.2 + - pandas >=2.2 - parse - pyyaml - rechunker @@ -29,7 +29,7 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray <2023.11.0 # FIXME: Remove when pandas 2.2 is released and xclim is fixed. + - xarray - xclim >=0.46.0 - xesmf >=0.7 - zarr diff --git a/environment.yml b/environment.yml index 0dfde47b..ac9babb2 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xscen channels: - conda-forge dependencies: - - python >=3.9 + - python >=3.9,<3.13 # Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py! # Also consider updating the list in xs.utils.show_versions if you add a new package. # Main packages @@ -21,7 +21,7 @@ dependencies: - netCDF4 - numcodecs - numpy - - pandas >=2.0,<2.2 + - pandas >=2.2 - parse - pyyaml - rechunker @@ -29,7 +29,7 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray <2023.11.0 # FIXME: Remove when pandas 2.2 is released and xclim is fixed. + - xarray - xclim >=0.46.0 - xesmf >=0.7 - zarr diff --git a/pyproject.toml b/pyproject.toml index b48a7bf8..77d11fa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - # "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Atmospheric Science" ] @@ -52,7 +52,7 @@ dependencies = [ "netCDF4", "numcodecs", "numpy", - "pandas>=2.0,<2.2", + "pandas>=2.2", "parse", # Used when opening catalogs. "pyarrow", @@ -62,7 +62,7 @@ dependencies = [ "shapely>=2.0", "sparse", "toolz", - "xarray<2023.11.0", # FIXME: Remove when pandas 2.2 is released and xclim is fixed. + "xarray", "xclim>=0.46.0", "zarr" ] diff --git a/tests/test_catutils.py b/tests/test_catutils.py index 4b9cf5cd..6748c3c2 100644 --- a/tests/test_catutils.py +++ b/tests/test_catutils.py @@ -230,7 +230,7 @@ def test_build_path(samplecat): def test_build_path_ds(): ds = xr.tutorial.open_dataset("air_temperature") - ds = ds.assign(time=xr.cftime_range("0001-01-01", freq="6H", periods=ds.time.size)) + ds = ds.assign(time=xr.cftime_range("0001-01-01", freq="6h", periods=ds.time.size)) ds.attrs.update(source="source", institution="institution") new_path = cu.build_path( ds, @@ -240,7 +240,7 @@ def test_build_path_ds(): }, ) assert new_path == Path( - "source/institution/air_6H/source_institution_air_6hr_0001-0002" + "source/institution/air_6h/source_institution_air_6hr_0001-0002" ) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index d11cd029..fed0f2a1 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -125,7 +125,7 @@ def test_variables(self): xs.diagnostics.health_checks(ds, variables_and_units={"tas": "degC"}) with pytest.warns( UserWarning, - match="Data units kelvin are not compatible with requested mm.", + match="are not compatible with requested mm.", ): xs.diagnostics.health_checks(ds, variables_and_units={"tas": "mm"}) @@ -162,7 +162,7 @@ def test_cfchecks(self): xs.diagnostics.health_checks(ds, cfchecks=bad_cfcheck) @pytest.mark.parametrize( - "freq, gap", [("D", False), ("MS", False), ("3H", False), ("D", True)] + "freq, gap", [("D", False), ("MS", False), ("3h", False), ("D", True)] ) def test_freq(self, freq, gap): ds = timeseries( diff --git a/tests/test_indicators.py b/tests/test_indicators.py index ad4c2503..3b7dd0b2 100644 --- a/tests/test_indicators.py +++ b/tests/test_indicators.py @@ -48,12 +48,12 @@ def test_input_types(self, input): def test_level(self, to_level): if to_level is None: ind_dict = xs.compute_indicators(self.ds, indicators=self.yaml_file) - assert "indicators" in ind_dict["AS-JAN"].attrs["cat:processing_level"] + assert "indicators" in ind_dict["YS-JAN"].attrs["cat:processing_level"] else: ind_dict = xs.compute_indicators( self.ds, indicators=self.yaml_file, to_level=to_level ) - assert to_level in ind_dict["AS-JAN"].attrs["cat:processing_level"] + assert to_level in ind_dict["YS-JAN"].attrs["cat:processing_level"] # Periods needed to cover both branches of the if/else in compute_indicators, so no need to test separately. @pytest.mark.parametrize( @@ -76,13 +76,13 @@ def test_output(self, periods): assert all( xrfreq in ind_dict[xrfreq].attrs["cat:xrfreq"] for xrfreq in ind_dict.keys() ) - assert all(v in ind_dict["AS-JAN"] for v in ["tg_min", "growing_degree_days"]) + assert all(v in ind_dict["YS-JAN"] for v in ["tg_min", "growing_degree_days"]) if periods is None: assert "time" not in ind_dict["fx"].dims - assert len(ind_dict["AS-JAN"].time) == 2 + assert len(ind_dict["YS-JAN"].time) == 2 else: assert len(ind_dict["fx"].time) == len(periods) - assert len(ind_dict["AS-JAN"].time) == 8 + assert len(ind_dict["YS-JAN"].time) == 8 def test_qs_dec(self): indicator = xclim.core.indicator.Indicator.from_dict( @@ -118,7 +118,7 @@ def test_multiple_outputs(self, periods): ) assert all( - v in ind_dict["AS-JAN"] + v in ind_dict["YS-JAN"] for v in [ "prcpavg", "rain_season_start", @@ -130,7 +130,7 @@ def test_multiple_outputs(self, periods): @pytest.mark.parametrize("restrict_years", [True, False]) def test_as_jul(self, restrict_years): indicator = xclim.core.indicator.Indicator.from_dict( - data={"base": "freezing_degree_days", "parameters": {"freq": "AS-JUL"}}, + data={"base": "freezing_degree_days", "parameters": {"freq": "YS-JUL"}}, identifier="degree_days_below_0_annual_start_july", module="tests", ) @@ -140,8 +140,8 @@ def test_as_jul(self, restrict_years): restrict_years=restrict_years, ) - assert "AS-JUL" in ind_dict["AS-JUL"].attrs["cat:xrfreq"] - out = ind_dict["AS-JUL"] + assert "YS-JUL" in ind_dict["YS-JUL"].attrs["cat:xrfreq"] + out = ind_dict["YS-JUL"] if restrict_years: assert len(out.time) == 3 # same length as input ds assert out.time[0].dt.strftime("%Y-%m-%d").item() == "2001-07-01" diff --git a/xscen/CVs/frequency_to_timedelta.json b/xscen/CVs/frequency_to_timedelta.json index 910dfdbe..7152502a 100644 --- a/xscen/CVs/frequency_to_timedelta.json +++ b/xscen/CVs/frequency_to_timedelta.json @@ -1,7 +1,7 @@ { - "1hr": "1H", - "3hr": "3H", - "6hr": "6H", + "1hr": "1h", + "3hr": "3h", + "6hr": "6h", "day": "1D", "sem": "1W", "2sem": "2W", diff --git a/xscen/CVs/frequency_to_xrfreq.json b/xscen/CVs/frequency_to_xrfreq.json index 394aa332..3c65edf6 100644 --- a/xscen/CVs/frequency_to_xrfreq.json +++ b/xscen/CVs/frequency_to_xrfreq.json @@ -1,7 +1,7 @@ { - "1hr": "H", - "3hr": "3H", - "6hr": "6H", + "1hr": "h", + "3hr": "3h", + "6hr": "6h", "day": "D", "sem": "W", "2sem": "2W", diff --git a/xscen/CVs/xrfreq_to_frequency.json b/xscen/CVs/xrfreq_to_frequency.json index 81f9149d..acda19fa 100644 --- a/xscen/CVs/xrfreq_to_frequency.json +++ b/xscen/CVs/xrfreq_to_frequency.json @@ -1,16 +1,19 @@ { "is_regex": true, + "h": "1hr", "H": "1hr", + "3h": "3hr", "3H": "3hr", + "6h": "6hr", "6H": "6hr", "D": "day", "W": "sem", "2W": "2sem", - "14d": "2sem", + "14D": "2sem", "M.*": "mon", "Q.*": "qtr", "2Q.*": "6mon", "A.*": "yr", - "YS": "yr", + "Y.*": "yr", "fx": "fx" } diff --git a/xscen/CVs/xrfreq_to_timedelta.json b/xscen/CVs/xrfreq_to_timedelta.json index 3dc7906e..7fbf42e6 100644 --- a/xscen/CVs/xrfreq_to_timedelta.json +++ b/xscen/CVs/xrfreq_to_timedelta.json @@ -1,7 +1,10 @@ { "is_regex": true, + "h": "1h", "H": "1H", + "3h": "3h", "3H": "3H", + "6h": "6h", "6H": "6H", "D": "1D", "W": "7D", @@ -10,6 +13,6 @@ "Q.*": "90D", "2Q.*": "180D", "A.*": "365D", - "YS": "365D", + "Y.*": "365D", "fx": "NAN" } diff --git a/xscen/indicators.py b/xscen/indicators.py index 85b4d0b4..02e8ff6e 100644 --- a/xscen/indicators.py +++ b/xscen/indicators.py @@ -170,7 +170,7 @@ def _infer_freq_from_meta(ind): else: freq = "fx" if freq == "YS": - freq = "AS-JAN" # To fix an inconsistency in xclim default 'freq' + freq = "YS-JAN" else: # Multiple time periods to concatenate @@ -197,7 +197,7 @@ def _infer_freq_from_meta(ind): freq = "fx" if freq == "YS": - freq = "AS-JAN" # To fix an inconsistency in xclim default 'freq' + freq = "YS-JAN" # In order to concatenate time periods, the indicator still needs a time dimension if freq == "fx": tmp = tmp.assign_coords({"time": ds_subset.time[0]}) From 5e22eee47403ad617faaf45e54c58d49b4c83ace Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 21 Feb 2024 15:27:50 -0500 Subject: [PATCH 02/30] pins --- environment-dev.yml | 4 ++-- environment.yml | 4 ++-- pyproject.toml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index e8b28c1c..dfe1106a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -29,8 +29,8 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray - - xclim >=0.46.0 + - xarray >=2023.11.0 + - xclim >=0.48.0 - xesmf >=0.7 - zarr # Opt diff --git a/environment.yml b/environment.yml index ac9babb2..c9982c4f 100644 --- a/environment.yml +++ b/environment.yml @@ -29,8 +29,8 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray - - xclim >=0.46.0 + - xarray >=2023.11.0 + - xclim >=0.48.0 - xesmf >=0.7 - zarr # To install from source and get translations diff --git a/pyproject.toml b/pyproject.toml index 77d11fa6..ffee4ef3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,8 +62,8 @@ dependencies = [ "shapely>=2.0", "sparse", "toolz", - "xarray", - "xclim>=0.46.0", + "xarray>=2023.11.0", + "xclim>=0.48.0", "zarr" ] From 27534774fd9f84e03507112d0c37c518ff043b41 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 21 Feb 2024 15:54:08 -0500 Subject: [PATCH 03/30] converted missing AS-JAN --- xscen/diagnostics.py | 2 +- xscen/utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xscen/diagnostics.py b/xscen/diagnostics.py index d5a4c093..ea4bd5f5 100644 --- a/xscen/diagnostics.py +++ b/xscen/diagnostics.py @@ -241,7 +241,7 @@ def _message(): _error( "The timesteps are irregular or cannot be inferred by xarray.", "freq" ) - elif freq.replace("YS", "AS-JAN") != inferred_freq: + elif freq.replace("YS", "YS-JAN") != inferred_freq: _error( f"The frequency is not '{freq}'. Received '{inferred_freq}'.", "freq" ) diff --git a/xscen/utils.py b/xscen/utils.py index 68b72bc9..965e0f66 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -1045,7 +1045,7 @@ def unstack_dates( Returns ------- xr.Dataset or DataArray - Same as ds but the time axis is now yearly (AS-JAN) and the seasons are along the new dimension. + Same as ds but the time axis is now yearly (YS-JAN) and the seasons are along the new dimension. Notes ----- @@ -1054,9 +1054,9 @@ def unstack_dates( - For MS, the coordinates are the month abbreviations in english (JAN, FEB, etc.) - For ?QS-? and other ?MS frequencies, the coordinates are the initials of the months in each season. Ex: QS-DEC (with winter_starts_year=True) : DJF, MAM, JJA, SON. - - For YS or AS-JAN, the new coordinate has a single value of "annual". - - For ?AS-? frequencies, the new coordinate has a single value of "annual-{anchor}", were "anchor" - is the abbreviation of the first month of the year. Ex: AS-JUL -> "annual-JUL". + - For YS or YS-JAN, the new coordinate has a single value of "annual". + - For ?YS-? frequencies, the new coordinate has a single value of "annual-{anchor}", were "anchor" + is the abbreviation of the first month of the year. Ex: YS-JUL -> "annual-JUL". """ # Get some info about the time axis freq = xr.infer_freq(ds.time) From 26bcf1786f57e49861ce7f342d32dad6ec36cfc1 Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:40:22 -0500 Subject: [PATCH 04/30] temporary removal of measure --- templates/1-basic_workflow_with_config/properties1.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/1-basic_workflow_with_config/properties1.yml b/templates/1-basic_workflow_with_config/properties1.yml index 483d0ff3..e97b0da3 100644 --- a/templates/1-basic_workflow_with_config/properties1.yml +++ b/templates/1-basic_workflow_with_config/properties1.yml @@ -31,4 +31,5 @@ indicators: long_name: Mean precipitation input: da: pr - measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. + #FIXME: Remove until xclim releases 0.49. + #measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. From 033d43b044a0b2de599e41f1b78566fe6b43fa19 Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:55:11 -0500 Subject: [PATCH 05/30] install xclim branch temporarily --- environment-dev.yml | 4 +++- templates/1-basic_workflow_with_config/properties1.yml | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index dfe1106a..29b7f9cd 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -30,7 +30,7 @@ dependencies: - sparse - toolz - xarray >=2023.11.0 - - xclim >=0.48.0 + #- xclim >=0.48.0 - xesmf >=0.7 - zarr # Opt @@ -71,3 +71,5 @@ dependencies: # packaging - build - wheel + - pip: # temporary fix until xclim 0.49 release + "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" diff --git a/templates/1-basic_workflow_with_config/properties1.yml b/templates/1-basic_workflow_with_config/properties1.yml index e97b0da3..483d0ff3 100644 --- a/templates/1-basic_workflow_with_config/properties1.yml +++ b/templates/1-basic_workflow_with_config/properties1.yml @@ -31,5 +31,4 @@ indicators: long_name: Mean precipitation input: da: pr - #FIXME: Remove until xclim releases 0.49. - #measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. + measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. From 8a91851ff66e9155af54fee18b917eca374af346 Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:57:16 -0500 Subject: [PATCH 06/30] tiret --- environment-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index 29b7f9cd..729e6e9c 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -72,4 +72,4 @@ dependencies: - build - wheel - pip: # temporary fix until xclim 0.49 release - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" + - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" From 35d606098122378568cf8b219c70af83e11894e6 Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:57:39 -0500 Subject: [PATCH 07/30] format --- environment-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index 729e6e9c..2a2b7b62 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -71,5 +71,6 @@ dependencies: # packaging - build - wheel - - pip: # temporary fix until xclim 0.49 release + # temporary fix until xclim 0.49 release + - pip: - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" From b275535039451dadd549a7603c4ac80abfea91bd Mon Sep 17 00:00:00 2001 From: RondeauG Date: Thu, 22 Feb 2024 10:39:09 -0500 Subject: [PATCH 08/30] activate testing for 3.12 --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3a8add64..06d21c58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,8 +68,8 @@ jobs: tox-build: "py310-coveralls" - python-version: "3.11" tox-build: "py311-coveralls" -# - python-version: "3.12" -# tox-build: "py312-esmpy-coveralls" + - python-version: "3.12" + tox-build: "py312-esmpy-coveralls" defaults: run: shell: bash -l {0} @@ -129,7 +129,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] defaults: run: shell: bash -l {0} From 1a265b96937455be02e90ac28a97521135a19f9e Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 21 Feb 2024 15:10:59 -0500 Subject: [PATCH 09/30] xclim 0.48 compatibility --- environment-dev.yml | 6 +++--- environment.yml | 6 +++--- pyproject.toml | 6 +++--- tests/test_catutils.py | 4 ++-- tests/test_diagnostics.py | 4 ++-- tests/test_indicators.py | 18 +++++++++--------- xscen/CVs/frequency_to_timedelta.json | 6 +++--- xscen/CVs/frequency_to_xrfreq.json | 6 +++--- xscen/CVs/xrfreq_to_frequency.json | 7 +++++-- xscen/CVs/xrfreq_to_timedelta.json | 5 ++++- xscen/indicators.py | 4 ++-- 11 files changed, 39 insertions(+), 33 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 1a2bcfd1..a7793d1d 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,7 +2,7 @@ name: xscen-dev channels: - conda-forge dependencies: - - python >=3.9 + - python >=3.9,<3.13 # Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py! # Also consider updating the list in xs.utils.show_versions if you add a new package. # Main packages @@ -21,7 +21,7 @@ dependencies: - netCDF4 - numcodecs - numpy - - pandas >=2.0,<2.2 + - pandas >=2.2 - parse - pyyaml - rechunker @@ -29,7 +29,7 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray <2023.11.0 # FIXME: Remove when pandas 2.2 is released and xclim is fixed. + - xarray - xclim >=0.46.0 - xesmf >=0.7 - zarr diff --git a/environment.yml b/environment.yml index 0dfde47b..ac9babb2 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xscen channels: - conda-forge dependencies: - - python >=3.9 + - python >=3.9,<3.13 # Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py! # Also consider updating the list in xs.utils.show_versions if you add a new package. # Main packages @@ -21,7 +21,7 @@ dependencies: - netCDF4 - numcodecs - numpy - - pandas >=2.0,<2.2 + - pandas >=2.2 - parse - pyyaml - rechunker @@ -29,7 +29,7 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray <2023.11.0 # FIXME: Remove when pandas 2.2 is released and xclim is fixed. + - xarray - xclim >=0.46.0 - xesmf >=0.7 - zarr diff --git a/pyproject.toml b/pyproject.toml index 7f47cf4f..122edd10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - # "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Atmospheric Science" ] @@ -52,7 +52,7 @@ dependencies = [ "netCDF4", "numcodecs", "numpy", - "pandas>=2.0,<2.2", + "pandas>=2.2", "parse", # Used when opening catalogs. "pyarrow", @@ -62,7 +62,7 @@ dependencies = [ "shapely>=2.0", "sparse", "toolz", - "xarray<2023.11.0", # FIXME: Remove when pandas 2.2 is released and xclim is fixed. + "xarray", "xclim>=0.46.0", "zarr" ] diff --git a/tests/test_catutils.py b/tests/test_catutils.py index 4b9cf5cd..6748c3c2 100644 --- a/tests/test_catutils.py +++ b/tests/test_catutils.py @@ -230,7 +230,7 @@ def test_build_path(samplecat): def test_build_path_ds(): ds = xr.tutorial.open_dataset("air_temperature") - ds = ds.assign(time=xr.cftime_range("0001-01-01", freq="6H", periods=ds.time.size)) + ds = ds.assign(time=xr.cftime_range("0001-01-01", freq="6h", periods=ds.time.size)) ds.attrs.update(source="source", institution="institution") new_path = cu.build_path( ds, @@ -240,7 +240,7 @@ def test_build_path_ds(): }, ) assert new_path == Path( - "source/institution/air_6H/source_institution_air_6hr_0001-0002" + "source/institution/air_6h/source_institution_air_6hr_0001-0002" ) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index c23336c6..9fc87c41 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -127,7 +127,7 @@ def test_variables(self): xs.diagnostics.health_checks(ds, variables_and_units={"tas": "degC"}) with pytest.warns( UserWarning, - match="Data units kelvin are not compatible with requested mm.", + match="are not compatible with requested mm.", ): xs.diagnostics.health_checks(ds, variables_and_units={"tas": "mm"}) @@ -164,7 +164,7 @@ def test_cfchecks(self): xs.diagnostics.health_checks(ds, cfchecks=bad_cfcheck) @pytest.mark.parametrize( - "freq, gap", [("D", False), ("MS", False), ("3H", False), ("D", True)] + "freq, gap", [("D", False), ("MS", False), ("3h", False), ("D", True)] ) def test_freq(self, freq, gap): ds = timeseries( diff --git a/tests/test_indicators.py b/tests/test_indicators.py index ad4c2503..3b7dd0b2 100644 --- a/tests/test_indicators.py +++ b/tests/test_indicators.py @@ -48,12 +48,12 @@ def test_input_types(self, input): def test_level(self, to_level): if to_level is None: ind_dict = xs.compute_indicators(self.ds, indicators=self.yaml_file) - assert "indicators" in ind_dict["AS-JAN"].attrs["cat:processing_level"] + assert "indicators" in ind_dict["YS-JAN"].attrs["cat:processing_level"] else: ind_dict = xs.compute_indicators( self.ds, indicators=self.yaml_file, to_level=to_level ) - assert to_level in ind_dict["AS-JAN"].attrs["cat:processing_level"] + assert to_level in ind_dict["YS-JAN"].attrs["cat:processing_level"] # Periods needed to cover both branches of the if/else in compute_indicators, so no need to test separately. @pytest.mark.parametrize( @@ -76,13 +76,13 @@ def test_output(self, periods): assert all( xrfreq in ind_dict[xrfreq].attrs["cat:xrfreq"] for xrfreq in ind_dict.keys() ) - assert all(v in ind_dict["AS-JAN"] for v in ["tg_min", "growing_degree_days"]) + assert all(v in ind_dict["YS-JAN"] for v in ["tg_min", "growing_degree_days"]) if periods is None: assert "time" not in ind_dict["fx"].dims - assert len(ind_dict["AS-JAN"].time) == 2 + assert len(ind_dict["YS-JAN"].time) == 2 else: assert len(ind_dict["fx"].time) == len(periods) - assert len(ind_dict["AS-JAN"].time) == 8 + assert len(ind_dict["YS-JAN"].time) == 8 def test_qs_dec(self): indicator = xclim.core.indicator.Indicator.from_dict( @@ -118,7 +118,7 @@ def test_multiple_outputs(self, periods): ) assert all( - v in ind_dict["AS-JAN"] + v in ind_dict["YS-JAN"] for v in [ "prcpavg", "rain_season_start", @@ -130,7 +130,7 @@ def test_multiple_outputs(self, periods): @pytest.mark.parametrize("restrict_years", [True, False]) def test_as_jul(self, restrict_years): indicator = xclim.core.indicator.Indicator.from_dict( - data={"base": "freezing_degree_days", "parameters": {"freq": "AS-JUL"}}, + data={"base": "freezing_degree_days", "parameters": {"freq": "YS-JUL"}}, identifier="degree_days_below_0_annual_start_july", module="tests", ) @@ -140,8 +140,8 @@ def test_as_jul(self, restrict_years): restrict_years=restrict_years, ) - assert "AS-JUL" in ind_dict["AS-JUL"].attrs["cat:xrfreq"] - out = ind_dict["AS-JUL"] + assert "YS-JUL" in ind_dict["YS-JUL"].attrs["cat:xrfreq"] + out = ind_dict["YS-JUL"] if restrict_years: assert len(out.time) == 3 # same length as input ds assert out.time[0].dt.strftime("%Y-%m-%d").item() == "2001-07-01" diff --git a/xscen/CVs/frequency_to_timedelta.json b/xscen/CVs/frequency_to_timedelta.json index 910dfdbe..7152502a 100644 --- a/xscen/CVs/frequency_to_timedelta.json +++ b/xscen/CVs/frequency_to_timedelta.json @@ -1,7 +1,7 @@ { - "1hr": "1H", - "3hr": "3H", - "6hr": "6H", + "1hr": "1h", + "3hr": "3h", + "6hr": "6h", "day": "1D", "sem": "1W", "2sem": "2W", diff --git a/xscen/CVs/frequency_to_xrfreq.json b/xscen/CVs/frequency_to_xrfreq.json index 394aa332..3c65edf6 100644 --- a/xscen/CVs/frequency_to_xrfreq.json +++ b/xscen/CVs/frequency_to_xrfreq.json @@ -1,7 +1,7 @@ { - "1hr": "H", - "3hr": "3H", - "6hr": "6H", + "1hr": "h", + "3hr": "3h", + "6hr": "6h", "day": "D", "sem": "W", "2sem": "2W", diff --git a/xscen/CVs/xrfreq_to_frequency.json b/xscen/CVs/xrfreq_to_frequency.json index 81f9149d..acda19fa 100644 --- a/xscen/CVs/xrfreq_to_frequency.json +++ b/xscen/CVs/xrfreq_to_frequency.json @@ -1,16 +1,19 @@ { "is_regex": true, + "h": "1hr", "H": "1hr", + "3h": "3hr", "3H": "3hr", + "6h": "6hr", "6H": "6hr", "D": "day", "W": "sem", "2W": "2sem", - "14d": "2sem", + "14D": "2sem", "M.*": "mon", "Q.*": "qtr", "2Q.*": "6mon", "A.*": "yr", - "YS": "yr", + "Y.*": "yr", "fx": "fx" } diff --git a/xscen/CVs/xrfreq_to_timedelta.json b/xscen/CVs/xrfreq_to_timedelta.json index 3dc7906e..7fbf42e6 100644 --- a/xscen/CVs/xrfreq_to_timedelta.json +++ b/xscen/CVs/xrfreq_to_timedelta.json @@ -1,7 +1,10 @@ { "is_regex": true, + "h": "1h", "H": "1H", + "3h": "3h", "3H": "3H", + "6h": "6h", "6H": "6H", "D": "1D", "W": "7D", @@ -10,6 +13,6 @@ "Q.*": "90D", "2Q.*": "180D", "A.*": "365D", - "YS": "365D", + "Y.*": "365D", "fx": "NAN" } diff --git a/xscen/indicators.py b/xscen/indicators.py index 85b4d0b4..02e8ff6e 100644 --- a/xscen/indicators.py +++ b/xscen/indicators.py @@ -170,7 +170,7 @@ def _infer_freq_from_meta(ind): else: freq = "fx" if freq == "YS": - freq = "AS-JAN" # To fix an inconsistency in xclim default 'freq' + freq = "YS-JAN" else: # Multiple time periods to concatenate @@ -197,7 +197,7 @@ def _infer_freq_from_meta(ind): freq = "fx" if freq == "YS": - freq = "AS-JAN" # To fix an inconsistency in xclim default 'freq' + freq = "YS-JAN" # In order to concatenate time periods, the indicator still needs a time dimension if freq == "fx": tmp = tmp.assign_coords({"time": ds_subset.time[0]}) From 15b4d74bb5aa06a7a30d1e53fbceec308c6d7ef4 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 21 Feb 2024 15:27:50 -0500 Subject: [PATCH 10/30] pins --- environment-dev.yml | 4 ++-- environment.yml | 4 ++-- pyproject.toml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index a7793d1d..ab79a5d7 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -29,8 +29,8 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray - - xclim >=0.46.0 + - xarray >=2023.11.0 + - xclim >=0.48.0 - xesmf >=0.7 - zarr # Opt diff --git a/environment.yml b/environment.yml index ac9babb2..c9982c4f 100644 --- a/environment.yml +++ b/environment.yml @@ -29,8 +29,8 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray - - xclim >=0.46.0 + - xarray >=2023.11.0 + - xclim >=0.48.0 - xesmf >=0.7 - zarr # To install from source and get translations diff --git a/pyproject.toml b/pyproject.toml index 122edd10..c06e9728 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,8 +62,8 @@ dependencies = [ "shapely>=2.0", "sparse", "toolz", - "xarray", - "xclim>=0.46.0", + "xarray>=2023.11.0", + "xclim>=0.48.0", "zarr" ] From 4ed09a655121b25418d52d354af4ad22afd2a4e7 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 21 Feb 2024 15:54:08 -0500 Subject: [PATCH 11/30] converted missing AS-JAN --- xscen/diagnostics.py | 2 +- xscen/utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xscen/diagnostics.py b/xscen/diagnostics.py index d37668e0..41c9192c 100644 --- a/xscen/diagnostics.py +++ b/xscen/diagnostics.py @@ -241,7 +241,7 @@ def _message(): _error( "The timesteps are irregular or cannot be inferred by xarray.", "freq" ) - elif freq.replace("YS", "AS-JAN") != inferred_freq: + elif freq.replace("YS", "YS-JAN") != inferred_freq: _error( f"The frequency is not '{freq}'. Received '{inferred_freq}'.", "freq" ) diff --git a/xscen/utils.py b/xscen/utils.py index 68b72bc9..965e0f66 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -1045,7 +1045,7 @@ def unstack_dates( Returns ------- xr.Dataset or DataArray - Same as ds but the time axis is now yearly (AS-JAN) and the seasons are along the new dimension. + Same as ds but the time axis is now yearly (YS-JAN) and the seasons are along the new dimension. Notes ----- @@ -1054,9 +1054,9 @@ def unstack_dates( - For MS, the coordinates are the month abbreviations in english (JAN, FEB, etc.) - For ?QS-? and other ?MS frequencies, the coordinates are the initials of the months in each season. Ex: QS-DEC (with winter_starts_year=True) : DJF, MAM, JJA, SON. - - For YS or AS-JAN, the new coordinate has a single value of "annual". - - For ?AS-? frequencies, the new coordinate has a single value of "annual-{anchor}", were "anchor" - is the abbreviation of the first month of the year. Ex: AS-JUL -> "annual-JUL". + - For YS or YS-JAN, the new coordinate has a single value of "annual". + - For ?YS-? frequencies, the new coordinate has a single value of "annual-{anchor}", were "anchor" + is the abbreviation of the first month of the year. Ex: YS-JUL -> "annual-JUL". """ # Get some info about the time axis freq = xr.infer_freq(ds.time) From 6b945c6cef7a030eec62fee66fd8e43dce6883ae Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:40:22 -0500 Subject: [PATCH 12/30] temporary removal of measure --- templates/1-basic_workflow_with_config/properties1.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/1-basic_workflow_with_config/properties1.yml b/templates/1-basic_workflow_with_config/properties1.yml index 483d0ff3..e97b0da3 100644 --- a/templates/1-basic_workflow_with_config/properties1.yml +++ b/templates/1-basic_workflow_with_config/properties1.yml @@ -31,4 +31,5 @@ indicators: long_name: Mean precipitation input: da: pr - measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. + #FIXME: Remove until xclim releases 0.49. + #measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. From a169a3ca2930b63a1febcb3f9dc807562c379d0e Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:55:11 -0500 Subject: [PATCH 13/30] install xclim branch temporarily --- environment-dev.yml | 4 +++- templates/1-basic_workflow_with_config/properties1.yml | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index ab79a5d7..79bdf546 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -30,7 +30,7 @@ dependencies: - sparse - toolz - xarray >=2023.11.0 - - xclim >=0.48.0 + #- xclim >=0.48.0 - xesmf >=0.7 - zarr # Opt @@ -72,3 +72,5 @@ dependencies: # packaging - build - wheel + - pip: # temporary fix until xclim 0.49 release + "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" diff --git a/templates/1-basic_workflow_with_config/properties1.yml b/templates/1-basic_workflow_with_config/properties1.yml index e97b0da3..483d0ff3 100644 --- a/templates/1-basic_workflow_with_config/properties1.yml +++ b/templates/1-basic_workflow_with_config/properties1.yml @@ -31,5 +31,4 @@ indicators: long_name: Mean precipitation input: da: pr - #FIXME: Remove until xclim releases 0.49. - #measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. + measure: xclim.sdba.measures.RATIO # This will instruct xclim to perform a ratio between the given input and the reference data. From c31c271dae16f5eb59e7b10b22bfd3204939ac17 Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:57:16 -0500 Subject: [PATCH 14/30] tiret --- environment-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index 79bdf546..25af2f16 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -73,4 +73,4 @@ dependencies: - build - wheel - pip: # temporary fix until xclim 0.49 release - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" + - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" From 18402d08c93f32bdb2b8b6b49ba5d3c58256525d Mon Sep 17 00:00:00 2001 From: juliettelavoie Date: Wed, 21 Feb 2024 16:57:39 -0500 Subject: [PATCH 15/30] format --- environment-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index 25af2f16..cdad898e 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -72,5 +72,6 @@ dependencies: # packaging - build - wheel - - pip: # temporary fix until xclim 0.49 release + # temporary fix until xclim 0.49 release + - pip: - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" From 46311d4d22b58db3f60cd67e607f32e2ec66320b Mon Sep 17 00:00:00 2001 From: RondeauG Date: Thu, 22 Feb 2024 10:39:09 -0500 Subject: [PATCH 16/30] activate testing for 3.12 --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3a8add64..06d21c58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,8 +68,8 @@ jobs: tox-build: "py310-coveralls" - python-version: "3.11" tox-build: "py311-coveralls" -# - python-version: "3.12" -# tox-build: "py312-esmpy-coveralls" + - python-version: "3.12" + tox-build: "py312-esmpy-coveralls" defaults: run: shell: bash -l {0} @@ -129,7 +129,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] defaults: run: shell: bash -l {0} From fe90910b0db78967cce27f65d0ea492b43273c28 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 22 Feb 2024 11:41:15 -0500 Subject: [PATCH 17/30] install coverage via PyPI --- environment-dev.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index cdad898e..a679580a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -41,8 +41,8 @@ dependencies: - black ==24.1.1 - blackdoc ==0.3.9 - bump-my-version >=0.17.1 - - coverage>=6.2.2,<7.0.0 - - coveralls>=3.3.1 +# - coverage>=6.2.2,<7.0.0 +# - coveralls>=3.3.1 - flake8 >=6.1.0 - flake8-rst-docstrings>=0.3.0 - ipykernel @@ -55,7 +55,7 @@ dependencies: - pooch - pre-commit >=3.3.2 - pytest >=7.3.1 - - pytest-cov >=4.0.0 +# - pytest-cov >=4.0.0 - ruff >=0.1.0 - sphinx - sphinx-autoapi @@ -66,12 +66,16 @@ dependencies: - sphinx-mdinclude - watchdog >=3.0.0 - xdoctest - - pip # Testing - tox >=4.5.1 # packaging - build - wheel - # temporary fix until xclim 0.49 release + - pip - pip: + # temporary fix until xclim 0.49 release - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" + # coverage is not available in conda-forge for Python3.12 + - coverage>=6.2.2,<7.0.0 + - coveralls>=3.3.1 + - pytest-cov >=4.0.0 From 75e13dca617a47846bb47591031e53afca4b47ea Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 23 Feb 2024 16:43:29 -0500 Subject: [PATCH 18/30] Steal code from xarray - ensure new xrfreq on cat open and parsing --- tests/test_catalog.py | 6 ++++++ xscen/catalog.py | 11 +++++++++-- xscen/catutils.py | 6 ++++++ xscen/utils.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index e04c405f..abb324ac 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,4 +1,5 @@ import pandas as pd +from conftest import SAMPLES_DIR from xscen import catalog @@ -29,3 +30,8 @@ def test_subset_file_coverage(): catalog.subset_file_coverage(df, [1951, 1976], coverage=0.8), pd.DataFrame(columns=df.columns), ) + + +def test_xrfreq_fix(): + cat = catalog.DataCatalog(SAMPLES_DIR.parent / "pangeo-cmip6.json") + assert set(cat.df.xrfreq) == {"3h", "D", "fx"} diff --git a/xscen/catalog.py b/xscen/catalog.py index 97016655..d10d3fb0 100644 --- a/xscen/catalog.py +++ b/xscen/catalog.py @@ -22,7 +22,13 @@ from intake_esm.cat import ESMCatalogModel from .config import CONFIG, args_as_str, recursive_update -from .utils import CV, date_parser, ensure_correct_time, standardize_periods # noqa +from .utils import ( # noqa + CV, + date_parser, + ensure_correct_time, + ensure_new_xrfreq, + standardize_periods, +) logger = logging.getLogger(__name__) # Monkey patch for attribute names in the output of to_dataset_dict @@ -126,10 +132,11 @@ def _parse_dates(elem): "dtype": { key: "category" if not key == "path" else "string[pyarrow]" for key in COLUMNS - if key not in ["variable", "date_start", "date_end"] + if key not in ["xrfreq", "variable", "date_start", "date_end"] }, "converters": { "variable": _parse_list_of_strings, + "xrfreq": ensure_new_xrfreq, }, } """Kwargs to pass to `pd.read_csv` when opening an official Ouranos catalog.""" diff --git a/xscen/catutils.py b/xscen/catutils.py index d7ae4ebb..87a29d6d 100644 --- a/xscen/catutils.py +++ b/xscen/catutils.py @@ -34,6 +34,7 @@ CV, date_parser, ensure_correct_time, + ensure_new_xrfreq, get_cat_attrs, standardize_periods, ) @@ -634,6 +635,10 @@ def parse_directory( # noqa:C901 if cvs: df = df.apply(_replace_in_row, axis=1, replacements=cvs) + # Fix potential legacy xrfreq + if "xrfreq" in df.columns: + df["xrfreq"] = df["xrfreq"].apply(ensure_new_xrfreq) + # translate xrfreq into frequencies and vice-versa if {"xrfreq", "frequency"}.issubset(df.columns): df["xrfreq"].fillna( @@ -643,6 +648,7 @@ def parse_directory( # noqa:C901 df["xrfreq"].apply(CV.xrfreq_to_frequency, default=pd.NA), inplace=True ) + # Esu # Parse dates # If we don't do the to_numpy(na_value=np.datetime64('')).astype(' str: + """Convert the frequency string to the newer syntax (pandas >= 2.2) if needed.""" + # Copied from xarray xr.coding.cftime_offsets._legacy_to_new_freq + # https://github.com/pydata/xarray/pull/8627/files + try: + freq_as_offset = cfoff.to_offset(freq, warn=False) + except ValueError: + # freq may be valid in pandas but not in xarray + return freq + + if isinstance(freq_as_offset, cfoff.MonthEnd) and "ME" not in freq: + freq = freq.replace("M", "ME") + elif isinstance(freq_as_offset, cfoff.QuarterEnd) and "QE" not in freq: + freq = freq.replace("Q", "QE") + elif isinstance(freq_as_offset, cfoff.YearBegin) and "YS" not in freq: + freq = freq.replace("AS", "YS") + elif isinstance(freq_as_offset, cfoff.YearEnd): + if "A-" in freq: + # Check for and replace "A-" instead of just "A" to prevent + # corrupting anchored offsets that contain "Y" in the month + # abbreviation, e.g. "A-MAY" -> "YE-MAY". + freq = freq.replace("A-", "YE-") + elif "Y-" in freq: + freq = freq.replace("Y-", "YE-") + elif freq.endswith("A"): + # the "A-MAY" case is already handled above + freq = freq.replace("A", "YE") + elif "YE" not in freq and freq.endswith("Y"): + # the "Y-MAY" case is already handled above + freq = freq.replace("Y", "YE") + elif isinstance(freq_as_offset, cfoff.Hour): + freq = freq.replace("H", "h") + elif isinstance(freq_as_offset, cfoff.Minute): + freq = freq.replace("T", "min") + elif isinstance(freq_as_offset, cfoff.Second): + freq = freq.replace("S", "s") + elif isinstance(freq_as_offset, cfoff.Millisecond): + freq = freq.replace("L", "ms") + elif isinstance(freq_as_offset, cfoff.Microsecond): + freq = freq.replace("U", "us") + + return freq From 4fe0ae362a50d1546b90d9bcae9252a85faab5dd Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:09:32 -0500 Subject: [PATCH 19/30] test against xclim main development branch --- .github/workflows/main.yml | 16 +++++++++++++--- tox.ini | 1 + 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 06d21c58..1ae5ff7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,7 +67,7 @@ jobs: - python-version: "3.10" tox-build: "py310-coveralls" - python-version: "3.11" - tox-build: "py311-coveralls" + tox-build: "py311-xclim-coveralls" - python-version: "3.12" tox-build: "py312-esmpy-coveralls" defaults: @@ -129,7 +129,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + include: + - python-version: "3.9" + intake-esm-downgrade: true + - python-version: "3.10" + - python-version: "3.11" + - python-version: "3.12" + xclim-upstream: true defaults: run: shell: bash -l {0} @@ -148,9 +154,13 @@ jobs: mamba python=${{ matrix.python-version }} - name: Downgrade intake-esm - if: matrix.python-version == '3.9' + if: matrix.intake-esm-downgrade run: | micromamba install -y -c conda-forge intake-esm=2023.11.10 + - name: Install xclim main branch + if: matrix.xclim-upstream + run: | + python -m pip install git+https://github.com/Ouranosinc/xclim.git@main - name: Conda and Mamba versions run: | mamba --version diff --git a/tox.ini b/tox.ini index bc187cbf..e5219c2f 100644 --- a/tox.ini +++ b/tox.ini @@ -69,6 +69,7 @@ download = true deps = coveralls: coveralls esmpy: git+https://github.com/esmf-org/esmf.git@v{env:ESMF_VERSION}\#subdirectory=src/addon/esmpy + xclim: git+https://github.com/Ouranosinc/xclim.git@main extras = dev conda_channels = From c35baca9eba69af48cf6711bca48430d439d3273 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:12:19 -0500 Subject: [PATCH 20/30] main -> master --- .github/workflows/main.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ae5ff7a..d661e0d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -160,7 +160,7 @@ jobs: - name: Install xclim main branch if: matrix.xclim-upstream run: | - python -m pip install git+https://github.com/Ouranosinc/xclim.git@main + python -m pip install git+https://github.com/Ouranosinc/xclim.git@master - name: Conda and Mamba versions run: | mamba --version diff --git a/tox.ini b/tox.ini index e5219c2f..adff0828 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ download = true deps = coveralls: coveralls esmpy: git+https://github.com/esmf-org/esmf.git@v{env:ESMF_VERSION}\#subdirectory=src/addon/esmpy - xclim: git+https://github.com/Ouranosinc/xclim.git@main + xclim: git+https://github.com/Ouranosinc/xclim.git@master extras = dev conda_channels = From 6331870bc6aa173e2230f23b63004c5dbf8944b0 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:28:25 -0500 Subject: [PATCH 21/30] remove branch install in environment-dev.yml --- environment-dev.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index a679580a..33677f6f 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -73,8 +73,6 @@ dependencies: - wheel - pip - pip: - # temporary fix until xclim 0.49 release - - "git+https://github.com/Ouranosinc/xclim.git@fix-yamale-measure" # coverage is not available in conda-forge for Python3.12 - coverage>=6.2.2,<7.0.0 - coveralls>=3.3.1 From e3f0473c92122e1c57f19ce0b6cae581cbb92c77 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:38:01 -0500 Subject: [PATCH 22/30] install xclim --- environment-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index 33677f6f..a209bcf9 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -30,7 +30,7 @@ dependencies: - sparse - toolz - xarray >=2023.11.0 - #- xclim >=0.48.0 + - xclim >=0.48.0 - xesmf >=0.7 - zarr # Opt From 55bd893d66925d2c24b7308168e6cfad4f39bdd0 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Mon, 26 Feb 2024 11:15:34 -0500 Subject: [PATCH 23/30] Skip freq converting in more cases --- xscen/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xscen/utils.py b/xscen/utils.py index 6d898696..4e0adad3 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -1350,10 +1350,13 @@ def xrfreq_to_timedelta(freq: str): return N * pd.Timedelta(CV.xrfreq_to_timedelta(B, "NaT")) -def ensure_new_xrfreq(freq: str) -> str: +def ensure_new_xrfreq(freq: str) -> str: # noqa: C901 """Convert the frequency string to the newer syntax (pandas >= 2.2) if needed.""" # Copied from xarray xr.coding.cftime_offsets._legacy_to_new_freq # https://github.com/pydata/xarray/pull/8627/files + if not isinstance(freq, str): + # For when freq is NaN or None in a catalog + return freq try: freq_as_offset = cfoff.to_offset(freq, warn=False) except ValueError: From e0ea3ffc7b6bef550ed33291a9d21c5d5a208305 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:22:17 -0500 Subject: [PATCH 24/30] master -> main --- .github/workflows/main.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d661e0d4..52b07a50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,7 @@ jobs: python -m tox -e lint test-pypi: - name: Test with Python${{ matrix.python-version }} (PyPI/tox) + name: ${{ matrix.tox-build }} (Python${{ matrix.python-version }}) needs: lint runs-on: ubuntu-latest env: @@ -124,16 +124,18 @@ jobs: # COVERALLS_SERVICE_NAME: github test-conda: - name: Test with Python${{ matrix.python-version }} (Anaconda) + name: Python${{ matrix.python-version }} (conda, upstream=${{ matrix.xclim-upstream }}) needs: lint runs-on: ubuntu-latest strategy: matrix: include: - python-version: "3.9" - intake-esm-downgrade: true + xclim-upstream: false - python-version: "3.10" + xclim-upstream: false - python-version: "3.11" + xclim-upstream: false - python-version: "3.12" xclim-upstream: true defaults: @@ -154,13 +156,13 @@ jobs: mamba python=${{ matrix.python-version }} - name: Downgrade intake-esm - if: matrix.intake-esm-downgrade + if: matrix.python-version == '3.9' run: | micromamba install -y -c conda-forge intake-esm=2023.11.10 - name: Install xclim main branch if: matrix.xclim-upstream run: | - python -m pip install git+https://github.com/Ouranosinc/xclim.git@master + python -m pip install git+https://github.com/Ouranosinc/xclim.git@main - name: Conda and Mamba versions run: | mamba --version From d741d283d463df3626ddb393990d223793e819b7 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:27:12 -0500 Subject: [PATCH 25/30] set base xclim pin to v0.48.2, formatting --- environment-dev.yml | 2 +- environment.yml | 2 +- pyproject.toml | 50 ++++++++++++++++++++++----------------------- tox.ini | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index a209bcf9..fe5b5405 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -30,7 +30,7 @@ dependencies: - sparse - toolz - xarray >=2023.11.0 - - xclim >=0.48.0 + - xclim >=0.48.2 - xesmf >=0.7 - zarr # Opt diff --git a/environment.yml b/environment.yml index c9982c4f..e827ccc1 100644 --- a/environment.yml +++ b/environment.yml @@ -30,7 +30,7 @@ dependencies: - sparse - toolz - xarray >=2023.11.0 - - xclim >=0.48.0 + - xclim >=0.48.2 - xesmf >=0.7 - zarr # To install from source and get translations diff --git a/pyproject.toml b/pyproject.toml index c06e9728..4944bb88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ - "setuptools>=60", - "setuptools-scm>=8.0", + "setuptools >=60", + "setuptools-scm >=8.0", "babel", "wheel" ] @@ -38,8 +38,8 @@ dependencies = [ "babel", "cartopy", "cftime", - "cf_xarray>=0.7.6", - "clisops>=0.10", + "cf_xarray >=0.7.6", + "clisops >=0.10", "dask", "flox", "fsspec", @@ -52,41 +52,41 @@ dependencies = [ "netCDF4", "numcodecs", "numpy", - "pandas>=2.2", + "pandas >=2.2", "parse", # Used when opening catalogs. "pyarrow", "pyyaml", "rechunker", "scipy", - "shapely>=2.0", + "shapely >=2.0", "sparse", "toolz", - "xarray>=2023.11.0", - "xclim>=0.48.0", + "xarray >=2023.11.0", + "xclim >=0.48.2", "zarr" ] [project.optional-dependencies] dev = [ # Dev tools and testing - "pip>=23.3.0", - "black==24.2.0", - "blackdoc==0.3.9", - "bump-my-version>=0.17.1", - "coverage>=6.2.2,<8.0.0", - "coveralls>=3.3.1", - "flake8-alphabetize>=0.0.21", - "flake8-rst-docstrings>=0.3.0", - "flake8>=6.1.0", - "isort==5.13.2", + "pip >=23.3.0", + "black ==24.2.0", + "blackdoc ==0.3.9", + "bump-my-version >=0.17.1", + "coverage >=6.2.2,<8.0.0", + "coveralls >=3.3.1", + "flake8-alphabetize >=0.0.21", + "flake8-rst-docstrings >=0.3.0", + "flake8 >=6.1.0", + "isort ==5.13.2", "pooch", - "pre-commit>=3.3.2", - "pytest-cov>=4.0.0", - "pytest>=7.3.1", - "ruff>=0.1.0", - "tox>=4.5.1", - "watchdog>=3.0.0", + "pre-commit >=3.3.2", + "pytest-cov >=4.0.0", + "pytest >=7.3.1", + "ruff >=0.1.0", + "tox >=4.5.1", + "watchdog >=3.0.0", "xdoctest" ] docs = [ @@ -102,7 +102,7 @@ docs = [ "sphinx-copybutton", "sphinx-intl", "sphinx-mdinclude", - "sphinx-rtd-theme>=1.0", + "sphinx-rtd-theme >=1.0", "sphinxcontrib-napoleon" ] extra = [ diff --git a/tox.ini b/tox.ini index adff0828..e5219c2f 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ download = true deps = coveralls: coveralls esmpy: git+https://github.com/esmf-org/esmf.git@v{env:ESMF_VERSION}\#subdirectory=src/addon/esmpy - xclim: git+https://github.com/Ouranosinc/xclim.git@master + xclim: git+https://github.com/Ouranosinc/xclim.git@main extras = dev conda_channels = From cf01d49353233e4a15de9bf2acb88a62937fadf9 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Tue, 27 Feb 2024 11:44:35 -0500 Subject: [PATCH 26/30] upd changes --- CHANGES.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fbdd1905..6f689d43 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,16 +4,23 @@ Changelog v0.8.3 (unreleased) ------------------- -Contributors to this version: Juliette Lavoie (:user:`juliettelavoie`), Trevor James Smith (:user:`Zeitsperre`). +Contributors to this version: Juliette Lavoie (:user:`juliettelavoie`), Trevor James Smith (:user:`Zeitsperre`), Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Bourgault (:user:`aulemahal`). Announcements ^^^^^^^^^^^^^ * `xscen` is now has a `security disclosure policy `_. (:pull:`353`). +* Various frequency-related changes to match the new `pandas` naming conventions. (:pull:`351`). Internal changes ^^^^^^^^^^^^^^^^ * Added tests for diagnostics. (:pull:`352`). +Breaking changes +^^^^^^^^^^^^^^^^ +* `xscen` now requires `pandas` >= 2.2 (:pull:`351`). +* Functions that output a dict with keys as xrfreq (such as ``extract_dataset``, ``compute_indicators``) will now return the new nomenclature (e.g. "YS-JAN" instead of "AS-JAN"). (:pull:`351`). +* Going from `xrfreq` to frequencies or timedeltas will still work, but the opposite (frequency --> xrfreq/timedelta) will now only result in the new pandas nomenclature. (:pull:`351`). + v0.8.2 (2024-02-12) ------------------- Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Pascal Bourgault (:user:`aulemahal`) From a9502982f16c3bdd9dddfa60792f70aaf376a217 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Tue, 27 Feb 2024 12:13:27 -0500 Subject: [PATCH 27/30] Update xrfreq to new syntax in notebook --- docs/notebooks/1_catalog.ipynb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/notebooks/1_catalog.ipynb b/docs/notebooks/1_catalog.ipynb index 058fc25b..f6d8f20f 100644 --- a/docs/notebooks/1_catalog.ipynb +++ b/docs/notebooks/1_catalog.ipynb @@ -324,7 +324,7 @@ "source": [ "import xscen as xs\n", "\n", - "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3H\", \"sftlf\": \"fx\"}\n", + "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3h\", \"sftlf\": \"fx\"}\n", "other_search_criteria = {\"institution\": [\"NOAA-GFDL\"]}\n", "\n", "cat_sim = xs.search_data_catalogs(\n", @@ -376,7 +376,7 @@ }, "outputs": [], "source": [ - "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3H\", \"sftlf\": \"fx\"}\n", + "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3h\", \"sftlf\": \"fx\"}\n", "other_search_criteria = {\"institution\": [\"NOAA-GFDL\"], \"experiment\": [\"ssp585\"]}\n", "\n", "cat_sim = xs.search_data_catalogs(\n", @@ -1129,6 +1129,14 @@ "PC.df[\"path\"] = newdf[\"new_path\"]\n", "PC.update()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7acfa82-e2e7-42ce-b24e-ab5d4acc6c57", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -1146,7 +1154,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.2" } }, "nbformat": 4, From d6b3c13dc5cb562cf63643a0dc2d8a002eb3b1ef Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:15:27 -0500 Subject: [PATCH 28/30] update CHANGES.rst --- CHANGES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6f689d43..6d944165 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,16 +8,18 @@ Contributors to this version: Juliette Lavoie (:user:`juliettelavoie`), Trevor J Announcements ^^^^^^^^^^^^^ -* `xscen` is now has a `security disclosure policy `_. (:pull:`353`). +* `xscen` now has a `security disclosure policy `_. (:pull:`353`). * Various frequency-related changes to match the new `pandas` naming conventions. (:pull:`351`). Internal changes ^^^^^^^^^^^^^^^^ * Added tests for diagnostics. (:pull:`352`). +* Added a `SECURITY.md` file to the repository and the documentation. (:pull:`353`). +* Added `tox` modifier for testing builds against the `main` development branch of `xclim`. (:pull:`351`). Breaking changes ^^^^^^^^^^^^^^^^ -* `xscen` now requires `pandas` >= 2.2 (:pull:`351`). +* `xscen` now requires `pandas` >= 2.2 and `xclim` >= 0.48.2. (:pull:`351`). * Functions that output a dict with keys as xrfreq (such as ``extract_dataset``, ``compute_indicators``) will now return the new nomenclature (e.g. "YS-JAN" instead of "AS-JAN"). (:pull:`351`). * Going from `xrfreq` to frequencies or timedeltas will still work, but the opposite (frequency --> xrfreq/timedelta) will now only result in the new pandas nomenclature. (:pull:`351`). From c833eefe0d7d2a42dd2e285983a0475159283592 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Tue, 27 Feb 2024 13:38:43 -0500 Subject: [PATCH 29/30] Update xscen/catutils.py --- xscen/catutils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xscen/catutils.py b/xscen/catutils.py index 87a29d6d..c6197571 100644 --- a/xscen/catutils.py +++ b/xscen/catutils.py @@ -648,7 +648,6 @@ def parse_directory( # noqa:C901 df["xrfreq"].apply(CV.xrfreq_to_frequency, default=pd.NA), inplace=True ) - # Esu # Parse dates # If we don't do the to_numpy(na_value=np.datetime64('')).astype(' Date: Tue, 27 Feb 2024 13:44:49 -0500 Subject: [PATCH 30/30] Update xscen/CVs/xrfreq_to_timedelta.json Co-authored-by: Pascal Bourgault --- xscen/CVs/xrfreq_to_timedelta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xscen/CVs/xrfreq_to_timedelta.json b/xscen/CVs/xrfreq_to_timedelta.json index 7fbf42e6..06158622 100644 --- a/xscen/CVs/xrfreq_to_timedelta.json +++ b/xscen/CVs/xrfreq_to_timedelta.json @@ -1,11 +1,11 @@ { "is_regex": true, "h": "1h", - "H": "1H", + "H": "1h", "3h": "3h", - "3H": "3H", + "3H": "3h", "6h": "6h", - "6H": "6H", + "6H": "6h", "D": "1D", "W": "7D", "2W": "14D",