Skip to content

Commit

Permalink
address comments from code review
Browse files Browse the repository at this point in the history
  • Loading branch information
RondeauG committed Nov 18, 2024
1 parent 184852d commit e7dd5c3
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 83 deletions.
36 changes: 25 additions & 11 deletions src/xscen/io.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Input/Output functions for xscen."""

import datetime
import json
import logging
import os
import shutil as sh
Expand Down Expand Up @@ -252,7 +253,7 @@ def clean_incomplete(
Name of variables that were completed. All other variables (except coordinates) will be removed.
Use either `complete` or `incomplete`, not both.
incomplete : sequence of strings, optional
Name of variables that should be removed.
Name of variables that should be removed. Coordinates and dimensions will never be removed through this function.
Use either `complete` or `incomplete`, not both.
Returns
Expand All @@ -261,6 +262,22 @@ def clean_incomplete(
"""
path = Path(path)

def _del_var(pth):
msg = f"Removing {pth} from disk"
logger.warning(msg)
sh.rmtree(pth)

# Update the .zmetadata file
with (Path(path) / ".zmetadata").open("r") as f:
metadata = json.load(f)
[
metadata["metadata"].pop(k)
for k in list(metadata["metadata"].keys())
if k.startswith(f"{pth.name}/.")
]
with (Path(path) / ".zmetadata").open("w") as f:
json.dump(metadata, f, indent=2)

if complete is not None and incomplete is not None:
raise ValueError("Use either `complete` or `incomplete`, not both.")

Expand All @@ -270,20 +287,17 @@ def clean_incomplete(

for fold in filter(lambda p: p.is_dir(), path.iterdir()):
if fold.name not in complete:
msg = f"Removing {fold} from disk"
logger.warning(msg)
sh.rmtree(fold)
_del_var(fold)

elif incomplete is not None:
with xr.open_zarr(path) as ds:
incomplete = [
v for v in incomplete if (v not in ds.coords) and (v not in ds.dims)
]

for fold in filter(lambda p: p.is_dir(), path.iterdir()):
if fold.name in incomplete:
msg = f"Removing {fold} from disk"
logger.warning(msg)
sh.rmtree(fold)

# Remove .zmetadata to avoid issues with zarr and xarray
if (path / ".zmetadata").exists():
Path.unlink(path / ".zmetadata")
_del_var(fold)


def _coerce_attrs(attrs):
Expand Down
89 changes: 18 additions & 71 deletions src/xscen/testing.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Testing utilities for xscen."""

import importlib.metadata
import os
import re
from io import StringIO
Expand Down Expand Up @@ -357,77 +358,23 @@ def show_versions(
-------
str or None
"""
if deps is None:
deps = [
"xscen",
# Main packages
"cartopy",
"cftime",
"cf_xarray",
"clisops",
"dask",
"flox",
"fsspec",
"geopandas",
"h5netcdf",
"h5py",
"intake_esm",
"matplotlib",
"netCDF4",
"numcodecs",
"numpy",
"pandas",
"parse",
"pyyaml",
"rechunker",
"scipy",
"shapely",
"sparse",
"toolz",
"xarray",
"xclim",
"xesmf",
"zarr",
# Opt
"nc-time-axis",
"pyarrow",
# Dev
"babel",
"black",
"blackdoc",
"bump-my-version",
"coverage",
"coveralls",
"flake8",
"flake8-rst-docstrings",
"ipykernel",
"ipython",
"isort",
"jupyter_client",
"nbsphinx",
"nbval",
"pandoc",
"pooch",
"pre-commit",
"pytest",
"pytest-cov",
"ruff",
"setuptools",
"setuptools-scm",
"sphinx",
"sphinx-autoapi",
"sphinx-rtd-theme",
"sphinxcontrib-napoleon",
"sphinx-codeautolink",
"sphinx-copybutton",
"sphinx-mdinclude",
"watchdog",
"xdoctest",
"tox",
"build",
"wheel",
"pip",
"flake8-alphabetize",

def _get_xscen_dependencies():
xscen_metadata = importlib.metadata.metadata("xscen")
requires = xscen_metadata.get_all("Requires-Dist")
requires = [
req.split("[")[0]
.split(";")[0]
.split(">")[0]
.split("<")[0]
.split("=")[0]
.split("!")[0]
for req in requires
]

return ["xscen"] + requires

if deps is None:
deps = _get_xscen_dependencies()

return _show_versions(file=file, deps=deps)
1 change: 0 additions & 1 deletion tests/test_ensembles.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import pytest
import xarray as xr
import xclim as xc
from packaging.version import Version

try:
import xesmf as xe
Expand Down
1 change: 1 addition & 0 deletions tests/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def test_complete(self, tmpdir, which):
xs.io.clean_incomplete(Path(tmpdir) / "test.zarr", incomplete=["pr"])
assert (Path(tmpdir) / "test.zarr/tas").exists()
assert not (Path(tmpdir) / "test.zarr/pr").exists()
assert (Path(tmpdir) / "test.zarr/.zmetadata").exists()

ds2 = xr.open_zarr(Path(tmpdir) / "test.zarr")
assert "pr" not in ds2
Expand Down

0 comments on commit e7dd5c3

Please sign in to comment.