diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea0c78b8..2b4641ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/adrienverge/yamllint - rev: "v1.26.0" + rev: "v1.29.0" hooks: - id: yamllint - repo: https://github.com/asottile/setup-cfg-fmt @@ -23,7 +23,7 @@ repos: hooks: - id: black-jupyter - repo: https://github.com/PyCQA/isort - rev: "5.9.3" + rev: 5.12.0 hooks: - id: isort # TODO renable when errors are fixed/ignored @@ -78,7 +78,7 @@ repos: rev: 1.1.0 hooks: - id: nbqa-isort - additional_dependencies: [isort==5.9.3] + additional_dependencies: [isort==5.11.2] - id: nbqa-mypy additional_dependencies: [mypy==0.910, types-python-dateutil] # TODO renable when errors are fixed/ignored diff --git a/docs/conf.py b/docs/conf.py index 247fa281..0cc75beb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -191,7 +191,7 @@ def setup(app): # noqa: D103 ] autodoc_mock_imports = [ - "basic_modeling_interface", + "bmipy", "cftime", "dask", "esmvalcore", diff --git a/setup.cfg b/setup.cfg index b8e8e3e7..458afab6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,10 +37,10 @@ packages = find: install_requires = Fiona Shapely - basic_modeling_interface + bmipy cftime esmvaltool>=2.4.0 - grpc4bmi>=0.2.12,<0.3 + grpc4bmi@git+https://github.com/eWaterCycle/grpc4bmi@bmi2 grpcio hydrostats matplotlib>=3.5.0 diff --git a/src/ewatercycle/config/_lisflood_versions.py b/src/ewatercycle/config/_lisflood_versions.py index f050f6b6..f80bf779 100644 --- a/src/ewatercycle/config/_lisflood_versions.py +++ b/src/ewatercycle/config/_lisflood_versions.py @@ -1,19 +1,10 @@ -""" -Versions of Lisflood container images -""" -from pathlib import Path +"""Versions of Lisflood container images.""" -version_images = { +from ewatercycle.container import VersionImages + +version_images: VersionImages = { "20.10": { "docker": "ewatercycle/lisflood-grpc4bmi:20.10", "singularity": "ewatercycle-lisflood-grpc4bmi_20.10.sif", } } - - -def get_docker_image(version): - return version_images[version]["docker"] - - -def get_singularity_image(version, singularity_dir: Path): - return singularity_dir / version_images[version]["singularity"] diff --git a/src/ewatercycle/container.py b/src/ewatercycle/container.py new file mode 100644 index 00000000..7033aa32 --- /dev/null +++ b/src/ewatercycle/container.py @@ -0,0 +1,97 @@ +"""Container utilities.""" +from pathlib import Path +from typing import Dict, Iterable, Literal, Mapping, Optional, Union + +from bmipy import Bmi +from grpc import FutureTimeoutError +from grpc4bmi.bmi_client_docker import BmiClientDocker +from grpc4bmi.bmi_client_singularity import BmiClientSingularity +from grpc4bmi.bmi_memoized import MemoizedBmi +from grpc4bmi.bmi_optionaldest import OptionalDestBmi + +from ewatercycle import CFG + +ContainerEngines = Literal["docker", "singularity"] +"""Supported container engines.""" + +ImageForContainerEngines = Dict[ContainerEngines, str] +"""Container image name for each container engine.""" + +VersionImages = Mapping[str, ImageForContainerEngines] +"""Dictionary of versions of a model. + +Each version has the image name for each container engine. +""" + + +def start_container( + work_dir: Union[str, Path], + image_engine: ImageForContainerEngines, + input_dirs: Optional[Iterable[str]] = None, + image_port=55555, + timeout=None, + delay=0, +) -> Bmi: + """Start container with model inside. + + The `ewatercycle.CFG['container_engine']` value determines + the engine used to start a container. + + Args: + work_dir: Work directory + version_image: Image name for each container engine. + input_dirs: Additional directories to mount inside container. + image_port: Docker port inside container where grpc4bmi server is running. + timeout: Number of seconds to wait for grpc connection. + delay: Number of seconds to wait before connecting. + + Raises: + ValueError: When unknown container technology is requested. + TimeoutError: When model inside container did not start quickly enough. + + Returns: + _description_ + """ + engine: ContainerEngines = CFG["container_engine"] + image = image_engine[engine] + if input_dirs is None: + input_dirs = [] + if engine == "docker": + try: + bmi = BmiClientDocker( + image=image, + image_port=image_port, + work_dir=str(work_dir), + input_dirs=input_dirs, + timeout=timeout, + delay=delay, + ) + except FutureTimeoutError as exc: + # https://github.com/eWaterCycle/grpc4bmi/issues/95 + # https://github.com/eWaterCycle/grpc4bmi/issues/100 + raise TimeoutError( + "Couldn't spawn container within allocated time limit " + f"({timeout} seconds). You may try pulling the docker image with" + f" `docker pull {image}` and then try again." + ) from exc + elif engine == "singularity": + image = str(CFG["singularity_dir"] / image) + try: + bmi = BmiClientSingularity( + image=image, + work_dir=str(work_dir), + input_dirs=input_dirs, + timeout=timeout, + delay=delay, + ) + except FutureTimeoutError as exc: + docker_image = image_engine["docker"] + raise TimeoutError( + "Couldn't spawn container within allocated time limit " + f"({timeout} seconds). You may try pulling the docker image with" + f" `singularity build {image} " + f"docker://{docker_image}` and then try again." + ) from exc + else: + raise ValueError(f"Unknown container technology: {CFG['container_engine']}") + return OptionalDestBmi(MemoizedBmi(bmi)) diff --git a/src/ewatercycle/forcing/_lisvap.py b/src/ewatercycle/forcing/_lisvap.py index a761a4a2..5af60ed2 100644 --- a/src/ewatercycle/forcing/_lisvap.py +++ b/src/ewatercycle/forcing/_lisvap.py @@ -26,9 +26,10 @@ from typing import Dict, Tuple from ewatercycle import CFG +from ewatercycle.container import ContainerEngines from ewatercycle.parametersetdb.config import XmlConfig -from ..config._lisflood_versions import get_docker_image, get_singularity_image +from ..config._lisflood_versions import version_images from ..util import get_time @@ -49,9 +50,10 @@ def lisvap( mask_map, forcing_dir, ) - + engine: ContainerEngines = CFG["container_engine"] + image = version_images[version][engine] if CFG["container_engine"].lower() == "singularity": - image = get_singularity_image(version, CFG["singularity_dir"]) + image = CFG["singularity_dir"] / image args = [ "singularity", "exec", @@ -62,7 +64,6 @@ def lisvap( image, ] elif CFG["container_engine"].lower() == "docker": - image = get_docker_image(version) args = [ "docker", "run", diff --git a/src/ewatercycle/models/abstract.py b/src/ewatercycle/models/abstract.py index 74904dc2..81c0221d 100644 --- a/src/ewatercycle/models/abstract.py +++ b/src/ewatercycle/models/abstract.py @@ -1,12 +1,13 @@ +"""Abstract class of a eWaterCycle model.""" import logging import textwrap from abc import ABCMeta, abstractmethod from datetime import datetime -from typing import Any, ClassVar, Generic, Iterable, Optional, Set, Tuple, TypeVar +from typing import Any, ClassVar, Generic, Iterable, Optional, Tuple, TypeVar import numpy as np import xarray as xr -from basic_modeling_interface import Bmi +from bmipy import Bmi from cftime import num2date from ewatercycle.forcing import DefaultForcing @@ -169,7 +170,7 @@ def get_value_as_xarray(self, name: str) -> xr.DataArray: @property @abstractmethod def parameters(self) -> Iterable[Tuple[str, Any]]: - """Default values for the setup() inputs""" + """Default values for the setup() inputs.""" @property def start_time(self) -> float: diff --git a/src/ewatercycle/models/hype.py b/src/ewatercycle/models/hype.py index e7ac7ca6..69d84d2d 100644 --- a/src/ewatercycle/models/hype.py +++ b/src/ewatercycle/models/hype.py @@ -1,19 +1,16 @@ +"""eWaterCycle wrapper around Hype BMI.""" import datetime import logging import shutil import types from typing import Any, Iterable, Optional, Tuple -import numpy as np import xarray as xr -from basic_modeling_interface import Bmi -from cftime import num2date from dateutil.parser import parse from dateutil.tz import UTC -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._hype import HypeForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -21,7 +18,7 @@ logger = logging.getLogger(__name__) -_version_images = { +_version_images: VersionImages = { "feb2021": { "docker": "ewatercycle/hype-grpc4bmi:feb2021", "singularity": "ewatercycle-hype-grpc4bmi_feb2021.sif", @@ -44,6 +41,7 @@ class Hype(AbstractModel[HypeForcing]): """ available_versions = tuple(_version_images.keys()) + """Show supported Hype versions in eWaterCycle""" def __init__( self, @@ -52,7 +50,6 @@ def __init__( forcing: Optional[HypeForcing] = None, ): super().__init__(version, parameter_set, forcing) - assert version in _version_images self._setup_default_config() def _setup_default_config(self): @@ -155,8 +152,10 @@ def setup( # type: ignore cfg_file.write_text(self._cfg, encoding="cp437") # start container - work_dir = str(cfg_dir_as_path) - self.bmi = _start_container(self.version, work_dir) + self.bmi = start_container( + image_engine=_version_images[self.version], + work_dir=cfg_dir_as_path, + ) since = self._start.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -167,7 +166,7 @@ def get_time_units(_self): self.bmi.get_time_units = types.MethodType(get_time_units, self.bmi) - return str(cfg_file), work_dir + return str(cfg_file), str(cfg_dir_as_path) @property def parameters(self) -> Iterable[Tuple[str, Any]]: @@ -220,26 +219,6 @@ def _setup_cfg_dir(cfg_dir: Optional[str] = None): return work_dir -def _start_container(version: str, work_dir: str): - if CFG["container_engine"].lower() == "singularity": - image = CFG["singularity_dir"] / _version_images[version]["singularity"] - return BmiClientSingularity( - image=str(image), - work_dir=work_dir, - ) - elif CFG["container_engine"].lower() == "docker": - image = _version_images[version]["docker"] - return BmiClientDocker( - image=image, - image_port=55555, # TODO needed? - work_dir=work_dir, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) - - def _get_code_in_cfg(content: str, code: str): lines = content.splitlines() for line in lines: @@ -265,5 +244,5 @@ def _set_code_in_cfg(content: str, code: str, value: str) -> str: def _get_hype_time(value: str) -> datetime.datetime: - """Converts `yyyy-mm-dd [HH:MM]` string to datetime object""" + """Converts `yyyy-mm-dd [HH:MM]` string to datetime object.""" return parse(value).replace(tzinfo=UTC) diff --git a/src/ewatercycle/models/lisflood.py b/src/ewatercycle/models/lisflood.py index 0a5a7d3a..68f004d3 100644 --- a/src/ewatercycle/models/lisflood.py +++ b/src/ewatercycle/models/lisflood.py @@ -8,15 +8,10 @@ import numpy as np import xarray as xr from cftime import num2date -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG -from ewatercycle.config._lisflood_versions import ( - get_docker_image, - get_singularity_image, - version_images, -) +from ewatercycle.config._lisflood_versions import version_images +from ewatercycle.container import start_container from ewatercycle.forcing._lisflood import LisfloodForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -115,27 +110,13 @@ def setup( # type: ignore # If not relative add dir input_dirs.append(str(mask_map.parent)) - if CFG["container_engine"].lower() == "singularity": - image = get_singularity_image(self.version, CFG["singularity_dir"]) - self.bmi = BmiClientSingularity( - image=str(image), - input_dirs=input_dirs, - work_dir=str(cfg_dir_as_path), - timeout=300, - ) - elif CFG["container_engine"].lower() == "docker": - image = get_docker_image(self.version) - self.bmi = BmiClientDocker( - image=image, - image_port=55555, - input_dirs=input_dirs, - work_dir=str(cfg_dir_as_path), - timeout=300, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) + self.bmi = start_container( + image_engine=version_images[self.version], + work_dir=cfg_dir_as_path, + input_dirs=input_dirs, + timeout=300, + ) + return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): diff --git a/src/ewatercycle/models/marrmot.py b/src/ewatercycle/models/marrmot.py index 57ea84bd..fd15c0d2 100644 --- a/src/ewatercycle/models/marrmot.py +++ b/src/ewatercycle/models/marrmot.py @@ -10,10 +10,9 @@ import scipy.io as sio import xarray as xr from cftime import num2date -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._marrmot import MarrmotForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.util import get_time, to_absolute_path @@ -52,6 +51,14 @@ def _generate_cfg_dir(cfg_dir: Optional[Path] = None) -> Path: return cfg_dir +_version_images: VersionImages = { + "2020.11": { + "docker": "ewatercycle/marrmot-grpc4bmi:2020.11", + "singularity": "ewatercycle-marrmot-grpc4bmi_2020.11.sif", + } +} + + class MarrmotM01(AbstractModel[MarrmotForcing]): """eWaterCycle implementation of Marrmot Collie River 1 (traditional bucket) model. @@ -72,7 +79,7 @@ class MarrmotM01(AbstractModel[MarrmotForcing]): model_name = "m_01_collie1_1p_1s" """Name of model in Matlab code.""" - available_versions = ("2020.11",) + available_versions = tuple(_version_images.keys()) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 @@ -82,18 +89,6 @@ def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 self.solver = Solver() self._check_forcing(forcing) - self._set_singularity_image() - self._set_docker_image() - - def _set_docker_image(self): - images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} - self.docker_image = images[self.version] - - def _set_singularity_image(self): - images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} - if CFG.get("singularity_dir"): - self.singularity_image = CFG["singularity_dir"] / images[self.version] - # unable to subclass with more specialized arguments so ignore type def setup( # type: ignore self, @@ -143,27 +138,13 @@ def setup( # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG["container_engine"].lower() == "singularity": - message = f"The singularity image {self.singularity_image} does not exist." - assert self.singularity_image.exists(), message - self.bmi = BmiClientSingularity( - image=str(self.singularity_image), - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - elif CFG["container_engine"].lower() == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) + self.bmi = start_container( + image_engine=_version_images[self.version], + work_dir=cfg_dir_as_path, + timeout=300, + delay=delay, + ) + return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): @@ -328,7 +309,7 @@ class MarrmotM14(AbstractModel[MarrmotForcing]): model_name = "m_14_topmodel_7p_2s" """Name of model in Matlab code.""" - available_versions = ("2020.11",) + available_versions = tuple(_version_images.keys()) """Versions for which ewatercycle grpc4bmi docker images are available.""" def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 @@ -338,18 +319,6 @@ def __init__(self, version: str, forcing: MarrmotForcing): # noqa: D107 self.solver = Solver() self._check_forcing(forcing) - self._set_singularity_image() - self._set_docker_image() - - def _set_docker_image(self): - images = {"2020.11": "ewatercycle/marrmot-grpc4bmi:2020.11"} - self.docker_image = images[self.version] - - def _set_singularity_image(self): - images = {"2020.11": "ewatercycle-marrmot-grpc4bmi_2020.11.sif"} - if CFG.get("singularity_dir"): - self.singularity_image = CFG["singularity_dir"] / images[self.version] - # unable to subclass with more specialized arguments so ignore type def setup( # type: ignore self, @@ -418,27 +387,13 @@ def setup( # type: ignore cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path) config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time) - if CFG["container_engine"].lower() == "singularity": - message = f"The singularity image {self.singularity_image} does not exist." - assert self.singularity_image.exists(), message - self.bmi = BmiClientSingularity( - image=str(self.singularity_image), - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - elif CFG["container_engine"].lower() == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(cfg_dir_as_path), - timeout=300, - delay=delay, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) + self.bmi = start_container( + image_engine=_version_images[self.version], + work_dir=cfg_dir_as_path, + timeout=300, + delay=delay, + ) + return str(config_file), str(cfg_dir_as_path) def _check_forcing(self, forcing): diff --git a/src/ewatercycle/models/pcrglobwb.py b/src/ewatercycle/models/pcrglobwb.py index 72265a72..65c4bc20 100644 --- a/src/ewatercycle/models/pcrglobwb.py +++ b/src/ewatercycle/models/pcrglobwb.py @@ -8,11 +8,9 @@ import numpy as np import xarray as xr from cftime import num2date -from grpc import FutureTimeoutError -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._pcrglobwb import PCRGlobWBForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -21,6 +19,13 @@ logger = logging.getLogger(__name__) +_version_images: VersionImages = { + "setters": { + "docker": "ewatercycle/pcrg-grpc4bmi:setters", + "singularity": "ewatercycle-pcrg-grpc4bmi_setters.sif", + } +} + class PCRGlobWB(AbstractModel[PCRGlobWBForcing]): """eWaterCycle implementation of PCRGlobWB hydrological model. @@ -34,7 +39,7 @@ class PCRGlobWB(AbstractModel[PCRGlobWBForcing]): """ - available_versions = ("setters",) + available_versions = tuple(_version_images.keys()) def __init__( # noqa: D107 self, @@ -43,22 +48,8 @@ def __init__( # noqa: D107 forcing: Optional[PCRGlobWBForcing] = None, ): super().__init__(version, parameter_set, forcing) - self._set_docker_image() self._setup_default_config() - def _set_docker_image(self): - images = { - "setters": "ewatercycle/pcrg-grpc4bmi:setters", - } - self.docker_image = images[self.version] - - def _singularity_image(self, singularity_dir): - images = { - "setters": "ewatercycle-pcrg-grpc4bmi_setters.sif", - } - image = singularity_dir / images[self.version] - return str(image) - def _setup_work_dir(self, cfg_dir: Optional[str] = None): if cfg_dir: self.work_dir = to_absolute_path(cfg_dir) @@ -130,19 +121,17 @@ def setup(self, cfg_dir: Optional[str] = None, **kwargs) -> Tuple[str, str]: # cfg_file = self._export_config() work_dir = self.work_dir - try: - self._start_container() - except FutureTimeoutError as exc: - # https://github.com/eWaterCycle/grpc4bmi/issues/95 - # https://github.com/eWaterCycle/grpc4bmi/issues/100 - raise ValueError( - "Couldn't spawn container within allocated time limit " - "(300 seconds). You may try pulling the docker image with" - f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} " - f"docker://{self.docker_image}` if you're using singularity," - " and then try again." - ) from exc + additional_input_dirs = [] + if self.parameter_set: + additional_input_dirs.append(str(self.parameter_set.directory)) + if self.forcing: + additional_input_dirs.append(str(self.forcing.directory)) + self.bmi = start_container( + image_engine=_version_images[self.version], + work_dir=self.work_dir, + input_dirs=additional_input_dirs, + timeout=300, + ) return str(cfg_file), str(work_dir) @@ -191,31 +180,6 @@ def _export_config(self) -> PathLike: self.cfg_file = new_cfg_file return self.cfg_file - def _start_container(self): - additional_input_dirs = [str(self.parameter_set.directory)] - if self.forcing: - additional_input_dirs.append(self.forcing.directory) - - if CFG["container_engine"] == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(self.work_dir), - input_dirs=additional_input_dirs, - timeout=300, - ) - elif CFG["container_engine"] == "singularity": - self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG["singularity_dir"]), - work_dir=str(self.work_dir), - input_dirs=additional_input_dirs, - timeout=300, - ) - else: - raise ValueError( - f"Unknown container technology in CFG: {CFG['container_engine']}" - ) - def _coords_to_indices( self, name: str, lat: Iterable[float], lon: Iterable[float] ) -> Iterable[int]: diff --git a/src/ewatercycle/models/wflow.py b/src/ewatercycle/models/wflow.py index dd4998af..44fde922 100644 --- a/src/ewatercycle/models/wflow.py +++ b/src/ewatercycle/models/wflow.py @@ -9,11 +9,9 @@ import numpy as np import xarray as xr from cftime import num2date -from grpc import FutureTimeoutError -from grpc4bmi.bmi_client_docker import BmiClientDocker -from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG +from ewatercycle.container import VersionImages, start_container from ewatercycle.forcing._wflow import WflowForcing from ewatercycle.models.abstract import AbstractModel from ewatercycle.parameter_sets import ParameterSet @@ -22,6 +20,25 @@ logger = logging.getLogger(__name__) +_version_images: VersionImages = { + # "2019.1": { + # "docker":"ewatercycle/wflow-grpc4bmi:2019.1", + # "singularity": "ewatercycle-wflow-grpc4bmi_2019.1.sif", + # }, # no good ini file + "2020.1.1": { + "docker": "ewatercycle/wflow-grpc4bmi:2020.1.1", + "singularity": "ewatercycle-wflow-grpc4bmi_2020.1.1.sif", + }, + "2020.1.2": { + "docker": "ewatercycle/wflow-grpc4bmi:2020.1.2", + "singularity": "ewatercycle-wflow-grpc4bmi_2020.1.2.sif", + }, + "2020.1.3": { + "docker": "ewatercycle/wflow-grpc4bmi:2020.1.3", + "singularity": "ewatercycle-wflow-grpc4bmi_2020.1.3.sif", + }, +} + class Wflow(AbstractModel[WflowForcing]): """Create an instance of the Wflow model class. @@ -34,7 +51,7 @@ class Wflow(AbstractModel[WflowForcing]): If None, it is assumed that forcing is included with the parameter_set. """ - available_versions = ("2020.1.1", "2020.1.2", "2020.1.3") + available_versions = tuple(_version_images.keys()) """Show supported WFlow versions in eWaterCycle""" def __init__( # noqa: D107 @@ -44,27 +61,8 @@ def __init__( # noqa: D107 forcing: Optional[WflowForcing] = None, ): super().__init__(version, parameter_set, forcing) - self._set_docker_image() self._setup_default_config() - def _set_docker_image(self): - images = { - # "2019.1": "ewatercycle/wflow-grpc4bmi:2019.1", # no good ini file - "2020.1.1": "ewatercycle/wflow-grpc4bmi:2020.1.1", - "2020.1.2": "ewatercycle/wflow-grpc4bmi:2020.1.2", - "2020.1.3": "ewatercycle/wflow-grpc4bmi:2020.1.3", - } - self.docker_image = images[self.version] - - def _singularity_image(self, singularity_dir): - images = { - "2020.1.1": "ewatercycle-wflow-grpc4bmi_2020.1.1.sif", - "2020.1.2": "ewatercycle-wflow-grpc4bmi_2020.1.2.sif", - "2020.1.3": "ewatercycle-wflow-grpc4bmi_2020.1.3.sif", - } - image = singularity_dir / images[self.version] - return str(image) - def _setup_default_config(self): config_file = self.parameter_set.config forcing = self.forcing @@ -124,19 +122,11 @@ def setup(self, cfg_dir: Optional[str] = None, **kwargs) -> Tuple[str, str]: # with updated_cfg_file.open("w") as filename: cfg.write(filename) - try: - self._start_container() - except FutureTimeoutError as exc: - # https://github.com/eWaterCycle/grpc4bmi/issues/95 - # https://github.com/eWaterCycle/grpc4bmi/issues/100 - raise ValueError( - "Couldn't spawn container within allocated time limit " - "(300 seconds). You may try pulling the docker image with" - f" `docker pull {self.docker_image}` or call `singularity " - f"build {self._singularity_image(CFG['singularity_dir'])} " - f"docker://{self.docker_image}` if you're using singularity," - " and then try again." - ) from exc + self.bmi = start_container( + image_engine=_version_images[self.version], + work_dir=self.work_dir, + timeout=300, + ) return ( str(updated_cfg_file), @@ -164,23 +154,6 @@ def _setup_working_directory(self, cfg_dir: Optional[str] = None): ) shutil.copy(src=forcing_path, dst=self.work_dir) - def _start_container(self): - if CFG["container_engine"] == "docker": - self.bmi = BmiClientDocker( - image=self.docker_image, - image_port=55555, - work_dir=str(self.work_dir), - timeout=300, - ) - elif CFG["container_engine"] == "singularity": - self.bmi = BmiClientSingularity( - image=self._singularity_image(CFG["singularity_dir"]), - work_dir=str(self.work_dir), - timeout=300, - ) - else: - raise ValueError(f"Unknown container technology: {CFG['container_engine']}") - def _coords_to_indices( self, name: str, lat: Iterable[float], lon: Iterable[float] ) -> Iterable[int]: diff --git a/tests/models/fake_models.py b/tests/models/fake_models.py new file mode 100644 index 00000000..002e223f --- /dev/null +++ b/tests/models/fake_models.py @@ -0,0 +1,137 @@ +"""Fake BMI models.""" +import numpy as np +from bmipy import Bmi + + +class SomeException(Exception): + pass + + +class FailingModel(Bmi): + def __init__(self, exc=SomeException()): + self.exc = exc + + def initialize(self, filename): + raise self.exc + + def update(self): + raise self.exc + + def update_until(self, time: float) -> None: + raise self.exc + + def finalize(self): + raise self.exc + + def get_component_name(self): + raise self.exc + + def get_input_item_count(self) -> int: + raise self.exc + + def get_output_item_count(self) -> int: + raise self.exc + + def get_input_var_names(self): + raise self.exc + + def get_output_var_names(self): + raise self.exc + + def get_start_time(self): + raise self.exc + + def get_current_time(self): + raise self.exc + + def get_end_time(self): + raise self.exc + + def get_time_step(self): + raise self.exc + + def get_time_units(self): + raise self.exc + + def get_var_type(self, name): + raise self.exc + + def get_var_units(self, name): + raise self.exc + + def get_var_itemsize(self, name): + raise self.exc + + def get_var_nbytes(self, name): + raise self.exc + + def get_var_grid(self, name): + raise self.exc + + def get_value(self, name, dest): + raise self.exc + + def get_value_ptr(self, name): + raise self.exc + + def get_value_at_indices(self, name, dest, inds): + raise self.exc + + def set_value(self, name, src): + raise self.exc + + def set_value_at_indices(self, name, inds, src): + raise self.exc + + def get_grid_shape(self, grid, shape): + raise self.exc + + def get_grid_x(self, grid, x): + raise self.exc + + def get_grid_y(self, grid, y): + raise self.exc + + def get_grid_z(self, grid, z): + raise self.exc + + def get_grid_spacing(self, grid, spacing): + raise self.exc + + def get_grid_origin(self, grid, origin): + raise self.exc + + def get_grid_rank(self, grid): + raise self.exc + + def get_grid_size(self, grid): + raise self.exc + + def get_grid_type(self, grid): + raise self.exc + + def get_var_location(self, name: str) -> str: + raise self.exc + + def get_grid_node_count(self, grid: int) -> int: + raise self.exc + + def get_grid_edge_count(self, grid: int) -> int: + raise self.exc + + def get_grid_face_count(self, grid: int) -> int: + raise self.exc + + def get_grid_edge_nodes(self, grid: int, edge_nodes: np.ndarray) -> np.ndarray: + raise self.exc + + def get_grid_face_nodes(self, grid: int, face_nodes: np.ndarray) -> np.ndarray: + raise self.exc + + def get_grid_nodes_per_face( + self, grid: int, nodes_per_face: np.ndarray + ) -> np.ndarray: + raise self.exc + + def get_grid_face_edges(self, grid: int, face_edges: np.ndarray) -> np.ndarray: + raise self.exc diff --git a/tests/models/test_abstract.py b/tests/models/test_abstract.py index 730c1041..1a074e17 100644 --- a/tests/models/test_abstract.py +++ b/tests/models/test_abstract.py @@ -7,7 +7,7 @@ import numpy as np import pytest import xarray as xr -from basic_modeling_interface import Bmi +from bmipy import Bmi from numpy.testing import assert_array_equal from ewatercycle import CFG @@ -66,7 +66,7 @@ def parameters(self) -> Iterable[Tuple[str, Any]]: @pytest.fixture -@patch("basic_modeling_interface.Bmi") +@patch("bmipy.Bmi") def bmi(MockedBmi): mocked_bmi = MockedBmi() mocked_bmi.get_start_time.return_value = 42.0 diff --git a/tests/models/test_hype.py b/tests/models/test_hype.py index 2fba2442..8db9663b 100644 --- a/tests/models/test_hype.py +++ b/tests/models/test_hype.py @@ -5,13 +5,14 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi +from bmipy import Bmi from grpc4bmi.bmi_client_singularity import BmiClientSingularity from ewatercycle import CFG from ewatercycle.forcing import load_foreign from ewatercycle.models.hype import Hype, _set_code_in_cfg from ewatercycle.parameter_sets import ParameterSet +from tests.models.fake_models import FailingModel @pytest.fixture @@ -93,6 +94,9 @@ def test_setup_container(self, model_with_setup, tmp_path): mocked_constructor.assert_called_once_with( image=f"{tmp_path}/ewatercycle-hype-grpc4bmi_feb2021.sif", work_dir=f"{tmp_path}/hype_20210102_030405", + input_dirs=[], + timeout=None, + delay=0, ) def test_setup_parameter_set_files(self, model_with_setup): @@ -141,7 +145,7 @@ def test_get_value_as_xarray(self, model): model.get_value_as_xarray("comp outflow olake") def test_get_value_at_coords(self, model): - class MockedBmi(Bmi): + class MockedBmi(FailingModel): """Pretend to be a real BMI model.""" def get_var_grid(self, name): @@ -192,6 +196,9 @@ def test_setup_container(self, model_with_setup, tmp_path): mocked_constructor.assert_called_once_with( image=f"{tmp_path}/ewatercycle-hype-grpc4bmi_feb2021.sif", work_dir=f"{tmp_path}/myworkdir", + input_dirs=[], + timeout=None, + delay=0, ) def test_setup_parameter_set_files(self, model_with_setup): @@ -329,6 +336,9 @@ def test_setup_container(self, model_with_setup, tmp_path): mocked_constructor.assert_called_once_with( image=f"{tmp_path}/ewatercycle-hype-grpc4bmi_feb2021.sif", work_dir=f"{tmp_path}/hype_20210102_030405", + input_dirs=[], + timeout=None, + delay=0, ) def test_setup_forcing_files(self, model_with_setup): diff --git a/tests/models/test_lisflood.py b/tests/models/test_lisflood.py index cf7f7046..1b32637d 100644 --- a/tests/models/test_lisflood.py +++ b/tests/models/test_lisflood.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi +from bmipy import Bmi from grpc4bmi.bmi_client_singularity import BmiClientSingularity from numpy.testing import assert_array_equal @@ -15,6 +15,7 @@ from ewatercycle.models.lisflood import Lisflood from ewatercycle.parameter_sets import ParameterSet, example_parameter_sets from ewatercycle.parametersetdb.config import XmlConfig +from tests.models.fake_models import FailingModel @pytest.fixture @@ -110,6 +111,7 @@ def test_setup(self, model_with_setup, tmp_path): ], work_dir=f"{tmp_path}/lisflood_20210102_030405", timeout=300, + delay=0, ) # Check content config file @@ -210,6 +212,7 @@ def test_setup(self, model_with_setup, tmp_path): ], work_dir=f"{tmp_path}/lisflood_20210102_030405", timeout=300, + delay=0, ) # Check content config file @@ -248,7 +251,7 @@ def test_parameters_after_setup( assert model.parameters == expected_parameters -class MockedBmi(Bmi): +class MockedBmi(FailingModel): """Mimic a real use case with realistic shape and abitrary high precision.""" def get_var_grid(self, name): diff --git a/tests/models/test_pcrglobwb.py b/tests/models/test_pcrglobwb.py index 09383dc1..5e633915 100644 --- a/tests/models/test_pcrglobwb.py +++ b/tests/models/test_pcrglobwb.py @@ -5,7 +5,6 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi from grpc import FutureTimeoutError from grpc4bmi.bmi_client_singularity import BmiClientSingularity @@ -13,9 +12,10 @@ from ewatercycle.forcing import load_foreign from ewatercycle.models import PCRGlobWB from ewatercycle.parameter_sets import ParameterSet, example_parameter_sets +from tests.models.fake_models import FailingModel -class MockedBmi(Bmi): +class MockedBmi(FailingModel): """Pretend to be a real BMI model.""" def initialize(self, config_file): @@ -102,13 +102,12 @@ def test_setup_withtimeoutexception(model, tmp_path): with patch.object( BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() ), patch("datetime.datetime") as mocked_datetime, pytest.raises( - ValueError + TimeoutError ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert "docker pull ewatercycle/pcrg-grpc4bmi:setters" in msg sif = tmp_path / "ewatercycle-pcrg-grpc4bmi_setters.sif" assert f"build {sif} docker://ewatercycle/pcrg-grpc4bmi:setters" in msg diff --git a/tests/models/test_wflow.py b/tests/models/test_wflow.py index af4c35ee..f766aea8 100644 --- a/tests/models/test_wflow.py +++ b/tests/models/test_wflow.py @@ -6,7 +6,6 @@ import numpy as np import pytest -from basic_modeling_interface import Bmi from grpc import FutureTimeoutError from grpc4bmi.bmi_client_singularity import BmiClientSingularity @@ -14,11 +13,15 @@ from ewatercycle.models import Wflow from ewatercycle.parameter_sets import ParameterSet from ewatercycle.parametersetdb.config import CaseConfigParser +from tests.models.fake_models import FailingModel -class MockedBmi(Bmi): +class MockedBmi(FailingModel): """Pretend to be a real BMI model.""" + def get_component_name(self) -> str: + return "mocked" + def initialize(self, config_file): pass @@ -156,13 +159,12 @@ def test_setup_withtimeoutexception(model, tmp_path): with patch.object( BmiClientSingularity, "__init__", side_effect=FutureTimeoutError() ), patch("datetime.datetime") as mocked_datetime, pytest.raises( - ValueError + TimeoutError ) as excinfo: mocked_datetime.now.return_value = datetime(2021, 1, 2, 3, 4, 5) model.setup() msg = str(excinfo.value) - assert "docker pull ewatercycle/wflow-grpc4bmi:2020.1.1" in msg sif = tmp_path / "ewatercycle-wflow-grpc4bmi_2020.1.1.sif" assert f"build {sif} docker://ewatercycle/wflow-grpc4bmi:2020.1.1" in msg