From cbc7e7eb521ab870ee5d4bcdbd7932fbe86fe078 Mon Sep 17 00:00:00 2001 From: Ben Avrahami Date: Sun, 10 Dec 2023 17:47:22 +0200 Subject: [PATCH] changes for 1.1.0 (#8) * changes for 1.1.0 * added lookupparser * lint --------- Co-authored-by: Ben Avrahami --- .gitignore | 6 ++ CHANGELOG.md | 11 +++ docs/basevar.rst | 27 +++++++ docs/string_parsing.rst | 42 ++++++++++- envolved/__init__.py | 3 +- envolved/_version.py | 2 +- envolved/basevar.py | 28 +++++-- envolved/describe.py | 8 +- envolved/parsers.py | 128 +++++++++++++++++++++++++------- envolved/py.typed | 0 pyproject.toml | 8 +- tests/unittests/test_parsers.py | 95 +++++++++++++++++++++++- tests/unittests/test_schema.py | 51 +++++++++++++ 13 files changed, 363 insertions(+), 46 deletions(-) create mode 100644 .gitignore create mode 100644 envolved/py.typed diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb0d180 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +docs/_build +**/__pycache__ +.vscode/settings.json +poetry.lock +coverage.xml +.coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index bd993a1..5e2e419 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,15 @@ # envolved Changelog +## 1.1.0 +### Added +* Single env vars can now accept pydantic models and type adapters, they will be parsed as jsons. +* added `py.typed` file to the package. +* added `inferred_env_var` to the root `envolved` namespace. +* schema env vars can now have keyword arguments passed to their `get` method, to add values to the schema. +* new parse: `LookupParser`, as a faster alternative to `MatchParser` (that does not support regex matches). +### Changed +* the special parser of `Enum`s is now `LookupParser` instead of `MatchParser`. +### Fixed +* `exclude_from_description` now ignores inferred env vars. ## 1.0.0 ### Added * `inferred_env_var` to explicitly infer the type, name and default value of an env var. diff --git a/docs/basevar.rst b/docs/basevar.rst index f59294e..c4bc5a9 100644 --- a/docs/basevar.rst +++ b/docs/basevar.rst @@ -140,6 +140,9 @@ EnvVar Classes When the value is retrieved, all its :attr:`args` and :attr:`pos_args` are retrieved, and are then used as keyword variables on the EnvVar's :attr:`type`. + Users can also supply keyword arguments to the :meth:`get` method, which will be supplied to the :attr:`type` in addition/instead of + the child EnvVars. + .. property:: type :type: collections.abc.Callable[..., T] @@ -171,5 +174,29 @@ EnvVar Classes :attr:`~EnvVar.default` is set. * If set to a value, that value will be returned. + .. method:: get(**kwargs)->T + + Return the value of the environment variable. The value will be created by calling the :attr:`type` callable + with the values of all the child EnvVars as keyword arguments, and the values of the ``kwargs`` parameter as + additional keyword arguments. + + :param kwargs: Additional keyword arguments to pass to the :attr:`type` callable. + :return: The value of the environment variable. + + .. code-block:: + :caption: Using SchemaEnvVar to create a class from multiple environment variables, with additional keyword arguments. + + from dataclasses import dataclass + + @dataclass + class User: + name: str + age: int + height: int + user_ev = env_var("USER_", type=User, + args={'name': env_var('NAME', type=str), + 'age': env_var('AGE', type=int)}) + user_ev.get(age=20, height=168) # will return a User object with the name taken from the environment variables, + # but with the age and height overridden by the keyword arguments. diff --git a/docs/string_parsing.rst b/docs/string_parsing.rst index d82f7f5..1a3f37f 100644 --- a/docs/string_parsing.rst +++ b/docs/string_parsing.rst @@ -44,7 +44,11 @@ All the special parsers are: * union type ``A | None`` or ``typing.Union[A, None]`` or ``typing.Optional[A]``: Will treat the parser as though it only parses ``A``. * enum type ``E``: translates each enum name to the corresponding enum member, disregarding cases (equivalent to - ``MatchParser.case_insensitive(E)`` see :class:`~parsers.MatchParser`). + ``LookupParser.case_insensitive(E)`` see :class:`~parsers.LookupParser`). +* pydantic ``BaseModel``: parses the string as JSON and validates it against the model (both pydnatic v1 and v2 + models are supported). +* pydantic ``TypeAdapter``: parses the string as JSON and validates it against the adapted type. + Utility Parsers --------------- @@ -138,7 +142,7 @@ Utility Parsers A parser that checks a string against a se of cases, returning the value of first case that matches. - :param cases: An iterable of match-value pairs. The match an be a string or a regex pattern (which will need to + :param cases: An iterable of match-value pairs. The match can be a string or a regex pattern (which will need to fully match the string). The case list can also be a mapping of strings to values, or an enum type, in which case the names of the enum members will be used as the matches. :param fallback: The value to return if no case matches. If not specified, an exception will be raised. @@ -163,4 +167,36 @@ Utility Parsers case-insensitivity, an error will be raised. :param cases: Acts the same as in the :class:`constructor `. Regex patterns are not supported. - :param fallback: Acts the same as in the :class:`constructor `. \ No newline at end of file + :param fallback: Acts the same as in the :class:`constructor `. + +.. class:: LookupParser(lookup: collection.abc.Iterable[tuple[str, T]] | \ + collections.abc.Mapping[str, T] | type[enum.Enum], fallback: T = ...) + + A parser that checks a string against a set of cases, returning the value of the matching case. This is a more efficient + version of :class:`MatchParser` that uses a dictionary to store the cases. + + :param lookup: An iterable of match-value pairs, a mapping of strings to values, or an enum type, + in which case the names of the enum members will be used as the matches. + :param fallback: The value to return if no case matches. If not specified, an exception will be raised. + + .. code-block:: + + class Color(enum.Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + color_ev = env_var("COLOR", type=LookupParser(Color)) + + os.environ["COLOR"] = "RED" + + assert color_ev.get() == Color.RED + + .. classmethod:: case_insensitive(lookup: collection.abc.Iterable[tuple[str, T]] | \ + collections.abc.Mapping[str, T] | type[enum.Enum], fallback: T = ...) -> LookupParser[T] + + Create a :class:`LookupParser` where the matches are case insensitive. If two cases are equivalent under + case-insensitivity, an error will be raised. + + :param lookup: Acts the same as in the :class:`constructor `. + :param fallback: Acts the same as in the :class:`constructor `. \ No newline at end of file diff --git a/envolved/__init__.py b/envolved/__init__.py index e1d7cc6..44120c2 100644 --- a/envolved/__init__.py +++ b/envolved/__init__.py @@ -3,5 +3,6 @@ from envolved.describe import describe_env_vars from envolved.envvar import env_var from envolved.exceptions import MissingEnvError +from envolved.infer_env_var import inferred_env_var -__all__ = ["__version__", "env_var", "EnvVar", "MissingEnvError", "describe_env_vars", "as_default"] +__all__ = ["__version__", "env_var", "EnvVar", "MissingEnvError", "describe_env_vars", "as_default", "inferred_env_var"] diff --git a/envolved/_version.py b/envolved/_version.py index 5becc17..6849410 100644 --- a/envolved/_version.py +++ b/envolved/_version.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.1.0" diff --git a/envolved/basevar.py b/envolved/basevar.py index 22dace6..f3d90e7 100644 --- a/envolved/basevar.py +++ b/envolved/basevar.py @@ -101,19 +101,22 @@ def __init__( self.monkeypatch: Union[T, Missing, Discard, NoPatch] = no_patch def get(self) -> T: + return self._get_with() + + def _get_with(self, **kwargs: Any) -> T: if self.monkeypatch is not no_patch: if self.monkeypatch is missing: raise MissingEnvError(self.describe()) return self.monkeypatch # type: ignore[return-value] - return self._get_validated().value # type: ignore[return-value] + return self._get_validated(**kwargs).value # type: ignore[return-value] def validator(self, validator: Callable[[T], T]) -> EnvVar[T]: self._validators.append(validator) return self - def _get_validated(self) -> _EnvVarResult[T]: + def _get_validated(self, **kwargs: Any) -> _EnvVarResult[T]: try: - value = self._get() + value = self._get(**kwargs) except SkipDefault as sd: raise sd.args[0] from None except MissingEnvError as mee: @@ -125,7 +128,7 @@ def _get_validated(self) -> _EnvVarResult[T]: return _EnvVarResult(value, exists=True) @abstractmethod - def _get(self) -> T: + def _get(self, **kwargs: Any) -> T: pass @abstractmethod @@ -176,7 +179,9 @@ def key(self) -> str: def type(self) -> Parser[T]: return self._type - def _get(self) -> T: + def _get(self, **kwargs: Any) -> T: + if kwargs: + raise TypeError(f"unexpected keyword arguments {kwargs!r}") try: raw_value = env_parser.get(self.case_sensitive, self._key) except KeyError as err: @@ -255,9 +260,12 @@ def on_partial(self, value: Union[T, Missing, AsDefault, Discard]): raise TypeError("on_partial cannot be as_default if default is missing") self._on_partial = value - def _get(self) -> T: + def get(self, **kwargs: Any) -> T: + return super()._get_with(**kwargs) + + def _get(self, **kwargs: Any) -> T: pos_values = [] - kw_values = {} + kw_values = kwargs any_exist = False errs: List[MissingEnvError] = [] for env_var in self._pos_args: @@ -272,9 +280,13 @@ def _get(self) -> T: if result.exists: any_exist = True for key, env_var in self._args.items(): + if key in kw_values: + # key could be in kwargs because it was passed in as a positional argument, if so, we don't want to + # overwrite it + continue try: result = env_var._get_validated() # noqa: SLF001 - except MissingEnvError as e: # noqa: PERF203 + except MissingEnvError as e: errs.append(e) else: if result.value is not discard: diff --git a/envolved/describe.py b/envolved/describe.py index 25532e4..98a86ad 100644 --- a/envolved/describe.py +++ b/envolved/describe.py @@ -2,6 +2,7 @@ from envolved.basevar import _Description from envolved.envvar import EnvVar, top_level_env_vars +from envolved.infer_env_var import InferEnvVar def describe_env_vars(**kwargs: Any) -> List[str]: @@ -9,7 +10,10 @@ def describe_env_vars(**kwargs: Any) -> List[str]: return _Description.combine(descriptions, [], allow_blanks=True).lines -T = TypeVar("T", bound=Union[EnvVar, Iterable[EnvVar], Mapping[Any, EnvVar]]) +T = TypeVar( + "T", + bound=Union[EnvVar, InferEnvVar, Iterable[Union[EnvVar, InferEnvVar]], Mapping[Any, Union[EnvVar, InferEnvVar]]], +) def exclude_from_description(to_exclude: T) -> T: @@ -17,6 +21,8 @@ def exclude_from_description(to_exclude: T) -> T: if isinstance(to_exclude, EnvVar): evs = frozenset((to_exclude,)) + elif isinstance(to_exclude, InferEnvVar): + evs = frozenset() elif isinstance(to_exclude, Mapping): evs = frozenset(to_exclude.values()) elif isinstance(to_exclude, Iterable): diff --git a/envolved/parsers.py b/envolved/parsers.py index 47c9003..5d6e8bb 100644 --- a/envolved/parsers.py +++ b/envolved/parsers.py @@ -20,26 +20,80 @@ Union, ) +from envolved.utils import extract_from_option + __all__ = ["Parser", "BoolParser", "CollectionParser", "parser"] -from envolved.utils import extract_from_option + +BaseModel1: Optional[Type] +BaseModel2: Optional[Type] +TypeAdapter: Optional[Type] + +try: # pydantic v2 + from pydantic import BaseModel as BaseModel2, TypeAdapter + from pydantic.v1 import BaseModel as BaseModel1 +except ImportError: + BaseModel2 = TypeAdapter = None + try: # pydantic v1 + from pydantic import BaseModel as BaseModel1 + except ImportError: + BaseModel1 = None T = TypeVar("T") Parser = Callable[[str], T] ParserInput = Union[Parser[T], Type[T]] -special_parsers: Dict[ParserInput[Any], Parser[Any]] = { +special_parser_inputs: Dict[ParserInput[Any], Parser[Any]] = { bytes: str.encode, } +parser_special_instances: Dict[Type, Callable[[Any], Parser]] = {} +if TypeAdapter is not None: + parser_special_instances[TypeAdapter] = lambda t: t.validate_json + +parser_special_superclasses: Dict[Type, Callable[[Type], Parser]] = {} +if BaseModel1 is not None: + parser_special_superclasses[BaseModel1] = lambda t: t.parse_raw +if BaseModel2 is not None: + parser_special_superclasses[BaseModel2] = lambda t: t.model_validate_json + def complex_parser(x: str) -> complex: x = x.replace("i", "j") return complex(x) -special_parsers[complex] = complex_parser +special_parser_inputs[complex] = complex_parser + + +def parser(t: ParserInput[T]) -> Parser[T]: + """ + Coerce an object into a parser. + :param t: The object to coerce to a parser. + :return: The best-match parser for `t`. + """ + special_parser = special_parser_inputs.get(t) + if special_parser is not None: + return special_parser + + from_option = extract_from_option(t) + if from_option is not None: + return parser(from_option) + + for special_cls, parser_factory in parser_special_instances.items(): + if isinstance(t, special_cls): + return parser_factory(t) + + if isinstance(t, type): + for supercls, parser_factory in parser_special_superclasses.items(): + if issubclass(t, supercls): + return parser_factory(t) + + if callable(t): + return t + + raise TypeError(f"cannot coerce type {t!r} to a parser") class BoolParser: @@ -85,31 +139,7 @@ def __call__(self, x: str) -> bool: return self.default -special_parsers[bool] = BoolParser(["true"], ["false"]) - - -def parser(t: ParserInput[T]) -> Parser[T]: - """ - Coerce an object into a parser. - :param t: The object to coerce to a parser. - :return: The best-match parser for `t`. - """ - special_parser = special_parsers.get(t) - if special_parser is not None: - return special_parser - - from_option = extract_from_option(t) - if from_option is not None: - return parser(from_option) - - if isinstance(t, type) and issubclass(t, Enum): - return MatchParser.case_insensitive(t) - - if callable(t): - return t - - raise TypeError(f"cannot coerce type {t!r} to a parser") - +special_parser_inputs[bool] = BoolParser(["true"], ["false"]) E = TypeVar("E") G = TypeVar("G") @@ -295,3 +325,45 @@ def __call__(self, x: str) -> T: if pattern.fullmatch(x): return value raise ValueError(f"no match for {x}") + + +LookupCases = Union[Iterable[Tuple[str, T]], Mapping[str, T], Type[Enum]] + + +class LookupParser(Generic[T]): + def __init__( + self, lookup: LookupCases, fallback: Union[T, NoFallback] = no_fallback, *, _case_sensitive: bool = True + ): + cases: Iterable[Tuple[str, T]] + if isinstance(lookup, Mapping): + cases = lookup.items() + elif isinstance(lookup, type) and issubclass(lookup, Enum): + cases = lookup.__members__.items() # type: ignore[assignment] + else: + cases = lookup + + if _case_sensitive: + self.lookup = dict(cases) + else: + self.lookup = {k.lower(): v for k, v in cases} + self.fallback = fallback + self.case_sensitive = _case_sensitive + + @classmethod + def case_insensitive(cls, lookup: Mapping[str, T], fallback: Union[T, NoFallback] = no_fallback) -> LookupParser[T]: + return cls(lookup, fallback, _case_sensitive=False) + + def __call__(self, x: str) -> T: + if not self.case_sensitive: + key = x.lower() + else: + key = x + try: + return self.lookup[key] + except KeyError as e: + if self.fallback is no_fallback: + raise ValueError(f"no match for {x}") from e + return self.fallback + + +parser_special_superclasses[Enum] = LookupParser.case_insensitive # type: ignore[assignment] diff --git a/envolved/py.typed b/envolved/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index e99be02..c33d5ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,14 @@ [tool.poetry] name = "envolved" -version = "1.0.0" +version = "1.1.0" description = "" authors = ["ben avrahami "] license = "MIT" readme = "README.md" repository = "https://github.com/bentheiii/envolved" packages = [ - {include="envolved"} + {include="envolved"}, + {include="envolved/py.typed"} ] [tool.poetry.dependencies] @@ -19,7 +20,8 @@ sphinx = {version="^7", python = ">=3.12"} sphinx-rtd-theme = {version="*", python = ">=3.12"} mypy = "*" pytest-cov = "^4.1.0" -ruff = "^0.1.2" +ruff = "*" +pydantic = "^2.5.2" [build-system] requires = ["poetry>=0.12"] diff --git a/tests/unittests/test_parsers.py b/tests/unittests/test_parsers.py index 8517038..36a062e 100644 --- a/tests/unittests/test_parsers.py +++ b/tests/unittests/test_parsers.py @@ -1,9 +1,12 @@ import re from enum import Enum +from typing import List +from pydantic import BaseModel as BaseModel2, RootModel, TypeAdapter +from pydantic.v1 import BaseModel as BaseModel1 from pytest import mark, raises -from envolved.parsers import BoolParser, CollectionParser, MatchParser, complex_parser +from envolved.parsers import BoolParser, CollectionParser, LookupParser, MatchParser, complex_parser, parser def test_complex(): @@ -150,3 +153,93 @@ def test_match_dict_caseignore(): with raises(ValueError): parser("D") + + +def test_lookup_dict(): + parser = LookupParser( + { + "a": 1, + "b": 2, + "c": 3, + } + ) + + assert parser("a") == 1 + assert parser("b") == 2 + assert parser("c") == 3 + + with raises(ValueError): + parser("A") + + +def test_lookup_enum(): + class MyEnum(Enum): + RED = 10 + BLUE = 20 + GREEN = 30 + + parser = LookupParser(MyEnum) + + assert parser("RED") is MyEnum.RED + assert parser("BLUE") is MyEnum.BLUE + assert parser("GREEN") is MyEnum.GREEN + + +def test_lookup_enum_caseignore(): + class MyEnum(Enum): + RED = 10 + BLUE = 20 + GREEN = 30 + + parser = LookupParser.case_insensitive(MyEnum) + + assert parser("RED") is MyEnum.RED + assert parser("blue") is MyEnum.BLUE + assert parser("green") is MyEnum.GREEN + + +def test_lookup_dict_caseignore(): + parser = LookupParser.case_insensitive( + { + "a": 1, + "b": 2, + "c": 3, + } + ) + + assert parser("A") == 1 + assert parser("b") == 2 + assert parser("C") == 3 + + with raises(ValueError): + parser("D") + + +def test_basemodel2(): + class M(BaseModel2): + a: int + b: str + + p = parser(M) + assert p('{"a": "1", "b": "hi"}') == M(a=1, b="hi") + + +def test_basemodel1(): + class M(BaseModel1): + a: int + b: str + + p = parser(M) + assert p('{"a": "1", "b": "hi"}') == M(a=1, b="hi") + + +def test_rootmodel(): + m = RootModel[List[int]] + p = parser(m) + assert p("[1,2,3]") == m([1, 2, 3]) + + +def test_typeadapter(): + t = TypeAdapter(List[int]) + p = parser(t) + assert p("[1,2,3]") == [1, 2, 3] diff --git a/tests/unittests/test_schema.py b/tests/unittests/test_schema.py index 724e98b..51d9dfd 100644 --- a/tests/unittests/test_schema.py +++ b/tests/unittests/test_schema.py @@ -404,3 +404,54 @@ class A: monkeypatch.setenv("a_c", "red") assert a.get() == A("hi", Color.RED) + + +def test_get_runtime(monkeypatch): + s = env_var( + "s", + type=dict, + args={ + "a": env_var("a", type=int), + "b": env_var("b", type=str), + }, + ) + + monkeypatch.setenv("sa", "12") + monkeypatch.setenv("sb", "foo") + + assert s.get(b="bla", d=12) == {"a": 12, "b": "bla", "d": 12} + + +def test_patch_beats_runtime(): + s = env_var( + "s", + type=dict, + args={ + "a": env_var("a", type=int), + "b": env_var("b", type=str), + }, + ) + + with s.patch({"foo": "bar"}): + assert s.get(c="bla", d=12) == {"foo": "bar"} + + +def test_validate_runtime(monkeypatch): + s = env_var( + "s", + type=dict, + args={ + "a": env_var("a", type=int), + "b": env_var("b", type=str), + }, + ) + + @s.validator + def validate(d): + d["d"] *= 2 + return d + + monkeypatch.setenv("sa", "12") + monkeypatch.setenv("sb", "foo") + + assert s.get(c="bla", d=12) == {"a": 12, "b": "foo", "c": "bla", "d": 24}