feat(ini): allow custom parsers/serializers for ini options (#2332)

Also moves INI-related features to a new module dedicated to serialization
and deserialization, antarest.core.serde.

Signed-off-by: Sylvain Leclerc <[email protected]>

alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py

@@ -15,7 +15,7 @@
 from sqlalchemy.engine import Connection  # type: ignore
 
 from antarest.study.css4_colors import COLOR_NAMES
-from antarest.core.serialization import from_json, to_json
+from antarest.core.serde.json import from_json, to_json
 
 # revision identifiers, used by Alembic.
 revision = "dae93f1d9110"

antarest/core/cache/business/local_chache.py

@@ -20,7 +20,7 @@
 from antarest.core.config import CacheConfig
 from antarest.core.interfaces.cache import ICache
 from antarest.core.model import JSON
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 
 logger = logging.getLogger(__name__)
 

antarest/core/cache/business/redis_cache.py

@@ -18,7 +18,8 @@
 
 from antarest.core.interfaces.cache import ICache
 from antarest.core.model import JSON
-from antarest.core.serialization import AntaresBaseModel, from_json
+from antarest.core.serde import AntaresBaseModel
+from antarest.core.serde.json import from_json
 
 logger = logging.getLogger(__name__)
 

antarest/core/configdata/model.py

@@ -17,7 +17,7 @@
 from typing_extensions import override
 
 from antarest.core.persistence import Base
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 
 
 class ConfigDataDTO(AntaresBaseModel):

antarest/core/configdata/repository.py

@@ -16,7 +16,7 @@
 from antarest.core.configdata.model import ConfigData
 from antarest.core.jwt import DEFAULT_ADMIN_USER
 from antarest.core.model import JSON
-from antarest.core.serialization import from_json, to_json_string
+from antarest.core.serde.json import from_json, to_json_string
 from antarest.core.utils.fastapi_sqlalchemy import db
 
 

antarest/core/core_blueprint.py

@@ -15,7 +15,7 @@
 from fastapi import APIRouter
 
 from antarest.core.config import Config
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 from antarest.core.utils.web import APITag
 from antarest.core.version_info import VersionInfoDTO, get_commit_id, get_dependencies
 

antarest/core/filesystem_blueprint.py

@@ -27,7 +27,7 @@
 from starlette.responses import PlainTextResponse, StreamingResponse
 
 from antarest.core.config import Config
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 from antarest.core.utils.web import APITag
 from antarest.login.auth import Auth
 

antarest/core/filetransfer/model.py

@@ -19,7 +19,7 @@
 from typing_extensions import override
 
 from antarest.core.persistence import Base
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 
 
 class FileDownloadNotFound(HTTPException):

antarest/core/interfaces/eventbus.py

@@ -17,7 +17,7 @@
 from typing_extensions import override
 
 from antarest.core.model import PermissionInfo
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 
 
 class EventType(StrEnum):

antarest/core/jwt.py

@@ -13,7 +13,7 @@
 from typing import List, Union
 
 from antarest.core.roles import RoleType
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 from antarest.login.model import ADMIN_ID, Group, Identity
 
 

antarest/core/model.py

@@ -13,7 +13,7 @@
 import enum
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 
 if TYPE_CHECKING:
     # These dependencies are only used for type checking with mypy.

antarest/core/serde/__init__.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+#
+# See AUTHORS.txt
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This file is part of the Antares project.
+
+"""
+This modules hosts technical components related to serializing and deserializing,
+for various formats: INI, JSON, ...
+"""
+import typing as t
+
+import pydantic
+
+
+class AntaresBaseModel(pydantic.BaseModel):
+    """
+    Due to pydantic migration from v1 to v2, we can have this issue:
+
+    class A(BaseModel):
+        a: str
+
+    A(a=2) raises ValidationError as we give an int instead of a str
+
+    To avoid this issue we created our own BaseModel class that inherits from BaseModel and allows such object creation.
+    """
+
+    model_config = pydantic.config.ConfigDict(coerce_numbers_to_str=True)

antarest/core/serde/ini_common.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+#
+# See AUTHORS.txt
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This file is part of the Antares project.
+
+import dataclasses
+from typing import Optional
+
+PrimitiveType = str | int | float | bool
+
+
+@dataclasses.dataclass(frozen=True)
+class OptionMatcher:
+    """
+    Used to match a location in an INI file:
+    a None section means any section.
+    """
+
+    section: Optional[str]
+    key: str
+
+
+def any_section_option_matcher(key: str) -> OptionMatcher:
+    """
+    Return a matcher which will match the provided key in any section.
+    """
+    return OptionMatcher(section=None, key=key)

antarest/study/storage/rawstudy/ini_reader.py → antarest/core/serde/ini_reader.py

@@ -9,25 +9,34 @@
 # SPDX-License-Identifier: MPL-2.0
 #
 # This file is part of the Antares project.
-
 import dataclasses
 import re
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Dict, Mapping, Optional, Pattern, Sequence, TextIO, cast
+from typing import Any, Callable, Dict, Mapping, Optional, Pattern, Sequence, TextIO, cast
 
 from typing_extensions import override
 
 from antarest.core.model import JSON
+from antarest.core.serde.ini_common import OptionMatcher, PrimitiveType, any_section_option_matcher
+
+ValueParser = Callable[[str], PrimitiveType]
+
+
+def _lower_case(input: str) -> str:
+    return input.lower()
+
+
+LOWER_CASE_PARSER: ValueParser = _lower_case
 
 
-def convert_value(value: str) -> str | int | float | bool:
+def _convert_value(value: str) -> PrimitiveType:
     """Convert value to the appropriate type for JSON."""
 
     try:
         # Infinity values are not supported by JSON, so we use a string instead.
         mapping = {"true": True, "false": False, "+inf": "+Inf", "-inf": "-Inf", "inf": "+Inf"}
-        return cast(str | int | float | bool, mapping[value.lower()])
+        return cast(PrimitiveType, mapping[value.lower()])
     except KeyError:
         try:
             return int(value)
@@ -38,7 +47,25 @@ def convert_value(value: str) -> str | int | float | bool:
                 return value
 
 
-@dataclasses.dataclass
+class ValueParsers:
+    def __init__(self, default_parser: ValueParser, parsers: Dict[OptionMatcher, ValueParser]):
+        self._default_parser = default_parser
+        self._parsers = parsers
+
+    def find_parser(self, section: str, key: str) -> ValueParser:
+        if self._parsers:
+            possible_keys = [
+                OptionMatcher(section=section, key=key),
+                any_section_option_matcher(key=key),
+            ]
+            for k in possible_keys:
+                if parser := self._parsers.get(k, None):
+                    return parser
+
+        return self._default_parser
+
+
+@dataclasses.dataclass(frozen=True)
 class IniFilter:
     """
     Filter sections and options in an INI file based on regular expressions.
@@ -115,8 +142,8 @@ def read(self, path: Any, **kwargs: Any) -> JSON:
         Parse `.ini` file to json object.
 
         Args:
-            path: Path to `.ini` file or file-like object.
-            kwargs: Additional options used for reading.
+            path:    Path to `.ini` file or file-like object.
+            options: Additional options used for reading.
 
         Returns:
             Dictionary of parsed `.ini` file which can be converted to JSON.
@@ -152,11 +179,17 @@ class IniReader(IReader):
     This class is not compatible with standard `.ini` readers.
     """
 
-    def __init__(self, special_keys: Sequence[str] = (), section_name: str = "settings") -> None:
+    def __init__(
+        self,
+        special_keys: Sequence[str] = (),
+        section_name: str = "settings",
+        value_parsers: Dict[OptionMatcher, ValueParser] | None = None,
+    ) -> None:
         super().__init__()
 
         # Default section name to use if `.ini` file has no section.
         self._special_keys = set(special_keys)
+        self._value_parsers = ValueParsers(default_parser=_convert_value, parsers=value_parsers or {})
 
         # List of keys which should be parsed as list.
         self._section_name = section_name
@@ -313,10 +346,12 @@ def _handle_option(self, ini_filter: IniFilter, section: str, key: str, value: s
     def _append_option(self, section: str, key: str, value: str) -> None:
         self._curr_sections.setdefault(section, {})
         values = self._curr_sections[section]
+        parser = self._value_parsers.find_parser(section, key)
+        parsed = parser(value)
         if key in self._special_keys:
-            values.setdefault(key, []).append(convert_value(value))
+            values.setdefault(key, []).append(parsed)
         else:
-            values[key] = convert_value(value)
+            values[key] = parsed
         self._curr_option = key
 
 

antarest/study/storage/rawstudy/ini_writer.py → antarest/core/serde/ini_writer.py

@@ -13,17 +13,49 @@
 import ast
 import configparser
 from pathlib import Path
-from typing import List, Optional
+from typing import Callable, Dict, List, Optional
 
 from typing_extensions import override
 
 from antarest.core.model import JSON
+from antarest.core.serde.ini_common import OptionMatcher, PrimitiveType, any_section_option_matcher
+
+# Value serializers may be used to customize the way INI options are serialized
+ValueSerializer = Callable[[str], PrimitiveType]
+
+
+def _lower_case(input: str) -> str:
+    return input.lower()
+
+
+LOWER_CASE_SERIALIZER: ValueSerializer = _lower_case
+
+
+class ValueSerializers:
+    def __init__(self, serializers: Dict[OptionMatcher, ValueSerializer]):
+        self._serializers = serializers
+
+    def find_serializer(self, section: str, key: str) -> Optional[ValueSerializer]:
+        if self._serializers:
+            possible_keys = [
+                OptionMatcher(section=section, key=key),
+                any_section_option_matcher(key=key),
+            ]
+            for k in possible_keys:
+                if parser := self._serializers.get(k, None):
+                    return parser
+        return None
 
 
 class IniConfigParser(configparser.RawConfigParser):
-    def __init__(self, special_keys: Optional[List[str]] = None) -> None:
+    def __init__(
+        self,
+        special_keys: Optional[List[str]] = None,
+        value_serializers: Optional[ValueSerializers] = None,
+    ) -> None:
         super().__init__()
         self.special_keys = special_keys
+        self._value_serializers = value_serializers or ValueSerializers({})
 
     # noinspection SpellCheckingInspection
     @override
@@ -41,6 +73,9 @@ def _write_line(  # type:ignore
         value = self._interpolation.before_write(  # type:ignore
             self, section_name, key, value
         )
+        if self._value_serializers:
+            if serializer := self._value_serializers.find_serializer(section_name, key):
+                value = serializer(value)
         if value is not None or not self._allow_no_value:  # type:ignore
             value = delimiter + str(value).replace("\n", "\n\t")
         else:
@@ -70,8 +105,13 @@ class IniWriter:
     Standard INI writer.
     """
 
-    def __init__(self, special_keys: Optional[List[str]] = None):
+    def __init__(
+        self,
+        special_keys: Optional[List[str]] = None,
+        value_serializers: Optional[Dict[OptionMatcher, ValueSerializer]] = None,
+    ):
         self.special_keys = special_keys
+        self._value_serializers = ValueSerializers(value_serializers or {})
 
     def write(self, data: JSON, path: Path) -> None:
         """
@@ -81,7 +121,7 @@ def write(self, data: JSON, path: Path) -> None:
             data: JSON content.
             path: path to `.ini` file.
         """
-        config_parser = IniConfigParser(special_keys=self.special_keys)
+        config_parser = IniConfigParser(special_keys=self.special_keys, value_serializers=self._value_serializers)
         config_parser.read_dict(data)
         with path.open("w") as fp:
             config_parser.write(fp)

antarest/core/serialization/__init__.py → antarest/core/serde/json.py

@@ -32,18 +32,3 @@ def to_json(data: Any, indent: Optional[int] = None) -> bytes:
 
 def to_json_string(data: Any, indent: Optional[int] = None) -> str:
     return to_json(data, indent=indent).decode("utf-8")
-
-
-class AntaresBaseModel(pydantic.BaseModel):
-    """
-    Due to pydantic migration from v1 to v2, we can have this issue:
-
-    class A(BaseModel):
-        a: str
-
-    A(a=2) raises ValidationError as we give an int instead of a str
-
-    To avoid this issue we created our own BaseModel class that inherits from BaseModel and allows such object creation.
-    """
-
-    model_config = pydantic.config.ConfigDict(coerce_numbers_to_str=True)

antarest/core/tasks/model.py

@@ -21,7 +21,7 @@
 from typing_extensions import override
 
 from antarest.core.persistence import Base
-from antarest.core.serialization import AntaresBaseModel
+from antarest.core.serde import AntaresBaseModel
 
 if TYPE_CHECKING:
     # avoid circular import