Merge pull request #1332 from gammasim/write-prod5-model-parameters

Replace model parameter default schema versions by value read from model_parameter.metaschema.yml

docs/source/api-reference/data_model.md

@@ -50,6 +50,15 @@ Data products ingested or produced by simtools generally follows the CTAO data m
    :members:
 ```
 
+(datamodelschema)=
+
+## schema
+
+```{eval-rst}
+.. automodule:: data_model.schema
+   :members:
+```
+
 (datamodelvalidatedata)=
 
 ## validate_data

src/simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -61,7 +61,7 @@
 import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
+from simtools.data_model import schema
 from simtools.io_operations.io_handler import IOHandler
 from simtools.simtel.simtel_config_reader import SimtelConfigReader
 
@@ -108,26 +108,6 @@ def _parse(label=None, description=None):
     return config.initialize(simulation_model=["telescope", "parameter_version"])
 
 
-def get_list_of_parameters_and_schema_files(schema_directory):
-    """
-    Return list of parameters and schema files located in schema file directory.
-
-    Returns
-    -------
-    list
-        List of parameters found in schema file directory.
-    list
-        List of schema files found in schema file directory.
-
-    """
-    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
-    parameters = []
-    for schema_file in schema_files:
-        schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        parameters.append(schema_dict.get("name"))
-    return parameters, schema_files
-
-
 def get_list_of_simtel_parameters(simtel_config_file, logger):
     """
     Return list of simtel parameters found in simtel configuration file.
@@ -205,7 +185,7 @@ def get_number_of_camera_pixel(args_dict, logger):
     """
     try:
         simtel_config_reader = SimtelConfigReader(
-            schema_file=MODEL_PARAMETER_SCHEMA_PATH / "camera_pixels.schema.yml",
+            schema_file=schema.get_model_parameter_schema_file("camera_pixels"),
             simtel_config_file=args_dict["simtel_cfg_file"],
             simtel_telescope_name=args_dict["simtel_telescope_name"],
         )
@@ -239,8 +219,8 @@ def read_and_export_parameters(args_dict, logger):
         List of simtools parameter not found in simtel configuration file.
 
     """
-    _parameters, _schema_files = get_list_of_parameters_and_schema_files(
-        args_dict.get("schema_directory", MODEL_PARAMETER_SCHEMA_PATH)
+    _parameters, _schema_files = schema.get_get_model_parameter_schema_files(
+        args_dict.get("schema_directory")
     )
     _simtel_parameters = get_list_of_simtel_parameters(args_dict["simtel_cfg_file"], logger)
 

src/simtools/applications/validate_file_using_schema.py

@@ -41,7 +41,7 @@
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import metadata_collector, metadata_model, validate_data
+from simtools.data_model import metadata_collector, metadata_model, schema, validate_data
 
 
 def _parse(label, description):
@@ -160,7 +160,7 @@ def validate_data_files(args_dict, logger):
         for file_name in _get_json_file_list(args_dict.get("file_directory")):
             tmp_args_dict["file_name"] = file_name
             parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
-            schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
+            schema_file = schema.get_model_parameter_schema_file(f"{parameter_name}")
             tmp_args_dict["schema"] = schema_file
             tmp_args_dict["data_type"] = "model_parameter"
             tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]

src/simtools/constants.py

@@ -2,8 +2,15 @@
 
 from importlib.resources import files
 
+# Schema path
+SCHEMA_PATH = files("simtools") / "schemas"
 # Path to metadata jsonschema
-METADATA_JSON_SCHEMA = files("simtools") / "schemas/metadata.metaschema.yml"
-
+METADATA_JSON_SCHEMA = SCHEMA_PATH / "metadata.metaschema.yml"
+# Path to model parameter metaschema
+MODEL_PARAMETER_METASCHEMA = SCHEMA_PATH / "model_parameter.metaschema.yml"
+# Path to model parameter description metaschema
+MODEL_PARAMETER_DESCRIPTION_METASCHEMA = (
+    SCHEMA_PATH / "model_parameter_and_data_schema.metaschema.yml"
+)
 # Path to model parameter schema files
-MODEL_PARAMETER_SCHEMA_PATH = files("simtools") / "schemas/model_parameters"
+MODEL_PARAMETER_SCHEMA_PATH = SCHEMA_PATH / "model_parameters"

src/simtools/data_model/metadata_collector.py

@@ -10,13 +10,12 @@
 import getpass
 import logging
 import uuid
-from importlib.resources import files
 from pathlib import Path
 
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
-from simtools.data_model import metadata_model
+from simtools.data_model import metadata_model, schema
 from simtools.io_operations import io_handler
 from simtools.utils import names
 
@@ -135,7 +134,7 @@ def get_data_model_schema_file_name(self):
         # from data model name
         if self.data_model_name:
             self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
-            return f"{files('simtools')}/schemas/model_parameters/{self.data_model_name}.schema.yml"
+            return str(schema.get_model_parameter_schema_file(self.data_model_name))
 
         # from input metadata
         try:

src/simtools/data_model/metadata_model.py

@@ -13,8 +13,8 @@
 
 import jsonschema
 
-import simtools.constants
 import simtools.utils.general as gen
+from simtools.constants import METADATA_JSON_SCHEMA
 from simtools.data_model import format_checkers
 from simtools.utils import names
 
@@ -94,7 +94,7 @@ def _load_schema(schema_file=None, schema_version=None):
 
     """
     if schema_file is None:
-        schema_file = files("simtools").joinpath(simtools.constants.METADATA_JSON_SCHEMA)
+        schema_file = METADATA_JSON_SCHEMA
 
     try:
         schema = gen.collect_data_from_file(file_name=schema_file)

src/simtools/data_model/model_data_writer.py

@@ -10,8 +10,7 @@
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import validate_data
+from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
@@ -176,7 +175,7 @@ def dump_model_parameter(
         return _json_dict
 
     def get_validated_parameter_dict(
-        self, parameter_name, value, instrument, parameter_version, schema_version="0.1.0"
+        self, parameter_name, value, instrument, parameter_version, schema_version=None
     ):
         """
         Get validated parameter dictionary.
@@ -200,7 +199,8 @@ def get_validated_parameter_dict(
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = self._read_model_parameter_schema(parameter_name)
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        self.schema_dict = gen.collect_data_from_file(schema_file)
 
         try:  # e.g. instrument is 'North"
             site = names.validate_site_name(instrument)
@@ -210,7 +210,7 @@ def get_validated_parameter_dict(
         value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
-            "schema_version": schema_version,
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": instrument,
             "site": site,
@@ -227,22 +227,6 @@ def get_validated_parameter_dict(
             is_model_parameter=True,
         )
 
-    def _read_model_parameter_schema(self, parameter_name):
-        """
-        Read model parameter schema.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Name of the parameter.
-        """
-        schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
-        try:
-            self.schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        except FileNotFoundError as exc:
-            raise FileNotFoundError(f"Schema file not found: {schema_file}") from exc
-        return schema_file
-
     def _get_parameter_type(self):
         """
         Return parameter type from schema.

src/simtools/data_model/schema.py

@@ -0,0 +1,77 @@
+"""Module providing functionality to read schema."""
+
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.constants import MODEL_PARAMETER_METASCHEMA, MODEL_PARAMETER_SCHEMA_PATH
+
+
+def get_get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PATH):
+    """
+    Return list of parameters and schema files located in schema file directory.
+
+    Returns
+    -------
+    list
+        List of parameters found in schema file directory.
+    list
+        List of schema files found in schema file directory.
+
+    """
+    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
+    if not schema_files:
+        raise FileNotFoundError(f"No schema files found in {schema_directory}")
+    parameters = []
+    for schema_file in schema_files:
+        schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        parameters.append(schema_dict.get("name"))
+    return parameters, schema_files
+
+
+def get_model_parameter_schema_file(parameter):
+    """
+    Return schema file path for a given model parameter.
+
+    Parameters
+    ----------
+    parameter: str
+        Model parameter name.
+
+    Returns
+    -------
+    Path
+        Schema file path.
+
+    """
+    schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter}.schema.yml"
+    if not schema_file.exists():
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+    return schema_file
+
+
+def get_model_parameter_schema_version(schema_version=None):
+    """
+    Validate  and return schema versions.
+
+    If no schema_version is given, the most recent version is provided.
+
+    Parameters
+    ----------
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    str
+        Schema version.
+
+    """
+    schemas = gen.collect_data_from_file(MODEL_PARAMETER_METASCHEMA)
+
+    if schema_version is None and schemas:
+        return schemas[0].get("version")
+
+    if any(schema.get("version") == schema_version for schema in schemas):
+        return schema_version
+
+    raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")

src/simtools/data_model/validate_data.py

@@ -12,8 +12,7 @@
 from astropy.utils.diff import report_diff_values
 
 import simtools.utils.general as gen
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import format_checkers
+from simtools.data_model import format_checkers, schema
 from simtools.utils import value_conversion
 
 __all__ = ["DataValidator"]
@@ -129,7 +128,7 @@ def validate_model_parameter(par_dict):
             Validated data dictionary
         """
         data_validator = DataValidator(
-            schema_file=MODEL_PARAMETER_SCHEMA_PATH / f"{par_dict['parameter']}.schema.yml",
+            schema_file=schema.get_model_parameter_schema_file(f"{par_dict['parameter']}"),
             data_dict=par_dict,
             check_exact_data_type=False,
         )
@@ -158,9 +157,7 @@ def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         if is_model_parameter:
             self._prepare_model_parameter()
 
-        if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")):
-            raise KeyError("Data dict does not contain a 'name' or 'parameter' key.")
-        self._data_description = self._read_validation_schema(self.schema_file_name, _name)
+        self._data_description = self._read_validation_schema(self.schema_file_name)
 
         value_as_list, unit_as_list = self._get_value_and_units_as_lists()
 
@@ -692,19 +689,14 @@ def _interval_check(data, axis_range, range_type):
 
         return False
 
-    def _read_validation_schema(self, schema_file, parameter=None):
+    def _read_validation_schema(self, schema_file):
         """
         Read validation schema from file.
 
         Parameters
         ----------
         schema_file: Path
             Schema file describing input data.
-            If this is a directory, a filename of
-            '<par>.schema.yml' is assumed.
-        parameter: str
-            Parameter name of required schema
-            (if None, return first schema in file)
 
         Returns
         -------
@@ -715,17 +707,11 @@ def _read_validation_schema(self, schema_file, parameter=None):
         ------
         KeyError
             if 'data' can not be read from dict in schema file
-
         """
         try:
-            if Path(schema_file).is_dir():
-                return gen.collect_data_from_file(
-                    file_name=Path(schema_file) / (parameter + ".schema.yml"),
-                )["data"]
             return gen.collect_data_from_file(file_name=schema_file)["data"]
-        except KeyError:
-            self._logger.error(f"Error reading validation schema from {schema_file}")
-            raise
+        except KeyError as exc:
+            raise KeyError(f"Error reading validation schema from {schema_file}") from exc
 
     def _get_data_description(self, column_name=None, status_test=False):
         """

src/simtools/layout/array_layout.py

@@ -9,7 +9,7 @@
 from astropy.table import QTable
 
 import simtools.utils.general as gen
-from simtools.data_model import data_reader
+from simtools.data_model import data_reader, schema
 from simtools.io_operations import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
@@ -582,7 +582,10 @@ def export_telescope_list_table(self, crs_name):
         return table
 
     def export_one_telescope_as_json(
-        self, crs_name, parameter_version=None, schema_version="0.2.0"
+        self,
+        crs_name,
+        parameter_version=None,
+        schema_version=None,
     ):
         """
         Return a list containing a single telescope in simtools-DB-style json.
@@ -626,7 +629,7 @@ def export_one_telescope_as_json(
             ]
 
         return {
-            "schema_version": schema_version,
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": table["telescope_name"][0],
             "site": self.site,