Require semantic version for model parameter version string

.github/workflows/CI-integrationtests.yml

@@ -71,7 +71,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        type: ['2024-02-01', 'prod5']
+        type: ['6.0.0', '5.0.0']
 
     defaults:
       run:

database_scripts/upload_from_model_repository_to_local_db.sh

@@ -63,7 +63,7 @@ for dir in "${model_directory}"*/; do
   else
     simtools-db-add-model-parameters-from-repository-to-db \
     --model_version "${model-version}" \
-    --input_path "${dir}"/verified_model \
+    --input_path "${dir}" \
     --db_name $SIMTOOLS_DB_SIMULATION_MODEL \
     --type "model_parameters"
   fi

docs/source/user-guide/model_parameters.md

@@ -192,7 +192,7 @@ A typical model parameter file looks like:
     "parameter": "num_gains",
     "instrument": "LSTN-01",
     "site": "North",
-    "version": "2024-02-01",
+    "version": "6.0.0",
     "value": 2,
     "unit": null,
     "type": "int64",

simtools/applications/db_get_array_layouts_from_db.py

@@ -32,20 +32,20 @@
 
 .. code-block:: console
 
-    simtools-db-get-array-layouts-from-db --site North --model_version "2024-02-01"
+    simtools-db-get-array-layouts-from-db --site North --model_version "6.0.0"
 
 Retrieve telescope positions for array layout 'test_layout' from database.
 
 .. code-block:: console
 
-    simtools-db-get-array-layouts-from-db --site North --model_version "2024-02-01"
+    simtools-db-get-array-layouts-from-db --site North --model_version "6.0.0"
         --array_layout_name test_layout
 
 Retrieve telescope positions from database (utm coordinate system) and write to an ecsv files
 
 .. code-block:: console
 
-    simtools-db-get-array-layouts-from-db --site North --model_version "2024-02-01"
+    simtools-db-get-array-layouts-from-db --site North --model_version "6.0.0"
       --array_element_list LSTN-01 LSTN-02 MSTN
       --coordinate_system utm
       --output_file telescope_positions-test_layout.ecsv

simtools/applications/db_get_parameter_from_db.py

@@ -43,7 +43,7 @@
          'File': True,
          'Type': 'str',
          'Value': 'mirror_CTA-N-LST1_v2019-03-31.dat',
-         'Version': '2020-06-28',
+         'Version': '5.0.0',
          '_id': ObjectId('608834f257df2db2531b8e78'),
          'entry_date': datetime.datetime(2021, 4, 27, 15, 59, 46, tzinfo=<bson.tz_util.FixedOffset \
           object at 0x7f601dd51d80>)}

simtools/data_model/validate_data.py

@@ -155,6 +155,8 @@ def _validate_data_dict(self):
         if len(value_as_list) == 1:
             self.data_dict["value"], self.data_dict["unit"] = value_as_list[0], unit_as_list[0]
 
+        self._check_version_string(self.data_dict.get("version"))
+
     def _validate_data_table(self):
         """Validate tabulated data."""
         try:
@@ -708,3 +710,25 @@ def _prepare_model_parameter(self):
                 if self.data_dict["unit"] is None
                 else gen.convert_string_to_list(self.data_dict["unit"])
             )
+
+    def _check_version_string(self, version):
+        """
+        Check that version string follows semantic versioning.
+
+        Parameters
+        ----------
+        version: str
+            version string
+
+        Raises
+        ------
+        ValueError
+            if version string does not follow semantic versioning
+
+        """
+        if version is None:
+            return
+        semver_regex = r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$"
+        if not re.match(semver_regex, version):
+            raise ValueError(f"Invalid version string '{version}'")
+        self._logger.debug(f"Valid version string '{version}'")

simtools/db/db_from_repo_handler.py

@@ -20,7 +20,6 @@ def update_model_parameters_from_repo(
     parameter_collection,
     model_version,
     db_simulation_model_url,
-    db_simulation_model="verified_model",
 ):
     """
     Update model parameters with values from a repository.
@@ -63,7 +62,6 @@ def update_model_parameters_from_repo(
             db_simulation_model_url,
             "model_versions",
             model_version,
-            db_simulation_model,
             array_element_name,
         )
         # use design array element model in case there is no model defined for this array element ID
@@ -75,7 +73,6 @@ def update_model_parameters_from_repo(
             db_simulation_model_url,
             "model_versions",
             model_version,
-            db_simulation_model,
             "OBS-" + site,
         )
         _design_model = None
@@ -95,7 +92,6 @@ def update_model_parameters_from_repo(
                     db_simulation_model_url,
                     "model_versions",
                     model_version,
-                    db_simulation_model,
                     _design_model,
                 )
                 _tmp_par = gen.collect_data_from_file_or_dict(

tests/conftest.py

@@ -187,7 +187,7 @@ def pytest_addoption(parser):
 @pytest.fixture()
 def model_version():
     """Simulation model version used in tests."""
-    return "2024-02-01"
+    return "6.0.0"
 
 
 @pytest.fixture()

tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml

@@ -5,7 +5,7 @@ CTA_SIMPIPE:
     SIMTEL_CFG_FILE: ./tests/resources/simtel_config_test_la_palma.cfg
     SIMTEL_TELESCOPE_NAME: CT1
     TELESCOPE: LSTN-01
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     SCHEMA_DIRECTORY: ./tests/resources/
     OUTPUT_PATH: simtools-tests
   INTEGRATION_TESTS:

tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml

@@ -5,7 +5,7 @@ CTA_SIMPIPE:
     INPUT: tests/resources/array_element_position_ground.json
     SITE: South
     EXPORT: utm
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     OUTPUT_PATH: simtools-tests
     OUTPUT_FILE: test.json
     LOG_LEVEL: DEBUG

tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml

@@ -5,7 +5,7 @@ CTA_SIMPIPE:
     INPUT: tests/resources/array_element_position_utm.json
     SITE: South
     EXPORT: ground
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     OUTPUT_PATH: simtools-tests
     OUTPUT_FILE: test.json
     LOG_LEVEL: DEBUG

tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml

@@ -5,7 +5,7 @@ CTA_SIMPIPE:
     SIMTEL_CFG_FILE: ./tests/resources/simtel_config_test_la_palma.cfg
     SIMTEL_TELESCOPE_NAME: CT1
     TELESCOPE: LSTN-01
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     SCHEMA: ./tests/resources/num_gains.schema.yml
     OUTPUT_PATH: simtools-tests
     OUTPUT_FILE: num_gains.json

tests/integration_tests/config/db_get_array_layouts_from_db_layout_list.yml

@@ -3,7 +3,7 @@ CTA_SIMPIPE:
   TEST_NAME: layout_list
   CONFIGURATION:
     SITE: North
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     ARRAY_ELEMENT_LIST: ["LSTS", "MSTS", "SSTS"]
     COORDINATE_SYSTEM: ground
     OUTPUT_PATH: simtools-tests

tests/integration_tests/config/db_get_array_layouts_from_db_layout_name.yml

@@ -3,5 +3,5 @@ CTA_SIMPIPE:
   TEST_NAME: layout_name
   CONFIGURATION:
     SITE: North
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     ARRAY_LAYOUT_NAME: test_layout

tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml

@@ -3,5 +3,5 @@ CTA_SIMPIPE:
   TEST_NAME: list_arrays
   CONFIGURATION:
     SITE: North
-    MODEL_VERSION: "2024-02-01"
+    MODEL_VERSION: "6.0.0"
     LIST_AVAILABLE_LAYOUTS: True

tests/integration_tests/config/validate_camera_efficiency_ssts.yml

@@ -1,6 +1,7 @@
 CTA_SIMPIPE:
   APPLICATION: simtools-validate-camera-efficiency
   TEST_NAME: SSTS
+  MODEL_VERSION_USE_CURRENT: true
   CONFIGURATION:
     SITE: South
     TELESCOPE: SSTS-design

tests/resources/array_element_position_ground.json

@@ -2,7 +2,7 @@
     "parameter": "array_element_position_ground",
     "instrument": "SSTS-09",
     "site": "South",
-    "version": "2024-02-01",
+    "version": "6.0.0",
     "value": "-4.79 -499.24 39.75",
     "unit": "m",
     "type": "float64",

tests/resources/array_element_position_utm.json

@@ -2,7 +2,7 @@
     "parameter": "array_element_position_utm",
     "instrument": "SSTS-09",
     "site": "South",
-    "version": "2024-02-01",
+    "version": "6.0.0",
     "value": "367321.0 7269466.0 2183.5",
     "unit": "m",
     "type": "float64",

tests/resources/num_gains.json

@@ -2,7 +2,7 @@
     "parameter": "num_gains",
     "instrument": "LSTN-01",
     "site": "North",
-    "version": "2024-02-01",
+    "version": "6.0.0",
     "value": 2,
     "unit": null,
     "type": "int64",

tests/unit_tests/data_model/test_validate_data.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import logging
+import re
 import shutil
 import sys
 from importlib.resources import files
@@ -695,3 +696,41 @@ def test_prepare_model_parameter():
     data_validator.data_dict["type"] = "int64"
     data_validator._prepare_model_parameter()
     assert isinstance(data_validator.data_dict["value"][0], int)
+
+
+def test_check_version_string(caplog):
+    data_validator = validate_data.DataValidator()
+
+    valid_versions = [
+        "1.0.0",
+        "0.1.0",
+        "2.3.4",
+        "1.0.0-alpha",
+        "1.0.0-alpha.1",
+        "1.0.0-0.3.7",
+        "1.0.0-x.7.z.92",
+        "1.0.0-alpha+001",
+        "1.0.0+20130313144700",
+        "1.0.0-beta+exp.sha.5114f85",
+    ]
+
+    for version in valid_versions:
+        with caplog.at_level("DEBUG"):
+            data_validator._check_version_string(version)
+        assert f"Valid version string '{version}'" in caplog.text
+        caplog.clear()
+
+    invalid_versions = [
+        "1.0",
+        "1.0.0.0",
+        "1.0.a",
+        "1.0.0-",
+        "1.0.0+",
+        "a.b.c",
+    ]
+
+    for version in invalid_versions:
+        with pytest.raises(ValueError, match=f"Invalid version string '{re.escape(version)}'"):
+            data_validator._check_version_string(version)
+
+    assert data_validator._check_version_string(None) is None

tests/unit_tests/db/test_db_from_repo_handler.py

@@ -17,10 +17,9 @@ def test_update_parameters_from_repo(caplog, db_config, simulation_model_url):
                 parameters={},
                 site="North",
                 array_element_name="MSTN-01",
-                model_version="2024-02-01",
+                model_version="6.0.0",
                 parameter_collection="telescopes",
                 db_simulation_model_url=None,
-                db_simulation_model="verified_model",
             )
             == {}
         )
@@ -33,10 +32,9 @@ def test_update_parameters_from_repo(caplog, db_config, simulation_model_url):
             parameters=dict.fromkeys(_pars_telescope_model, None),
             site="North",
             array_element_name=_tel,
-            model_version="2024-02-01",
+            model_version="6.0.0",
             parameter_collection="telescopes",
             db_simulation_model_url=simulation_model_url,
-            db_simulation_model="verified_model",
         )
         assert len(_pars_mstn) > 0
         assert "telescope_axis_height" in _pars_mstn
@@ -53,10 +51,9 @@ def test_update_parameters_from_repo(caplog, db_config, simulation_model_url):
         parameters=dict.fromkeys(_pars_site_model, None),
         site="South",
         array_element_name=None,
-        model_version="2024-02-01",
+        model_version="6.0.0",
         parameter_collection="site",
         db_simulation_model_url=simulation_model_url,
-        db_simulation_model="verified_model",
     )
     assert len(_pars_south) > 0
 
@@ -66,10 +63,9 @@ def test_update_parameters_from_repo(caplog, db_config, simulation_model_url):
                 parameters=dict.fromkeys(_pars_telescope_model, None),
                 site="North",
                 array_element_name="MSTN-01",
-                model_version="2024-02-01",
+                model_version="6.0.0",
                 parameter_collection="not_a_collection",
                 db_simulation_model_url=simulation_model_url,
-                db_simulation_model="verified_model",
             )
 
     # Test with a parameter that is not in the repository (no error should be raised)
@@ -78,10 +74,9 @@ def test_update_parameters_from_repo(caplog, db_config, simulation_model_url):
         parameters=dict.fromkeys(_pars_site_model, None),
         site="South",
         array_element_name=None,
-        model_version="2024-02-01",
+        model_version="6.0.0",
         parameter_collection="site",
         db_simulation_model_url=simulation_model_url,
-        db_simulation_model="verified_model",
     )
 
 
@@ -92,7 +87,7 @@ def test_update_telescope_parameters_from_repo(db_config, simulation_model_url):
         site="North",
         array_element_name="MSTN-01",
         parameter_collection="telescopes",
-        model_version="2024-02-01",
+        model_version="6.0.0",
         db_simulation_model_url=simulation_model_url,
     )
     assert len(_pars) > 0
@@ -102,7 +97,7 @@ def test_update_telescope_parameters_from_repo(db_config, simulation_model_url):
         site="North",
         array_element_name="MSTN-design",
         parameter_collection="telescopes",
-        model_version="2024-02-01",
+        model_version="6.0.0",
         db_simulation_model_url=simulation_model_url,
     )
     assert len(_pars_design) > 0
@@ -119,7 +114,7 @@ def test_update_site_parameters_from_repo(db_config):
         site="South",
         array_element_name=None,
         parameter_collection="site",
-        model_version="2024-02-01",
+        model_version="6.0.0",
         db_simulation_model_url=db_config["db_simulation_model_url"],
     )
     assert len(_pars) > 0