Merge pull request #502 from iiasa/enh/gams-version

Record Python package versions in GDX files

README.md

@@ -45,7 +45,7 @@ See [`doc/README.rst`](doc/README.rst) for further details.
 
 ## License
 
-Copyright © 2017–2023 IIASA Energy, Climate, and Environment (ECE) program
+Copyright © 2017–2024 IIASA Energy, Climate, and Environment (ECE) program
 
 `ixmp` is licensed under the Apache License, Version 2.0 (the "License"); you
 may not use the files in this repository except in compliance with the License.

RELEASE_NOTES.rst

@@ -17,6 +17,8 @@ All changes
 - :mod:`ixmp` is tested and compatible with `Python 3.12 <https://www.python.org/downloads/release/python-3120/>`__ (:pull:`504`).
 - Support for Python 3.7 is dropped (:pull:`492`), as it has reached end-of-life.
 - Rename :mod:`ixmp.report` and :mod:`ixmp.util` (:pull:`500`).
+- New option :py:`record_version_packages` to :class:`.GAMSModel` (:pull:`502`).
+  Versions of the named Python packages are recorded in a special set in GDX-format input and output files to help associate these files with generating code.
 - New reporting operators :func:`.from_url`, :func:`.get_ts`, and :func:`.remove_ts` (:pull:`500`).
 - New CLI command :program:`ixmp platform copy` and :doc:`CLI documentation <cli>` (:pull:`500`).
 - New argument :py:`indexed_by=...` to :meth:`.Scenario.items` (thus :meth:`.Scenario.par_list` and similar methods) to iterate over (or list) only items that are indexed by a particular set (:issue:`402`, :pull:`500`).

doc/conf.py

@@ -3,13 +3,14 @@
 # This file only contains a selection of the most common options. For a full list see
 # the documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html
 
-# Import so that autodoc can find code
-import ixmp
+import importlib.metadata
+from pathlib import Path
+from typing import Optional
 
 # -- Project information ---------------------------------------------------------------
 
 project = "ixmp"
-copyright = "2017–2023, IIASA Energy, Climate, and Environment (ECE) program"
+copyright = "2017–2024, IIASA Energy, Climate, and Environment (ECE) program"
 author = "ixmp Developers"
 
 
@@ -44,7 +45,7 @@
 
 # A string of reStructuredText that will be included at the beginning of every source
 # file that is read.
-version = ixmp.__version__
+version = importlib.metadata.version("ixmp")
 rst_prolog = rf"""
 .. role:: py(code)
    :language: python
@@ -85,9 +86,25 @@
 
 # -- Options for sphinx.ext.intersphinx ------------------------------------------------
 
+
+def local_inv(name: str, *parts: str) -> Optional[str]:
+    """Construct the path to a local intersphinx inventory."""
+
+    from importlib.util import find_spec
+
+    spec = find_spec(name)
+    if spec is None:
+        return None
+
+    if 0 == len(parts):
+        parts = ("doc", "_build", "html")
+    assert spec.origin is not None
+    return str(Path(spec.origin).parents[1].joinpath(*parts, "objects.inv"))
+
+
 intersphinx_mapping = {
     "dask": ("https://docs.dask.org/en/stable/", None),
-    "genno": ("https://genno.readthedocs.io/en/latest/", None),
+    "genno": ("https://genno.readthedocs.io/en/latest/", (local_inv("genno"), None)),
     "jpype": ("https://jpype.readthedocs.io/en/latest", None),
     "message_ix": ("https://docs.messageix.org/en/latest/", None),
     "message-ix-models": (

doc/reporting.rst

@@ -55,7 +55,7 @@ The following top-level objects from :mod:`genno` may also be imported from
       from_scenario
       set_filters
 
-   The Computer class provides the following methods:
+   The Reporter class inherits from Computer the following methods:
 
    .. currentmodule:: genno
    .. autosummary::
@@ -74,7 +74,7 @@ The following top-level objects from :mod:`genno` may also be imported from
       ~Computer.visualize
       ~Computer.write
 
-   The following methods are deprecated; equivalent or better functionality is available through :meth:`.Computer.add`.
+   The following methods are deprecated; equivalent or better functionality is available through :meth:`Reporter.add <genno.Computer.add>`.
    See the genno documentation for each method for suggested changes/migrations.
 
    .. autosummary::
@@ -132,6 +132,10 @@ Configuration
    The only difference from :func:`genno.config.units` is that this handler keeps the configuration values stored in ``Reporter.graph["config"]``.
    This is so that :func:`.data_for_quantity` can make use of ``["units"]["apply"]``
 
+.. automethod:: ixmp.report.configure
+
+   This is the same as :func:`genno.configure`.
+
 
 Operators
 =========
@@ -186,7 +190,7 @@ Utilities
 .. autodata:: RENAME_DIMS
 
    User code **should** avoid directly manipulating :data:`RENAME_DIMS`.
-   Instead, call :func:`.configure`:
+   Instead, call :func:`~genno.configure`:
 
    .. code-block:: python
 

ixmp/backend/jdbc.py

@@ -604,7 +604,7 @@ def write_file(self, path, item_type: ItemType, **kwargs):
 
         ts, filters = self._handle_rw_filters(kwargs.pop("filters", {}))
         if path.suffix == ".gdx" and item_type is ItemType.SET | ItemType.PAR:
-            if len(filters):
+            if len(filters):  # pragma: no cover
                 raise NotImplementedError("write to GDX with filters")
             elif not isinstance(ts, Scenario):  # pragma: no cover
                 raise ValueError("write to GDX requires a Scenario object")
@@ -952,7 +952,13 @@ def init_item(self, s, type, name, idx_sets, idx_names):
                 _raise_jexception(e)
 
     def delete_item(self, s, type, name):
-        getattr(self.jindex[s], f"remove{type.title()}")(name)
+        try:
+            getattr(self.jindex[s], f"remove{type.title()}")(name)
+        except jpype.JException as e:
+            if "There exists no" in e.args[0]:
+                raise KeyError(name)
+            else:  # pragma: no cover
+                _raise_jexception(e)
         self.cache_invalidate(s, type, name)
 
     def item_index(self, s, name, sets_or_names):

ixmp/model/dantzig.gms

@@ -23,6 +23,8 @@ Sets
     j   markets
 ;
 
+Set ixmp_version(*,*) "Versions of Python packages";
+
 Parameters
     a(i)   capacity of plant i in cases
     b(j)   demand at market j in cases
@@ -45,6 +47,7 @@ $IF NOT set out $SETGLOBAL out 'ix_transport_results.gdx'
 
 $GDXIN '%in%'
 $LOAD i, j, a, b, d, f
+$LOAD ixmp_version
 $GDXIN
 
 Parameter c(i,j)  transport cost in thousands of dollars per case ;

ixmp/model/gams.py

@@ -6,7 +6,7 @@
 from copy import copy
 from pathlib import Path
 from subprocess import CalledProcessError, run
-from typing import Any, MutableMapping
+from typing import Any, MutableMapping, Optional
 
 from ixmp.backend import ItemType
 from ixmp.model.base import Model, ModelError
@@ -15,8 +15,8 @@
 log = logging.getLogger(__name__)
 
 
-def gams_version():
-    """Return the GAMS version as a string, e.g. '24.7.4'."""
+def gams_version() -> Optional[str]:
+    """Return the GAMS version as a string, for instance "24.7.4"."""
     # NB check_output(['gams'], ...) does not work, because GAMS writes directly to the
     #    console instead of to stdout. check_output(['gams', '-LogOption=3'], ...) does
     #    not work, because GAMS does not accept options without an input file to
@@ -44,8 +44,10 @@ def gams_version():
     tmp_dir.rmdir()
 
     # Find and return the version string
-    pattern = r"^GAMS ([\d\.]+)\s*Copyright"
-    return re.search(pattern, output, re.MULTILINE).groups()[0]
+    if match := re.search(r"^GAMS ([\d\.]+)\s*Copyright", output, re.MULTILINE):
+        return match.group(0)
+    else:  # pragma: no cover
+        return None
 
 
 #: Return codes used by GAMS, from
@@ -93,28 +95,26 @@ def gams_version():
 
 
 class GAMSModel(Model):
-    """General class for ixmp models using `GAMS <https://gams.com/>`_.
+    """Generic base class for :mod:`ixmp` models using `GAMS <https://gams.com>`_.
 
-    GAMSModel solves a Scenario using the following steps:
+    GAMSModel solves a :class:`.Scenario` using the following steps:
 
     1. All Scenario data is written to a model input file in GDX format.
-    2. A GAMS program is run to perform calculations, producing output in a
-       GDX file.
-    3. Output, or solution, data is read from the GDX file and stored in the
-       Scenario.
+    2. A GAMS program is run to perform calculations, producing output in a GDX file.
+    3. Output, or solution, data is read from the GDX file and stored in the Scenario.
 
-    When created and :meth:`run`, GAMSModel constructs file paths and other
-    necessary values using format strings. The :attr:`defaults` may be
-    overridden by the keyword arguments to the constructor:
+    When created and :meth:`run`, GAMSModel constructs file paths and other necessary
+    values using format strings. The :attr:`defaults` may be overridden by the keyword
+    arguments to the constructor:
 
     Other parameters
     ----------------
     name : str, optional
-        Override the :attr:`name` attribute to provide the `model_name` for
-        format strings.
+        Override the :attr:`name` attribute to provide the `model_name` for format
+        strings.
     model_file : str, optional
-        Path to GAMS file, including '.gms' extension. Default: ``'{model_name}.gms'``
-        in the current directory.
+        Path to GAMS file, including :file:`.gms` extension. Default:
+        :file:`{model_name}.gms` in the current directory.
     case : str, optional
         Run or case identifier to use in GDX file names. Default:
         ``'{scenario.model}_{scenario.name}'``, where `scenario` is the
@@ -138,12 +138,12 @@ class GAMSModel(Model):
         control solver options or behaviour. See the `GAMS Documentation <https://www.gams.com/latest/docs/UG_GamsCall.html#UG_GamsCall_ListOfCommandLineParameters>`_.
         For example:
 
-        - ``["iterLim=10"]`` limits the solver to 10 iterations.
+        - :py:`gams_args=["iterLim=10"]` limits the solver to 10 iterations.
     quiet: bool, optional
-        If :obj:`True`, add "LogOption=2" to `gams_args` to redirect most console
-        output during the model run to the log file. Default :obj:`False`, so
-        "LogOption=4" is added. Any "LogOption" value provided explicitly via
-        `gams_args` takes precedence.
+        If :obj:`True`, add "LogOption=2" to `gams_args` to redirect most console output
+        during the model run to the log file. Default :obj:`False`, so "LogOption=4" is
+        added. Any "LogOption" value provided explicitly via `gams_args` takes
+        precedence.
     check_solution : bool, optional
         If :obj:`True`, raise an exception if the GAMS solver did not reach optimality.
         (Only for MESSAGE-scheme Scenarios.)
@@ -154,6 +154,9 @@ class GAMSModel(Model):
         Equations to be imported from the `out_file`. Default: all.
     var_list : list of str, optional
         Variables to be imported from the `out_file`. Default: all.
+    record_version_packages : list of str, optional
+        Names of Python packages to record versions. Default: :py:`["ixmp"]`.
+        See :meth:`record_versions`.
     """  # noqa: E501
 
     #: Model name.
@@ -174,6 +177,7 @@ class GAMSModel(Model):
         "var_list": None,
         "quiet": False,
         "use_temp_dir": True,
+        "record_version_packages": ["ixmp"],
     }
 
     def __init__(self, name_=None, **model_options):
@@ -188,23 +192,27 @@ def __init__(self, name_=None, **model_options):
             # Not set; use `quiet` to determine the value
             self.gams_args.append(f"LogOption={'2' if self.quiet else '4'}")
 
-    def format_exception(self, exc, model_file, backend_class):
+    def format_exception(
+        self, exc: Exception, model_file: Path, backend_class: type
+    ) -> Exception:
         """Format a user-friendly exception when GAMS errors."""
         lst_file = Path(self.cwd).joinpath(model_file.name).with_suffix(".lst")
         lp_5 = "LP status (5): optimal with unscaled infeasibilities"
 
-        if getattr(exc, "returncode", 0) > 0:
+        if rc := getattr(exc, "returncode", 0):
             # Convert a Windows return code >256 to its POSIX equivalent
             msg = [
-                f"GAMS errored with return code {exc.returncode}:",
-                f"    {RETURN_CODE[exc.returncode % 256]}",
+                f"GAMS errored with return code {rc}:",
+                f"    {RETURN_CODE[rc % 256]}",
             ]
         elif lst_file.exists() and lp_5 in lst_file.read_text():  # pragma: no cover
             msg = [
                 "GAMS returned 0 but indicated:",
                 f"    {lp_5}",
                 f"and {backend_class.__name__} could not read the solution.",
             ]
+        else:  # pragma: no cover
+            return exc  # Other exception
 
         # Add a reference to the listing file, if it exists
         msg.extend(
@@ -230,6 +238,37 @@ def format(self, value):
             # Something like a Path; don't format it
             return value
 
+    def record_versions(self):
+        """Store Python package versions as set elements to be written to GDX.
+
+        The values are stored in a 2-dimensional set named ``ixmp_version``, where the
+        first element is the package name, and the second is its version according to
+        :func:`importlib.metadata.version`). If the package is not installed, the
+        string "(not installed)" is stored.
+        """
+        from importlib.metadata import PackageNotFoundError, version
+
+        name = "ixmp_version"
+        with self.scenario.transact():
+            try:
+                # Initialize the set
+                self.scenario.init_set(name, ["*", "*"], ["package", "version"])
+            except ValueError as e:  # pragma: no cover
+                # NB this will only occur if the user directly creates the set; the one
+                #    created here is deleted in run()
+                if "already exists" not in e.args[0]:
+                    raise
+
+            # Handle each identified package
+            for package in self.record_version_packages:
+                try:
+                    # Retrieve the version; replace characters not supported by GAMS
+                    package_version = version(package).replace(".", "-")
+                except PackageNotFoundError:
+                    package_version = "(not installed)"  # Not installed
+                # Add to the set
+                self.scenario.add_set(name, (package, package_version))
+
     def remove_temp_dir(self, msg="after run()"):
         """Remove the temporary directory, if any."""
         try:
@@ -247,10 +286,24 @@ def __del__(self):
         self.remove_temp_dir("at GAMSModel teardown")
 
     def run(self, scenario):
-        """Execute the model."""
+        """Execute the model.
+
+        Among other steps:
+
+        - :meth:`record_versions` is called.
+        - Data is written to a GDX file using the associated :class:`.Backend`.
+        - The ``ixmp_version`` set created by :meth:`record_versions` is deleted.
+        - :program:`gams` is invoked to execute the model file.
+        - If :program:`gams` completes successfully:
+
+          - GAMS output/model solution data is read from a GDX file.
+        """
         # Store the scenario so its attributes can be referenced by format()
         self.scenario = scenario
 
+        # Record versions of packages listed in `record_version_packages`
+        self.record_versions()
+
         # Format or retrieve the model file option
         model_file = Path(self.format_option("model_file"))
 
@@ -296,6 +349,10 @@ def run(self, scenario):
                 "GAMSModel requires a Backend that can write to GDX files, e.g. "
                 "JDBCBackend"
             )
+        else:
+            # Remove ixmp_version set entirely
+            with scenario.transact():
+                scenario.remove_set("ixmp_version")
 
         try:
             # Invoke GAMS

ixmp/report/operator.py

@@ -129,7 +129,7 @@ def data_for_quantity(
         # First rename, then set index
         data = data.rename(columns=common.RENAME_DIMS).set_index(dims)
 
-    # Convert to a Quantity, assign attrbutes and name
+    # Convert to a Quantity, assign attributes and name
     qty = Quantity(
         data[column], name=name + ("-margin" if column == "mrg" else ""), attrs=attrs
     )

ixmp/tests/backend/test_jdbc.py

@@ -172,6 +172,11 @@ def test_init_item(self, mp, be, name, idx_sets):
         # Initialize the item; succeeds
         be.init_item(s, type="var", name=name, idx_sets=idx_sets, idx_names=idx_sets)
 
+    def test_delete_item(self, mp, be):
+        s = ixmp.Scenario(mp, "model name", "scenario name", version="new")
+        with pytest.raises(KeyError):
+            be.delete_item(s, "set", "foo")
+
     def test_set_data_inf(self, mp):
         """:meth:`JDBCBackend.set_data` errors on :data:`numpy.inf` values."""
         # Make `mp` think it is connected to an Oracle database