Merge branch 'release/0.6.2'

.gitignore

@@ -11,3 +11,5 @@ Testing/
 CMakeFiles/
 CMakeCache.txt
 *.bak
+.vim
+*.egg-info

CMakeLists.txt

@@ -7,6 +7,7 @@ project( gribjump LANGUAGES C CXX )
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 ########################################################################################################################
 ### dependencies and options
@@ -25,7 +26,7 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT)
   set(GRIBJUMP_HAVE_FDB 1)
 
   ecbuild_find_package( NAME eccodes VERSION 2.32.1 REQUIRED )
-  
+
   ### AEC
   # Override eccodes' aec with our own: we need a newer version.
   unset(AEC_INCLUDE_DIRS CACHE)
@@ -46,14 +47,21 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT)
 
 endif()
 
+# python api test are currently by default disabled because we cannot run them in the ci
+ecbuild_add_option(
+    FEATURE PYTHON_API_TESTS
+    DEFAULT OFF
+    DESCRIPTION "Will execute python tests against pygribjump / libgribjump")
+
 ########################################################################################################################
 # contents
 
-include(cmake/compiler_warnings.cmake) # optionally handle compiler specific warnings
+include(compiler_warnings) # optionally handle compiler specific warnings
+include(find_python_module)
 
 set( gribjump_INCLUDE_DIRS  ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/src )
 
-include_directories( 
+include_directories(
   ${AEC_INCLUDE_DIRS}
   ${gribjump_INCLUDE_DIRS}
   ${eckit_INCLUDE_DIRS}
@@ -70,19 +78,50 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT)
   add_subdirectory( tests )
 endif()
 
-########################################################################################################################
-# PLUGIN
-# Write and install the manifest file for the plugin
-set( gribjump_plugin_file share/plugins/gribjump-plugin.yml )
-file( WRITE  ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "plugin:\n" )
-file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  name:      gribjump-plugin\n" )
-file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  namespace: int.ecmwf\n" )
-file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  url:       http://www.ecmwf.int\n" )
-file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  version:   ${${PROJECT_NAME}_VERSION}\n" )
-file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  library:   gribjump\n" )
-
-install( FILES ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} DESTINATION share/plugins )
+if(HAVE_PYTHON_API_TESTS)
+    ecbuild_find_package( NAME Python
+                          VERSION 3.10
+                          COMPONENTS Interpreter
+                          REQUIRED )
+
+    find_python_module(cffi)
+    find_python_module(numpy)
+    find_python_module(pytest)
+    find_python_module(setuptools)
+    find_python_module(findlibs)
+
+    # Depending on if gribjumps dependencies are installed or inside a build
+    # tree locations must be made known to fdb and eccodes.
+    # If this cmake file is not the top-level cmake file we deduce that this is
+    # a bundle build and all dependencies are inside a build tree.
+    # Otherwise it is expected that all dependencies reside in an install tree
+    get_property(is_subproject DIRECTORY PROPERTY PARENT_DIRECTORY)
+    if(is_subproject)
+        set(FDB5_DIR "${CMAKE_BINARY_DIR}")
+        # This is not required if eccodes is build with ENABLE_MEMFS=ON, but in
+        # that case the environment variable is ignored.
+        set(REDEFINED_ECCODES_DEFINITION_PATH
+            "${CMAKE_BINARY_DIR}/share/eccodes/definitions")
+    else()
+        set(REDEFINED_ECCODES_DEFINITION_PATH "")
+        set(FDB5_DIR "${fdb5_DIR}/../../..")
+    endif()
+
+    configure_file(
+        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run_pytest.sh.in
+        ${CMAKE_CURRENT_BINARY_DIR}/run_pytest.sh
+        @ONLY
+    )
+
+    add_test(
+        NAME pygribjump_pytests
+        COMMAND ${CMAKE_CURRENT_BINARY_DIR}/run_pytest.sh -vv -s
+                --basetemp=${CMAKE_CURRENT_BINARY_DIR}/pytest-out
+    )
 
+endif()
+
+########################################################################################################################
 ### finalize
 
 ecbuild_install_project( NAME gribjump )

VERSION

@@ -1 +1 @@
-0.6.1
+0.6.2

cmake/find_python_module.cmake

@@ -0,0 +1,29 @@
+# Finding python modules
+# ======================
+# This function tries to import the specified python module and print its version.
+# This function will abort the configure priocess if the module cannot be imported.
+# This code depends on that the python interpreter has been found with
+# find_package(Python)
+#
+# Arguments
+# ---------
+#   python_module: The module to search for
+#
+# Example Usage
+# -------------
+# find_python_module(numpy)
+function(find_python_module python_module)
+    execute_process(
+            COMMAND
+            ${Python_EXECUTABLE} "-c" "import ${python_module}; print(${python_module}.__version__)"
+            RESULT_VARIABLE status
+            OUTPUT_VARIABLE python_module_version
+            ERROR_QUIET
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    if(NOT status)
+        message("Found Python module ${python_module} ${python_module_version}")
+    else()
+        message(FATAL_ERROR "Did not find package ${python_module}")
+    endif()
+endfunction()

cmake/run_pytest.sh.in

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+set -ex
+export GRIBJUMP_HOME="@CMAKE_BINARY_DIR@"
+export FDB5_DIR="@FDB5_DIR@"
+export PYTHONPATH="${PYTHONPATH}:@CMAKE_CURRENT_SOURCE_DIR@/pygribjump/src"
+
+eccodes_definition_path="@REDEFINED_ECCODES_DEFINITION_PATH@"
+if [ -n "${eccodes_definition_path}" ]; then
+    export ECCODES_DEFINITION_PATH=${eccodes_definition_path}
+fi
+
+@Python_EXECUTABLE@ -m pytest @CMAKE_CURRENT_SOURCE_DIR@ --basetemp="@CMAKE_CURRENT_BINARY_DIR@/test-results" "$@"
+

pygribjump/requirements.txt

@@ -0,0 +1,5 @@
+cffi~=1.17
+numpy~=2.1
+pytest~=8.3
+setuptools~=75.1
+findlibs~=0.0.5

pygribjump/pygribjump.py → pygribjump/src/pygribjump/pygribjump.py

@@ -16,7 +16,6 @@
 import numpy as np
 import cffi
 import os
-from pkg_resources import parse_version
 import findlibs
 
 ffi = cffi.FFI()
@@ -58,11 +57,8 @@ def __init__(self):
                 print("Error retrieving attribute", f, "from library")
 
         # Initialise the library, and set it up for python-appropriate behaviour
-
         self.gribjump_initialise()
 
-        # TODO <see pyfdb version checking here.>
-
     def __read_header(self, hdr_path):
         with open(hdr_path, 'r') as f:
             return f.read()

pygribjump/tests/conftest.py

@@ -0,0 +1,22 @@
+import os
+import pathlib
+
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def gribjump_env() -> None:
+    """
+    Sets default environment variables that are not dependent on individual testsetup.
+    """
+    os.environ["GRIBJUMP_IGNORE_GRID"] = "1"
+
+
+@pytest.fixture
+def data_path(request) -> pathlib.Path:
+    """
+    Provides path to test data at '<src-root>/pygribjump/tests/data'
+    """
+    path = request.config.rootpath / "pygribjump" / "tests" / "data"
+    assert path.exists()
+    return path

pygribjump/tests/data/schema

@@ -0,0 +1,30 @@
+# Default types
+
+param:      Param;
+step:       Step;
+date:       Date;
+levelist:   Double;
+grid:       Grid;
+expver:     Expver;
+time:       Time;
+number:     Integer;
+
+########################################################
+# The are the rules matching most of the fields
+# oper/dcda
+[ class, expver, stream=oper/dcda/scda, date, time, domain?
+       [ type, levtype
+               [ step, levelist?, param ]]
+]
+# enfo
+[ class, expver, stream=enfo/efov/eefo, date, time, domain
+       [ type, levtype
+               [ step, quantile?, number?, levelist?, param ]]
+]
+
+# waef/weov
+[ class, expver, stream=waef/weov/weef, date, time, domain
+       [ type, levtype
+               [ step, number?, param, frequency?, direction? ]]
+]
+

pygribjump/tests/test_pygribjump.py

@@ -0,0 +1,85 @@
+"""
+NOTE:
+
+To be able to run the tests pyfdb and pygribjump need to find the native libraries.
+If not installed into a default location export the following environent
+variables in the shell running pytest:
+    GRIBJUMP_HOME = <cmake-build-root-for-gribjump> or <custom-install-location>
+    FDB5_DIR = <cmake-build-root-for-fdb> or <custom-install-location>
+"""
+
+import os
+import pathlib
+import shutil
+
+import numpy
+import pytest
+import yaml
+
+# Do not reorder the imports otherwise this will trigger an assertion in eckits
+# library registration handling.
+import pygribjump as pygj
+import pyfdb
+
+
+@pytest.fixture(scope="function")
+def read_only_fdb_setup(data_path, tmp_path) -> pathlib.Path:
+    """
+    Creates a FDB setup in this tests temp directory.
+    Test FDB currently reads all grib files in `tests/data`
+    """
+    db_store_path = tmp_path / "db_store"
+    db_store_path.mkdir(exist_ok=True)
+    schema_path = tmp_path / "schema"
+    config = dict(
+        type="local",
+        engine="toc",
+        schema=str(schema_path),
+        spaces=[
+            dict(
+                handler="Default",
+                roots=[
+                    {"path": str(db_store_path)},
+                ],
+            )
+        ],
+    )
+    config_path = tmp_path / "config.yaml"
+    config_path.write_text(yaml.dump(config))
+    shutil.copy(data_path / "schema", schema_path)
+    os.environ["FDB5_CONFIG_FILE"] = str(config_path)
+
+    # grib_files = data_path.glob("*.grib")
+    grib_files = [data_path / "synth11.grib"]
+    fdb = pyfdb.FDB()
+    for f in grib_files:
+        fdb.archive(f.read_bytes())
+    return tmp_path
+
+
+def test_extract_simple_sunshine_case(read_only_fdb_setup) -> None:
+    grib_jump = pygj.GribJump()
+    reqstrs = [
+        {
+            "domain": "g",
+            "levtype": "sfc",
+            "date": "20230508",
+            "time": "1200",
+            "step": "1",
+            "param": "151130",
+            "class": "od",
+            "type": "fc",
+            "stream": "oper",
+            "expver": "0001",
+        },
+    ]
+    ranges = [
+        [(0, 6)],
+    ]
+
+    polyrequest = [(a, b) for a, b in zip(reqstrs, ranges)]
+
+    expected = numpy.array([0.0, numpy.nan, numpy.nan, 3.0, 4.0, numpy.nan])
+
+    actual = grib_jump.extract(polyrequest)
+    assert numpy.array_equal(expected, actual[0][0][0][0], equal_nan=True)

pyproject.toml

@@ -0,0 +1,6 @@
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "-vv -s"
+testpaths = [
+    "pygribjump/tests"
+]

setup.py

@@ -3,9 +3,8 @@
 
 from setuptools import find_packages, setup
 
-__version__ = re.search(
-    r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', io.open("pygribjump/version.py", encoding="utf_8_sig").read()
-).group(1)
+with open("VERSION", "r") as f:
+    __version__ = f.read().strip()
 
 setup(
     name="pygribjump",

src/gribjump/CMakeLists.txt

@@ -154,3 +154,26 @@ ecbuild_add_library(
         eckit_option
         ${SERVER_LIBS}
 )
+
+############################################################################################
+# Plugin
+# Must happen atomically, and only after the library has been built.
+# NB: In future this will be added to the ecbuild_add_library call above
+
+set( gribjump_plugin_file_in gribjump-plugin.yml.in )
+set( gribjump_plugin_file_out ${CMAKE_BINARY_DIR}/share/plugins/int.ecmwf.gribjump-plugin.yml )
+set( gribjump_plugin_file_tmp gribjump-plugin.yml.tmp)
+
+# configure to temporary file
+configure_file( ${gribjump_plugin_file_in} ${gribjump_plugin_file_tmp} @ONLY )
+
+# move file atomically
+add_custom_command( TARGET gribjump POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/share/plugins
+    COMMAND ${CMAKE_COMMAND} -E copy ${gribjump_plugin_file_tmp} ${gribjump_plugin_file_out}
+    BYPRODUCTS ${gribjump_plugin_file_out}
+)
+
+install( FILES ${gribjump_plugin_file_out} DESTINATION share/plugins)
+
+############################################################################################