mapping loading with mapping type specified or different method name (#…

…344)

* mapping loading with mapping type specified or different method name

* fix mapping paths

* add missing mapping files and updated path to config file in test_bluebrain_nexus

* reset notebooks

* mapping tests

* change back load file to load when using mapping in store

* added test cases for mapping type specific methods

* improve tests

* hjson import change

* little change

---------

Co-authored-by: mouffok <[email protected]>

kgforge/core/archetypes/mapping.py

@@ -15,13 +15,20 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any
+from enum import Enum
 
 import requests
 from requests import RequestException
 
 from kgforge.core.commons.attributes import repr_class
 
 
+class MappingType(Enum):
+    URL = "url"
+    FILE = "file"
+    STR = "str"
+
+
 class Mapping(ABC):
 
     # See dictionaries.py in kgforge/specializations/mappings/ for a reference implementation.
@@ -49,20 +56,56 @@ def __eq__(self, other: object) -> bool:
         raise NotImplementedError
 
     @classmethod
-    def load(cls, source: str):
+    def load(cls, source: str, mapping_type: MappingType = None):
         # source: Union[str, FilePath, URL].
         # Mappings could be loaded from a string, a file, or an URL.
-        filepath = Path(source)
-        if filepath.is_file():
-            text = filepath.read_text()
+
+        if mapping_type is None:
+            e = cls.load_file(source, raise_ex=False)
+            e = e if e is not None else cls.load_url(source, raise_ex=False)
+            e = e if e is not None else cls.load_str(source, raise_ex=False)
+            if e is not None:
+                return e
+            raise Exception("Mapping loading failed")
+
+        if mapping_type == MappingType.FILE:
+            return cls.load_file(source)
+        elif mapping_type == MappingType.URL:
+            return cls.load_url(source)
+        elif mapping_type == MappingType.STR:
+            return cls.load_str(source)
         else:
-            try:
-                response = requests.get(source)
-                response.raise_for_status()
-                text = response.text
-            except RequestException:
-                text = source
-        return cls(text)
+            raise NotImplementedError
+
+    @classmethod
+    def load_file(cls, filepath, raise_ex=True):
+        try:
+            filepath = Path(filepath)
+
+            if filepath.is_file():
+                return cls(filepath.read_text())
+            else:
+                raise OSError
+        except OSError:
+            if raise_ex:
+                raise FileNotFoundError
+            return None
+
+    @classmethod
+    def load_url(cls, url, raise_ex=True):
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            return cls(response.text)
+        except RequestException as e:
+            if raise_ex:
+                raise e
+            return None
+
+    @classmethod
+    @abstractmethod
+    def load_str(cls, source: str, raise_ex=True):
+        ...
 
     def save(self, path: str) -> None:
         # path: FilePath.

kgforge/core/archetypes/store.py

@@ -17,11 +17,11 @@
 import time
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Match, Optional, Tuple, Union
+from typing import Any, Dict, List, Match, Optional, Tuple, Union, Type
 
 from kgforge.core import Resource
+from kgforge.core.archetypes import Mapping, Mapper
 from kgforge.core.commons.attributes import repr_class
-from kgforge.specializations.mappers import DictionaryMapper
 from kgforge.core.commons.context import Context
 from kgforge.core.commons.exceptions import (
     DeprecationError,
@@ -142,14 +142,16 @@ def __repr__(self) -> str:
         return repr_class(self)
 
     @property
-    def mapping(self) -> Optional[Callable]:
+    @abstractmethod
+    def mapping(self) -> Type[Mapping]:
         """Mapping class to load file_resource_mapping."""
-        return None
+        ...
 
     @property
-    def mapper(self) -> Optional[Callable]:
+    @abstractmethod
+    def mapper(self) -> Type[Mapper]:
         """Mapper class to map file metadata to a Resource with file_resource_mapping."""
-        return None
+        ...
 
     # [C]RUD.
 

kgforge/specializations/mappings/dictionaries.py

@@ -32,3 +32,14 @@ def _load_rules(mapping: str) -> OrderedDict:
     @staticmethod
     def _normalize_rules(rules: OrderedDict) -> str:
         return hjson.dumps(rules, indent=4, item_sort_key=sort_attrs)
+
+    @classmethod
+    def load_str(cls, source: str, raise_ex=True):
+        # hjson loading doesn't make one line strings (non dictionaries) fail
+        if len(source.strip()) > 0 and source.strip()[0] != "{":
+            if raise_ex:
+                raise hjson.scanner.HjsonDecodeError(
+                    f"Invalid hjson mapping", doc=source, pos=0
+                )
+            return None
+        return cls(source)

kgforge/specializations/stores/demo_store.py

@@ -13,11 +13,11 @@
 # along with Blue Brain Nexus Forge. If not, see <https://choosealicense.com/licenses/lgpl-3.0/>.
 
 from copy import deepcopy
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Type
 from uuid import uuid4
 
 from kgforge.core import Resource
-from kgforge.core.archetypes import Resolver, Store
+from kgforge.core.archetypes import Resolver, Store, Mapper, Mapping
 from kgforge.core.commons.context import Context
 from kgforge.core.commons.exceptions import (DeprecationError, RegistrationError,
                                              RetrievalError, TaggingError, UpdatingError)
@@ -37,6 +37,17 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None,
         super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping,
                          model_context)
 
+    @property
+    def mapping(self) -> Type[Mapping]:
+        """Mapping class to load file_resource_mapping."""
+        return None
+
+    @property
+    def mapper(self) -> Type[Mapper]:
+        """Mapper class to map file metadata to a Resource with file_resource_mapping."""
+        return None
+
+
     # [C]RUD.
 
     def _register_one(self, resource: Resource, schema_id: str) -> None:

tests/conftest.py

@@ -12,10 +12,9 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with Blue Brain Nexus Forge. If not, see <https://choosealicense.com/licenses/lgpl-3.0/>.
 
-import os
 from typing import Callable, List, Union, Dict, Optional
 from uuid import uuid4
-
+from utils import full_path_relative_to_root
 import pytest
 from pytest_bdd import given, parsers, then, when
 
@@ -42,7 +41,6 @@ def do(fun: Callable, data: Union[Resource, List[Resource]], *args) -> None:
 
 # Resource(s) creation.
 
-
 @pytest.fixture
 def json_one():
     return {"id": "123", "type": "Type", "p1": "v1a", "p2": "v2a"}
@@ -173,7 +171,7 @@ def forge():
         "Model": {
             "name": "DemoModel",
             "origin": "directory",
-            "source": "tests/data/demo-model/",
+            "source": full_path_relative_to_root("tests/data/demo-model/")
         },
         "Store": {
             "name": "DemoStore",
@@ -312,7 +310,7 @@ def building():
 
 @pytest.fixture(scope="session")
 def context_file_path():
-    return os.sep.join((os.path.abspath("."), "tests/data/shacl-model/context.json"))
+    return full_path_relative_to_root("tests/data/shacl-model/context.json")
 
 
 @pytest.fixture(scope="session")
@@ -465,7 +463,7 @@ def config(model, store, resolver):
         "Model": {
             "name": model,
             "origin": "directory",
-            "source": "tests/data/demo-model/",
+            "source": full_path_relative_to_root("tests/data/demo-model/"),
         },
         "Store": {
             "name": store,
@@ -476,7 +474,7 @@ def config(model, store, resolver):
                 {
                     "resolver": resolver,
                     "origin": "directory",
-                    "source": "tests/data/demo-resolver/",
+                    "source": full_path_relative_to_root("tests/data/demo-resolver/"),
                     "targets": [
                         {
                             "identifier": "sex",
@@ -493,7 +491,9 @@ def config(model, store, resolver):
                             ]
                         },
                     ],
-                    "result_resource_mapping": "../../configurations/demo-resolver/term-to-resource-mapping.hjson",
+                    "result_resource_mapping": full_path_relative_to_root(
+                        "./examples/configurations/demo-resolver/term-to-resource-mapping.hjson"
+                    )
                 },
             ],
         },

tests/data/nexus-store/file-to-resource-mapping.hjson

@@ -0,0 +1,27 @@
+{
+    type: DataDownload
+    contentSize:
+    {
+        unitCode: f"bytes"
+        value: x._bytes
+    }
+    digest:
+    {
+        algorithm: x._digest._algorithm
+        value: x._digest._value
+    }
+    encodingFormat: x._mediaType
+    name: x._filename
+    contentUrl: forge.format(uri=x['@id'], formatter='URI_REWRITER', is_file=True)
+    atLocation:
+    {
+        type: Location
+        store: 
+        {
+            id: x._storage["@id"]
+            type: x._storage["@type"] if '@type' in x._storage else None
+            _rev: x._storage["_rev"] if '_rev' in x._storage else None
+        }
+        location: x._location if '_location' in x else None
+    }
+}

tests/data/nexus-store/publishing-file-to-resource-mapping.hjson

@@ -0,0 +1,16 @@
+{
+    type: DataDownload
+    contentSize:
+    {
+        unitCode: f"bytes"
+        value: x._bytes
+    }
+    digest:
+    {
+        algorithm: x._digest._algorithm
+        value: x._digest._value
+    }
+    encodingFormat: x._mediaType
+    name: x._filename
+    contentUrl: x._self
+}

tests/specializations/mappers/test_mappers.py

@@ -13,3 +13,47 @@
 # along with Blue Brain Nexus Forge. If not, see <https://choosealicense.com/licenses/lgpl-3.0/>.
 
 # Placeholder for the generic parameterizable test suite for mappers.
+
+import pytest
+from contextlib import nullcontext as does_not_raise
+
+from kgforge.core import KnowledgeGraphForge, Resource
+from kgforge.specializations.mappers import DictionaryMapper
+from kgforge.specializations.mappings import DictionaryMapping
+
+
+@pytest.fixture
+def mapping_str():
+    return """
+    {
+        type: x.type
+        id: x.id
+        content_type: {
+            unitCode: f"bytes"
+            value: x.p1
+        }
+        encodingFormat: x.p2
+    }
+    """
+
+
+@pytest.mark.parametrize(
+    "json_to_map, exception",
+    [
+        ({"id": "123", "type": "Type", "p1": "v1a", "p2": "v2a"}, does_not_raise()),
+        ({"id": "123", "p1": "v1a", "p2": "v2a"}, pytest.raises(AttributeError)),
+
+    ],
+)
+def test_mapping_load_no_mapping_type(
+        exception, config, json_to_map, mapping_str
+):
+
+    forge = KnowledgeGraphForge(config)
+
+    mapping = DictionaryMapping.load(mapping_str)
+
+    with exception:
+        DictionaryMapper(forge).map(json_to_map, mapping, None)
+
+