From f51c9ec312063b401a0877a8ea114e7a97954702 Mon Sep 17 00:00:00 2001 From: Sarah Date: Thu, 19 Oct 2023 13:44:44 +0200 Subject: [PATCH] mapping loading with mapping type specified or different method name (#344) * mapping loading with mapping type specified or different method name * fix mapping paths * add missing mapping files and updated path to config file in test_bluebrain_nexus * reset notebooks * mapping tests * change back load file to load when using mapping in store * added test cases for mapping type specific methods * improve tests * hjson import change * little change --------- Co-authored-by: mouffok --- kgforge/core/archetypes/mapping.py | 65 +++++++++-- kgforge/core/archetypes/store.py | 14 ++- .../specializations/mappings/dictionaries.py | 11 ++ kgforge/specializations/stores/demo_store.py | 15 ++- tests/conftest.py | 16 +-- .../file-to-resource-mapping.hjson | 27 +++++ .../publishing-file-to-resource-mapping.hjson | 16 +++ tests/specializations/mappers/test_mappers.py | 44 +++++++ .../specializations/mappings/test_mappings.py | 109 ++++++++++++++++++ .../specializations/models/test_demo_model.py | 4 +- .../specializations/models/test_rdf_model.py | 3 +- .../stores/test_bluebrain_nexus.py | 6 +- utils.py | 19 ++- 13 files changed, 311 insertions(+), 38 deletions(-) create mode 100644 tests/data/nexus-store/file-to-resource-mapping.hjson create mode 100644 tests/data/nexus-store/publishing-file-to-resource-mapping.hjson diff --git a/kgforge/core/archetypes/mapping.py b/kgforge/core/archetypes/mapping.py index af67e626..172bfe5f 100644 --- a/kgforge/core/archetypes/mapping.py +++ b/kgforge/core/archetypes/mapping.py @@ -15,6 +15,7 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Any +from enum import Enum import requests from requests import RequestException @@ -22,6 +23,12 @@ from kgforge.core.commons.attributes import repr_class +class MappingType(Enum): + URL = "url" + FILE = "file" + STR = "str" + + class Mapping(ABC): # See dictionaries.py in kgforge/specializations/mappings/ for a reference implementation. @@ -49,20 +56,56 @@ def __eq__(self, other: object) -> bool: raise NotImplementedError @classmethod - def load(cls, source: str): + def load(cls, source: str, mapping_type: MappingType = None): # source: Union[str, FilePath, URL]. # Mappings could be loaded from a string, a file, or an URL. - filepath = Path(source) - if filepath.is_file(): - text = filepath.read_text() + + if mapping_type is None: + e = cls.load_file(source, raise_ex=False) + e = e if e is not None else cls.load_url(source, raise_ex=False) + e = e if e is not None else cls.load_str(source, raise_ex=False) + if e is not None: + return e + raise Exception("Mapping loading failed") + + if mapping_type == MappingType.FILE: + return cls.load_file(source) + elif mapping_type == MappingType.URL: + return cls.load_url(source) + elif mapping_type == MappingType.STR: + return cls.load_str(source) else: - try: - response = requests.get(source) - response.raise_for_status() - text = response.text - except RequestException: - text = source - return cls(text) + raise NotImplementedError + + @classmethod + def load_file(cls, filepath, raise_ex=True): + try: + filepath = Path(filepath) + + if filepath.is_file(): + return cls(filepath.read_text()) + else: + raise OSError + except OSError: + if raise_ex: + raise FileNotFoundError + return None + + @classmethod + def load_url(cls, url, raise_ex=True): + try: + response = requests.get(url) + response.raise_for_status() + return cls(response.text) + except RequestException as e: + if raise_ex: + raise e + return None + + @classmethod + @abstractmethod + def load_str(cls, source: str, raise_ex=True): + ... def save(self, path: str) -> None: # path: FilePath. diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index 0af43410..08b3e3fe 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -17,11 +17,11 @@ import time from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Callable, Dict, List, Match, Optional, Tuple, Union +from typing import Any, Dict, List, Match, Optional, Tuple, Union, Type from kgforge.core import Resource +from kgforge.core.archetypes import Mapping, Mapper from kgforge.core.commons.attributes import repr_class -from kgforge.specializations.mappers import DictionaryMapper from kgforge.core.commons.context import Context from kgforge.core.commons.exceptions import ( DeprecationError, @@ -142,14 +142,16 @@ def __repr__(self) -> str: return repr_class(self) @property - def mapping(self) -> Optional[Callable]: + @abstractmethod + def mapping(self) -> Type[Mapping]: """Mapping class to load file_resource_mapping.""" - return None + ... @property - def mapper(self) -> Optional[Callable]: + @abstractmethod + def mapper(self) -> Type[Mapper]: """Mapper class to map file metadata to a Resource with file_resource_mapping.""" - return None + ... # [C]RUD. diff --git a/kgforge/specializations/mappings/dictionaries.py b/kgforge/specializations/mappings/dictionaries.py index 8d2f823d..27fa2ffa 100644 --- a/kgforge/specializations/mappings/dictionaries.py +++ b/kgforge/specializations/mappings/dictionaries.py @@ -32,3 +32,14 @@ def _load_rules(mapping: str) -> OrderedDict: @staticmethod def _normalize_rules(rules: OrderedDict) -> str: return hjson.dumps(rules, indent=4, item_sort_key=sort_attrs) + + @classmethod + def load_str(cls, source: str, raise_ex=True): + # hjson loading doesn't make one line strings (non dictionaries) fail + if len(source.strip()) > 0 and source.strip()[0] != "{": + if raise_ex: + raise hjson.scanner.HjsonDecodeError( + f"Invalid hjson mapping", doc=source, pos=0 + ) + return None + return cls(source) diff --git a/kgforge/specializations/stores/demo_store.py b/kgforge/specializations/stores/demo_store.py index b453ce18..f1c944dc 100644 --- a/kgforge/specializations/stores/demo_store.py +++ b/kgforge/specializations/stores/demo_store.py @@ -13,11 +13,11 @@ # along with Blue Brain Nexus Forge. If not, see . from copy import deepcopy -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Union, Type from uuid import uuid4 from kgforge.core import Resource -from kgforge.core.archetypes import Resolver, Store +from kgforge.core.archetypes import Resolver, Store, Mapper, Mapping from kgforge.core.commons.context import Context from kgforge.core.commons.exceptions import (DeprecationError, RegistrationError, RetrievalError, TaggingError, UpdatingError) @@ -37,6 +37,17 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, model_context) + @property + def mapping(self) -> Type[Mapping]: + """Mapping class to load file_resource_mapping.""" + return None + + @property + def mapper(self) -> Type[Mapper]: + """Mapper class to map file metadata to a Resource with file_resource_mapping.""" + return None + + # [C]RUD. def _register_one(self, resource: Resource, schema_id: str) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index a61b4a5e..99122160 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,10 +12,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . -import os from typing import Callable, List, Union, Dict, Optional from uuid import uuid4 - +from utils import full_path_relative_to_root import pytest from pytest_bdd import given, parsers, then, when @@ -42,7 +41,6 @@ def do(fun: Callable, data: Union[Resource, List[Resource]], *args) -> None: # Resource(s) creation. - @pytest.fixture def json_one(): return {"id": "123", "type": "Type", "p1": "v1a", "p2": "v2a"} @@ -173,7 +171,7 @@ def forge(): "Model": { "name": "DemoModel", "origin": "directory", - "source": "tests/data/demo-model/", + "source": full_path_relative_to_root("tests/data/demo-model/") }, "Store": { "name": "DemoStore", @@ -312,7 +310,7 @@ def building(): @pytest.fixture(scope="session") def context_file_path(): - return os.sep.join((os.path.abspath("."), "tests/data/shacl-model/context.json")) + return full_path_relative_to_root("tests/data/shacl-model/context.json") @pytest.fixture(scope="session") @@ -465,7 +463,7 @@ def config(model, store, resolver): "Model": { "name": model, "origin": "directory", - "source": "tests/data/demo-model/", + "source": full_path_relative_to_root("tests/data/demo-model/"), }, "Store": { "name": store, @@ -476,7 +474,7 @@ def config(model, store, resolver): { "resolver": resolver, "origin": "directory", - "source": "tests/data/demo-resolver/", + "source": full_path_relative_to_root("tests/data/demo-resolver/"), "targets": [ { "identifier": "sex", @@ -493,7 +491,9 @@ def config(model, store, resolver): ] }, ], - "result_resource_mapping": "../../configurations/demo-resolver/term-to-resource-mapping.hjson", + "result_resource_mapping": full_path_relative_to_root( + "./examples/configurations/demo-resolver/term-to-resource-mapping.hjson" + ) }, ], }, diff --git a/tests/data/nexus-store/file-to-resource-mapping.hjson b/tests/data/nexus-store/file-to-resource-mapping.hjson new file mode 100644 index 00000000..aa0d78fd --- /dev/null +++ b/tests/data/nexus-store/file-to-resource-mapping.hjson @@ -0,0 +1,27 @@ +{ + type: DataDownload + contentSize: + { + unitCode: f"bytes" + value: x._bytes + } + digest: + { + algorithm: x._digest._algorithm + value: x._digest._value + } + encodingFormat: x._mediaType + name: x._filename + contentUrl: forge.format(uri=x['@id'], formatter='URI_REWRITER', is_file=True) + atLocation: + { + type: Location + store: + { + id: x._storage["@id"] + type: x._storage["@type"] if '@type' in x._storage else None + _rev: x._storage["_rev"] if '_rev' in x._storage else None + } + location: x._location if '_location' in x else None + } +} diff --git a/tests/data/nexus-store/publishing-file-to-resource-mapping.hjson b/tests/data/nexus-store/publishing-file-to-resource-mapping.hjson new file mode 100644 index 00000000..310a5437 --- /dev/null +++ b/tests/data/nexus-store/publishing-file-to-resource-mapping.hjson @@ -0,0 +1,16 @@ +{ + type: DataDownload + contentSize: + { + unitCode: f"bytes" + value: x._bytes + } + digest: + { + algorithm: x._digest._algorithm + value: x._digest._value + } + encodingFormat: x._mediaType + name: x._filename + contentUrl: x._self +} diff --git a/tests/specializations/mappers/test_mappers.py b/tests/specializations/mappers/test_mappers.py index e2ee2a95..0dbf0508 100644 --- a/tests/specializations/mappers/test_mappers.py +++ b/tests/specializations/mappers/test_mappers.py @@ -13,3 +13,47 @@ # along with Blue Brain Nexus Forge. If not, see . # Placeholder for the generic parameterizable test suite for mappers. + +import pytest +from contextlib import nullcontext as does_not_raise + +from kgforge.core import KnowledgeGraphForge, Resource +from kgforge.specializations.mappers import DictionaryMapper +from kgforge.specializations.mappings import DictionaryMapping + + +@pytest.fixture +def mapping_str(): + return """ + { + type: x.type + id: x.id + content_type: { + unitCode: f"bytes" + value: x.p1 + } + encodingFormat: x.p2 + } + """ + + +@pytest.mark.parametrize( + "json_to_map, exception", + [ + ({"id": "123", "type": "Type", "p1": "v1a", "p2": "v2a"}, does_not_raise()), + ({"id": "123", "p1": "v1a", "p2": "v2a"}, pytest.raises(AttributeError)), + + ], +) +def test_mapping_load_no_mapping_type( + exception, config, json_to_map, mapping_str +): + + forge = KnowledgeGraphForge(config) + + mapping = DictionaryMapping.load(mapping_str) + + with exception: + DictionaryMapper(forge).map(json_to_map, mapping, None) + + diff --git a/tests/specializations/mappings/test_mappings.py b/tests/specializations/mappings/test_mappings.py index 6dcdc9a5..79b503a5 100644 --- a/tests/specializations/mappings/test_mappings.py +++ b/tests/specializations/mappings/test_mappings.py @@ -13,3 +13,112 @@ # along with Blue Brain Nexus Forge. If not, see . # Placeholder for the generic parameterizable test suite for mappings. + + +import pytest +from contextlib import nullcontext as does_not_raise + +from hjson.scanner import HjsonDecodeError +from requests import RequestException + +from kgforge.core.archetypes.mapping import MappingType +from kgforge.specializations.mappings import DictionaryMapping +from utils import full_path_relative_to_root + +mapping_url_valid = "https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples" \ + "/configurations/nexus-store/file-to-resource-mapping.hjson" + +mapping_path_valid = full_path_relative_to_root( + "tests/data/nexus-store/file-to-resource-mapping.hjson" +) + +mapping_str_valid = "{}" + +mapping_str_invalid = "i" + +mapping_str_invalid_2 = "{something}" + +mapping_str_invalid_3 = "{a:b}" + +mapping_str_valid_2 = """ +{ + a:b +} +""" + + +@pytest.mark.parametrize( + "source, exception", + [ + (mapping_path_valid, does_not_raise()), + (mapping_url_valid, does_not_raise()), + ], +) +def test_mapping_load_no_mapping_type(source, exception): + with exception: + mapping = DictionaryMapping.load(source) + + +@pytest.mark.parametrize( + "source, mapping_type, exception", + [ + (mapping_path_valid, MappingType.FILE, does_not_raise()), + (mapping_url_valid, MappingType.URL, does_not_raise()), + (mapping_path_valid, MappingType.URL, pytest.raises(RequestException)), + (mapping_url_valid, MappingType.FILE, pytest.raises(FileNotFoundError)), + (mapping_path_valid, MappingType.STR, pytest.raises(Exception)), + (mapping_url_valid, MappingType.STR, pytest.raises(Exception)), + ("i", MappingType.URL, pytest.raises(Exception)), + ], +) +def test_mapping_load_mapping_type(source, mapping_type, exception): + with exception: + mapping = DictionaryMapping.load(source, mapping_type) + + +@pytest.mark.parametrize( + "source, exception", + [ + (mapping_path_valid, does_not_raise()), + (mapping_url_valid, pytest.raises(FileNotFoundError)), + (mapping_str_invalid, pytest.raises(FileNotFoundError)), + (mapping_str_valid, pytest.raises(FileNotFoundError)), + ], +) +def test_mapping_load_file(source, exception): + with exception: + mapping = DictionaryMapping.load_file(source) + + +@pytest.mark.parametrize( + "source, exception", + [ + (mapping_url_valid, does_not_raise()), + (mapping_path_valid, pytest.raises(RequestException)), + (mapping_str_invalid, pytest.raises(RequestException)), + (mapping_str_valid, pytest.raises(RequestException)), + + ], +) +def test_mapping_load_url(source, exception): + with exception: + mapping = DictionaryMapping.load_url(source) + + +@pytest.mark.parametrize( + "source, exception", + [ + (mapping_path_valid, pytest.raises(HjsonDecodeError)), + (mapping_url_valid, pytest.raises(HjsonDecodeError)), + (mapping_str_invalid, pytest.raises(HjsonDecodeError)), + (mapping_str_invalid_2, pytest.raises(HjsonDecodeError)), + (mapping_str_invalid_3, pytest.raises(HjsonDecodeError)), + (mapping_str_valid_2, does_not_raise()), + (mapping_str_valid, does_not_raise()), + ], +) +def test_mapping_load_str(source, exception): + with exception: + mapping = DictionaryMapping.load_str(source) + + diff --git a/tests/specializations/models/test_demo_model.py b/tests/specializations/models/test_demo_model.py index 43ebbc20..53c7f4e9 100644 --- a/tests/specializations/models/test_demo_model.py +++ b/tests/specializations/models/test_demo_model.py @@ -15,8 +15,8 @@ from pytest_bdd import given, parsers, scenarios, when from kgforge.specializations.models.demo_model import DemoModel +from utils import full_path_relative_to_root from tests.conftest import check_report - # TODO To be port to the generic parameterizable test suite for models in test_models.py. DKE-135. @@ -25,7 +25,7 @@ @given("A model instance.") def model(): - return DemoModel("tests/data/demo-model/", origin="directory") + return DemoModel(full_path_relative_to_root("tests/data/demo-model/"), origin="directory") @given("A validated resource.", target_fixture="data") diff --git a/tests/specializations/models/test_rdf_model.py b/tests/specializations/models/test_rdf_model.py index 4546a771..c3895c26 100644 --- a/tests/specializations/models/test_rdf_model.py +++ b/tests/specializations/models/test_rdf_model.py @@ -18,11 +18,12 @@ from kgforge.core.commons.exceptions import ValidationError from kgforge.specializations.models import RdfModel from tests.specializations.models.data import * +from utils import full_path_relative_to_root @pytest.fixture def rdf_model(context_iri_file): - return RdfModel("tests/data/shacl-model", + return RdfModel(full_path_relative_to_root("tests/data/shacl-model"), context={"iri": context_iri_file}, origin="directory") diff --git a/tests/specializations/stores/test_bluebrain_nexus.py b/tests/specializations/stores/test_bluebrain_nexus.py index 0482e6b2..14af735f 100644 --- a/tests/specializations/stores/test_bluebrain_nexus.py +++ b/tests/specializations/stores/test_bluebrain_nexus.py @@ -37,6 +37,7 @@ # FIXME mock Nexus for unittests # TODO To be port to the generic parameterizable test suite for stores in test_stores.py. DKE-135. from kgforge.specializations.stores.nexus import Service +from utils import full_path_relative_to_root BUCKET = "test/kgforge" NEXUS = "https://nexus-instance.org" @@ -44,8 +45,9 @@ NEXUS_PROJECT_CONTEXT = {"base": "http://data.net", "vocab": "http://vocab.net", "apiMappings": [{'namespace': 'https://neuroshapes.org/dash/', 'prefix': 'datashapes'}]} VERSIONED_TEMPLATE = "{x.id}?rev={x._store_metadata._rev}" -FILE_RESOURCE_MAPPING = os.sep.join( - (os.path.curdir, "tests", "data", "nexus-store", "file-to-resource-mapping.hjson") + +FILE_RESOURCE_MAPPING = full_path_relative_to_root( + "./tests/data/nexus-store/file-to-resource-mapping.hjson" ) diff --git a/utils.py b/utils.py index de6788f7..d1375d3b 100644 --- a/utils.py +++ b/utils.py @@ -14,11 +14,12 @@ import nexussdk as nxs from urllib.parse import quote_plus +import os TOKEN = "" base_prod_v1= "https://bbp.epfl.ch/nexus/v1" - + nxs.config.set_environment(base_prod_v1) nxs.config.set_token(TOKEN) @@ -39,24 +40,24 @@ https://bbp.epfl.ch/nexus/v1/files/bbp/mmb-point-neuron-framework-model/https%3A%2F%2Fbbp.epfl.ch%2Fneurosciencegraph%2Fdata%2F79b51b75-81e2-4b2f-98fc-666130512cea """ def uri_formatter_using_previous_project_config(nxs, uri, org, project): - # Retrieve current project description + # Retrieve current project description current_project_description = nxs.projects.fetch(org,project) current_project_description = dict(current_project_description) #current_base = current_project_description["base"] #current_vocab = current_project_description["vocab"] current_project_revision = current_project_description['_rev'] - + if current_project_revision <= 1: raise Exception("The targeted project {org}/{project} does not have a previous revision. It's config was never changed.") - - # Retrieve previous project description + + # Retrieve previous project description previous_project_revision = current_project_revision - 1 previous_project_description = nxs.projects.fetch(org,project, rev=previous_project_revision) previous_project_description = dict(previous_project_description) previous_base = previous_project_description["base"] #previous_vocab = previous_project_description["vocab"] previous_project_revision = previous_project_description['_rev'] - + uri_parts = uri.split("/") uri_last_path = uri_parts[-1] uri_last_path = uri_last_path.split("?") # in case ? params are in the url @@ -70,3 +71,9 @@ def uri_formatter_using_previous_project_config(nxs, uri, org, project): return formatter_uri +def full_path_relative_to_root(path: str): + """ + Provided a path relative to the root of the repository, it is transformed into an absolute path + so that it will be independent of what the working directory is + """ + return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)