From a3e35d8073fec87e24fd75acde44a1715e82e93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 3 Oct 2022 18:09:07 +0200 Subject: [PATCH 01/30] Added DatabaseSources to forge. --- .../database-sources/prod-nexus-sources.yml | 72 ++ .../DictionaryMapping/NeuronMorphology.hjson | 73 ++ .../database_sources/MouseLight/metadata.json | 8 + .../NeuroElectro/metadata.json | 8 + .../NeuronElectrophysiologicalFeature.hjson | 38 + .../UniProt/jsonld_context.json | 0 .../database_sources/UniProt/metadata.json | 8 + .../DictionaryMapping/NeuronMorphology.hjson | 64 + .../DictionaryMapping/PatchedCell.hjson | 41 + .../DictionaryMapping/Subject.hjson | 8 + .../DictionaryMapping/Association.hjson | 12 + .../DictionaryMapping/Contribution.hjson | 12 + .../17 - Database-sources.ipynb | 1104 +++++++++++++++++ kgforge/core/forge.py | 18 + kgforge/specializations/resources/__init__.py | 1 + kgforge/specializations/resources/datasets.py | 10 +- .../specializations/resources/db_sources.py | 66 + 17 files changed, 1538 insertions(+), 5 deletions(-) create mode 100644 examples/configurations/database-sources/prod-nexus-sources.yml create mode 100644 examples/database_sources/MouseLight/DictionaryMapping/NeuronMorphology.hjson create mode 100644 examples/database_sources/MouseLight/metadata.json create mode 100644 examples/database_sources/NeuroElectro/metadata.json create mode 100644 examples/database_sources/UniProt/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson create mode 100644 examples/database_sources/UniProt/jsonld_context.json create mode 100644 examples/database_sources/UniProt/metadata.json create mode 100644 examples/database_sources/allen-cell-types-database/DictionaryMapping/NeuronMorphology.hjson create mode 100644 examples/database_sources/allen-cell-types-database/DictionaryMapping/PatchedCell.hjson create mode 100644 examples/database_sources/allen-cell-types-database/DictionaryMapping/Subject.hjson create mode 100644 examples/database_sources/scientists-database/DictionaryMapping/Association.hjson create mode 100644 examples/database_sources/scientists-database/DictionaryMapping/Contribution.hjson create mode 100644 examples/notebooks/getting-started/17 - Database-sources.ipynb create mode 100644 kgforge/specializations/resources/db_sources.py diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml new file mode 100644 index 00000000..6f3b1c3d --- /dev/null +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -0,0 +1,72 @@ +Model: + name: RdfModel + origin: store + source: BlueBrainNexus + context: + iri: "https://bbp.neuroshapes.org" + bucket: "neurosciencegraph/datamodels" +Store: + name: BlueBrainNexus + endpoint: https://bbp.epfl.ch/nexus/v1 + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + elastic: + endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" + mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" + default_str_keyword_field: "keyword" + vocabulary: + metadata: + iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" + local_iri: "https://bluebrainnexus.io/contexts/metadata.json" + namespace: "https://bluebrain.github.io/nexus/vocabulary/" + deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" + project_property: "https://bluebrain.github.io/nexus/vocabulary/project" + max_connection: 50 + versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" + file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson + +Resolvers: + ontology: + - resolver: OntologyResolver + origin: store + source: BlueBrainNexus + targets: + - identifier: terms + bucket: neurosciencegraph/datamodels + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson + agent: + - resolver: AgentResolver + origin: store + source: BlueBrainNexus + targets: + - identifier: agents + bucket: bbp/agents + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson + +Formatters: + identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} + identifier_neuroelectro: http://neuroelectro.org/api/1/{}/{} + identifier_neurolex: http://neurolex.org/wiki/{} + +DatabaseSources: + UniProt: + origin: store + source: SPARQLStore + endpoint: https://sparql.uniprot.org/sparql + definition: + origin: directory + source: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources + iri: UniProt + NeuroElectro: + origin: store + source: BlueBrainNexus + bucket: bbp/neuroelectro + definition: + iri: nexus_resource_id \ No newline at end of file diff --git a/examples/database_sources/MouseLight/DictionaryMapping/NeuronMorphology.hjson b/examples/database_sources/MouseLight/DictionaryMapping/NeuronMorphology.hjson new file mode 100644 index 00000000..a723c936 --- /dev/null +++ b/examples/database_sources/MouseLight/DictionaryMapping/NeuronMorphology.hjson @@ -0,0 +1,73 @@ +{ + type: [ + Dataset + NeuronMorphology + ] + id: forge.format("identifier", "neuronmorphologies/mouselight", x.neurons[0]["idString"]) + brainLocation: + { + type: BrainLocation + brainRegion: + { + id: f"http://api.brain-map.org/api/v2/data/Structure/{x.neurons[0]['soma']['allenId']}" + label: x.neurons[0]["allenLabel"] + } + coordinatesInBrainAtlas: + { + valueX: x.neurons[0]["soma"]["x"] + valueY: x.neurons[0]["soma"]["y"] + valueZ: x.neurons[0]["soma"]["z"] + } + } + contribution: + { + type: Contribution + agent: + { + type: Organization + id: https://www.grid.ac/institutes/grid.443970.d + label: "Janelia Research Campus" + } + } + distribution: forge.attach(f"./mouselight/{x.neurons[0]['idString']}.swc", content_type="application/swc") + identifier: x.neurons[0]["idString"] + name: x.neurons[0]["idString"] + generation: + { + type: Generation + activity: + { + type: "nsg:NeuronMorphologyReconstruction" + hadProtocol:{ + + } + } + } + subject: + { + type: Subject + species: { + id: http://purl.obolibrary.org/obo/NCBITaxon_10090 + label: Mus musculus + } + strain: { + label: x.neurons[0]["sample"]["strain"] + } + } + license: + { + type: License + id: https://mouselight.janelia.org + } + objectOfStudy: + { + type: ObjectOfStudy + id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells + label: Single Cell + } + dateCreated: x.neurons[0]["sample"]["date"] + version: x.neurons[0]["annotationSpace"]["version"] + description: x.neurons[0]["annotationSpace"]["description"] + virus: x.neurons[0]["label"]["virus"] + fluorophore: x.neurons[0]["label"]["fluorophore"] +} \ No newline at end of file diff --git a/examples/database_sources/MouseLight/metadata.json b/examples/database_sources/MouseLight/metadata.json new file mode 100644 index 00000000..5f4f52e0 --- /dev/null +++ b/examples/database_sources/MouseLight/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain. ", + "url": "https://www.janelia.org/project-team/mouselight/neuronbrowser", + "license" : { + "id": "https://creativecommons.org/licenses/by-nc/4.0", + "label": "CC BY-NC 4.0" + } +} \ No newline at end of file diff --git a/examples/database_sources/NeuroElectro/metadata.json b/examples/database_sources/NeuroElectro/metadata.json new file mode 100644 index 00000000..b04a23d7 --- /dev/null +++ b/examples/database_sources/NeuroElectro/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The goal of the NeuroElectro Project is to extract information about the electrophysiological properties (e.g. resting membrane potentials and membrane time constants) of diverse neuron types from the existing literature and place it into a centralized database.", + "url": "https://neuroelectro.org", + "license" : { + "id": "https://creativecommons.org/licenses/by-sa/4.0", + "label": "CC BY-SA 4.0" + } +} \ No newline at end of file diff --git a/examples/database_sources/UniProt/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson b/examples/database_sources/UniProt/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson new file mode 100644 index 00000000..67b19146 --- /dev/null +++ b/examples/database_sources/UniProt/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson @@ -0,0 +1,38 @@ +{ + type: 'Class', + id: forge.format("identifier_neuroelectro", "e", x.id) + label: x.name if 'name' in x else '' + prefLabel: x.name if 'name' in x else '' + definition: x.definition, if 'definition' in x and x.definition else '' + notation: x.short_name if 'short_name' in x and x.short_name else '' + subClassOf: 'NeuronElectrophysiologicalFeature' + isDefinedBy: 'https://neuroelectro.org', + maxValue: x.max_range if 'max_range' in x and x.max_range else '', + minValue: x.min_range, if 'min_range' in x and x.min_range else '', + "unitCode": { + "id":, + "type": "http://qudt.org/schema/qudt/Unit", + "sameAs": + }, + "plot_transform": x.plot_transform if 'plot_transform' in x and x.plot_transform else '', + "norm_criteria": x.norm_criteria, if 'norm_criteria' in x and x.norm_criteria else '', + sameAs,: forge.format("identifier_neurolex",x.nlex_id) if 'nlex_id' in x and x.nlex_id else '' +} + + +{ + "definition": "Input resistance measured at steady-state voltage response to current injection", + "id": 2, + "max_range": 20000.0, + "min_range": 5.0, + "name": "input resistance", + "nlex_id": null, + "norm_criteria": "Values corrected for differences in units, but are otherwise unchanged. Refer to individual articles for specific definitions and calculation methodologies.", + "plot_transform": "log10", + "short_name": "rin", + "units": { + "id": 2, + "name": "Ω", + "prefix": "M" + } +} \ No newline at end of file diff --git a/examples/database_sources/UniProt/jsonld_context.json b/examples/database_sources/UniProt/jsonld_context.json new file mode 100644 index 00000000..e69de29b diff --git a/examples/database_sources/UniProt/metadata.json b/examples/database_sources/UniProt/metadata.json new file mode 100644 index 00000000..b04a23d7 --- /dev/null +++ b/examples/database_sources/UniProt/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The goal of the NeuroElectro Project is to extract information about the electrophysiological properties (e.g. resting membrane potentials and membrane time constants) of diverse neuron types from the existing literature and place it into a centralized database.", + "url": "https://neuroelectro.org", + "license" : { + "id": "https://creativecommons.org/licenses/by-sa/4.0", + "label": "CC BY-SA 4.0" + } +} \ No newline at end of file diff --git a/examples/database_sources/allen-cell-types-database/DictionaryMapping/NeuronMorphology.hjson b/examples/database_sources/allen-cell-types-database/DictionaryMapping/NeuronMorphology.hjson new file mode 100644 index 00000000..fe4adecb --- /dev/null +++ b/examples/database_sources/allen-cell-types-database/DictionaryMapping/NeuronMorphology.hjson @@ -0,0 +1,64 @@ +{ + # There is no NeuronMorphologyShape nor NeuronMorphology type in Neuroshapes at the moment (09.08.2019). + # Using ReconstructedPatchedCellShape meanwhile. + type: NeuronMorphology + id: forge.format("identifier", "neuronmorphologies", x.specimen__id) + # This property is not part of the ReconstructedPatchedCell shape at the moment (09.08.2019). + apicalDendrite: x.tag__apical + brainLocation: + { + type: BrainLocation + brainRegion: + { + id: f"http://api.brain-map.org/api/v2/data/Structure/{x.structure__id}" + label: x.structure__acronym + } + coordinatesInBrainAtlas: + { + valueX: x.csl__x + valueY: x.csl__y + valueZ: x.csl__z + } + layer: forge.resolve(f"layer {x.structure__layer}", scope="terms", target="structure-layer") + } + contribution: + { + type: Contribution + agent: + { + # 'Organization' is a subclass of 'Agent'. + type: Organization + id: https://www.grid.ac/institutes/grid.417881.3 + } + } + derivation: + [ + { + type: Derivation + entity: + { + # 'Subject' is a subclass of 'Entity'. + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } + } + { + type: Derivation + entity: + { + # 'PatchedCell' is a subclass of 'Entity'. + type: PatchedCell + id: forge.format("identifier", "patchedcells", x.specimen__id) + } + } + ] + distribution: forge.attach(f"./allen_cell_types_database/specimen_{x.specimen__id}/reconstruction.swc", content_type="application/swc") + identifier: x.specimen__id + name: x.specimen__name + # This property is not part of the ReconstructedPatchedCell shape at the moment (09.08.2019). + subject: + { + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } +} \ No newline at end of file diff --git a/examples/database_sources/allen-cell-types-database/DictionaryMapping/PatchedCell.hjson b/examples/database_sources/allen-cell-types-database/DictionaryMapping/PatchedCell.hjson new file mode 100644 index 00000000..a7c266cb --- /dev/null +++ b/examples/database_sources/allen-cell-types-database/DictionaryMapping/PatchedCell.hjson @@ -0,0 +1,41 @@ +{ + type: PatchedCell + id: forge.format("identifier", "patchedcells", x.specimen__id) + brainLocation: + { + type: BrainLocation + brainRegion: + { + id: f"http://api.brain-map.org/api/v2/data/Structure/{x.structure__id}" + label: x.structure__acronym + } + } + contribution: + { + type: Contribution + agent: + { + # 'Organization' is a subclass of 'Agent'. + type: Organization + id: https://www.grid.ac/institutes/grid.417881.3 + } + } + derivation: + { + type: Derivation + entity: + { + # 'Subject' is a subclass of 'Entity'. + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } + } + identifier: x.specimen__id + name: x.specimen__name + # This property is not part of the PatchedCell shape at the moment (09.08.2019). + subject: + { + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } +} \ No newline at end of file diff --git a/examples/database_sources/allen-cell-types-database/DictionaryMapping/Subject.hjson b/examples/database_sources/allen-cell-types-database/DictionaryMapping/Subject.hjson new file mode 100644 index 00000000..175493d0 --- /dev/null +++ b/examples/database_sources/allen-cell-types-database/DictionaryMapping/Subject.hjson @@ -0,0 +1,8 @@ +{ + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + identifier: x.donor__id + name: x.donor__name + sex: forge.resolve(x.donor__sex, scope="terms", target="sex") + species: forge.resolve(x.donor__species, scope="terms", target="species") +} \ No newline at end of file diff --git a/examples/database_sources/scientists-database/DictionaryMapping/Association.hjson b/examples/database_sources/scientists-database/DictionaryMapping/Association.hjson new file mode 100644 index 00000000..9c35618e --- /dev/null +++ b/examples/database_sources/scientists-database/DictionaryMapping/Association.hjson @@ -0,0 +1,12 @@ +{ + type: Association + agent: + { + id: forge.format("identifier", "persons", x.id) + type: Person + additionalName: x.middle_name + gender: forge.resolve(x.gender, scope="terms") + name: x.name + } + distribution: forge.attach(f"../../data/scientists-database/{'_'.join(x.name.lower().split())}.txt") +} \ No newline at end of file diff --git a/examples/database_sources/scientists-database/DictionaryMapping/Contribution.hjson b/examples/database_sources/scientists-database/DictionaryMapping/Contribution.hjson new file mode 100644 index 00000000..63f02a77 --- /dev/null +++ b/examples/database_sources/scientists-database/DictionaryMapping/Contribution.hjson @@ -0,0 +1,12 @@ +{ + type: Contribution + agent: + { + id: forge.format("identifier", "persons", x.id) + type: Person + additionalName: x.middle_name + gender: forge.resolve(x.gender, scope="terms") + name: x.name + } + distribution: forge.attach(f"../../data/scientists-database/{'_'.join(x.name.lower().split())}.txt") +} \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb new file mode 100644 index 00000000..d4ed5ccd --- /dev/null +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -0,0 +1,1104 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Querying external database sources of interest\n", + "\n", + "* Enable users to integrate data from external databases of interest within BBP KG\n", + "* While using the Nexus Forge interface and BMO vocabulary as much as possible as\n", + "* While benefiting from out of the box (meta)data transformation to make them ready for BBP internal pipelines and applications\n", + "* Demo with Mouselight, NeuroElectro, UniProt" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2019-09-23T18:50:20.068658Z", + "start_time": "2019-09-23T18:50:19.054054Z" + } + }, + "outputs": [], + "source": [ + "from kgforge.core import KnowledgeGraphForge\n", + "from kgforge.specializations.resources import Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "token = getpass.getpass()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A configuration file is needed in order to create a KnowledgeGraphForge session. A configuration can be generated using the notebook [00-Initialization.ipynb](00%20-%20Initialization.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", + "BUCKET = \"neurosciencegraph/datamodels\"\n", + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", token=token, bucket=BUCKET)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# List of Data sources" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Database sources with managed mappings:\n", + "{\n", + " type: Database\n", + " definition:\n", + " {\n", + " iri: UniProt\n", + " origin: directory\n", + " source: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources\n", + " }\n", + " endpoint: https://sparql.uniprot.org/sparql\n", + " name: UniProt\n", + " origin: store\n", + " source: SPARQLStore\n", + "}\n", + "{\n", + " type: Database\n", + " bucket: bbp/neuroelectro\n", + " definition:\n", + " {\n", + " iri: nexus_resource_id\n", + " }\n", + " name: NeuroElectro\n", + " origin: store\n", + " source: BlueBrainNexus\n", + "}\n" + ] + } + ], + "source": [ + "sources = forge.db_sources(pretty=True)\n", + "sources" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "data = {\n", + " 'origin': 'store',\n", + " 'source': 'MouseLight', \n", + " 'description': 'The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.',\n", + " 'url': 'https://www.janelia.org/project-team/mouselight/neuronbrowser',\n", + " 'protocol': 'https://www.janelia.org/project-team/mouselight/resources', \n", + " 'license': [{'id': 'https://creativecommons.org/licenses/by-nc/4.0', \n", + " 'label': 'CC BY-NC 4.0'}\n", + " ],\n", + " 'mappings': {\n", + " 'NeuronMorphology': {\n", + " 'DictionaryMapping': '/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson'\n", + " }\n", + " }\n", + " }\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from kgforge.specializations.resources import DatabaseSource\n", + "ds = DatabaseSource(name=\"MouseLight\", **data)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " type: Database\n", + " description: The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", + " license:\n", + " [\n", + " {\n", + " id: https://creativecommons.org/licenses/by-nc/4.0\n", + " label: CC BY-NC 4.0\n", + " }\n", + " ]\n", + " mappings:\n", + " {\n", + " NeuronMorphology:\n", + " {\n", + " DictionaryMapping: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson\n", + " }\n", + " }\n", + " name: source_name\n", + " origin: store\n", + " protocol: https://www.janelia.org/project-team/mouselight/resources\n", + " source: MouseLight\n", + " url: https://www.janelia.org/project-team/mouselight/neuronbrowser\n", + "}\n" + ] + } + ], + "source": [ + "print(ds)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "None\n" + ] + } + ], + "source": [ + "print(sources)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data source metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " type: Datasource\n", + " _store:\n", + " {\n", + " context: null\n", + " bucket: null\n", + " endpoint: null\n", + " file_mapping: null\n", + " metadata_context: null\n", + " model_context: null\n", + " service:\n", + " {\n", + " archives: {}\n", + " records: {}\n", + " tags: {}\n", + " }\n", + " token: null\n", + " versioned_id_template: null\n", + " }\n", + " datatypes:\n", + " [\n", + " NeuronMorphology\n", + " ]\n", + " description: The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", + " license:\n", + " {\n", + " id: https://creativecommons.org/licenses/by-nc/4.0\n", + " label: CC BY-NC 4.0\n", + " }\n", + " mappings:\n", + " {\n", + " NeuronMorphology:\n", + " {\n", + " DictionaryMapping: LazyAction(operation=Mapping.load, args=['/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson'])\n", + " }\n", + " }\n", + " name: Mouselight\n", + " protocol: https://www.janelia.org/project-team/mouselight/resources\n", + " url: https://www.janelia.org/project-team/mouselight/neuronbrowser\n", + "}\n" + ] + } + ], + "source": [ + "mouselight= sources[\"Mouselight\"]\n", + "print(mouselight)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Name, description, url, license, protocol => more can be added through configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mouselight\n", + "The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", + "https://www.janelia.org/project-team/mouselight/neuronbrowser\n", + "['NeuronMorphology']\n", + "https://www.janelia.org/project-team/mouselight/resources\n", + "{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}\n" + ] + } + ], + "source": [ + "print(mouselight.name)\n", + "print(mouselight.description)\n", + "print(mouselight.url)\n", + "print(mouselight.datatypes)\n", + "print(mouselight.protocol)\n", + "print(mouselight.license)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get data mappings (hold transformations logic) per data type\n", + "\n", + "* Data mappings are used to transform results obtained from the external data sources so that they are ready for consumption by BBP tools\n", + "* Perform automatic ontology linking" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " not_supported\n", + " NotSupportedError: RdfModel is not supporting _mappings()\n", + "\n" + ] + } + ], + "source": [ + "forge.mappings(\"Mouselight\", pretty=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from kgforge.specializations.mappings import DictionaryMapping\n", + "mapping = forge.mapping(\"NeuronMorphology\", \"Mouselight\", type=DictionaryMapping)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " id: forge.format(\"identifier\", \"neuronmorphologies/mouselight\", x.neurons[0][\"idString\"])\n", + " type:\n", + " [\n", + " Dataset\n", + " NeuronMorphology\n", + " ]\n", + " brainLocation:\n", + " {\n", + " type: BrainLocation\n", + " brainRegion:\n", + " {\n", + " id: f\"http://api.brain-map.org/api/v2/data/Structure/{x.neurons[0]['soma']['allenId']}\"\n", + " label: x.neurons[0][\"allenLabel\"]\n", + " }\n", + " coordinatesInBrainAtlas:\n", + " {\n", + " valueX: x.neurons[0][\"soma\"][\"x\"]\n", + " valueY: x.neurons[0][\"soma\"][\"y\"]\n", + " valueZ: x.neurons[0][\"soma\"][\"z\"]\n", + " }\n", + " }\n", + " contribution:\n", + " {\n", + " type: Contribution\n", + " agent:\n", + " {\n", + " id: https://www.grid.ac/institutes/grid.443970.d\n", + " type: Organization\n", + " label: Janelia Research Campus\n", + " }\n", + " }\n", + " dateCreated: x.neurons[0][\"sample\"][\"date\"]\n", + " description: x.neurons[0][\"annotationSpace\"][\"description\"]\n", + " distribution: forge.attach(f\"./mouselight/{x.neurons[0]['idString']}.swc\", content_type=\"application/swc\")\n", + " fluorophore: x.neurons[0][\"label\"][\"fluorophore\"]\n", + " generation:\n", + " {\n", + " type: Generation\n", + " activity:\n", + " {\n", + " type: nsg:NeuronMorphologyReconstruction\n", + " hadProtocol: {}\n", + " }\n", + " }\n", + " identifier: x.neurons[0][\"idString\"]\n", + " license:\n", + " {\n", + " id: https://mouselight.janelia.org\n", + " type: License\n", + " }\n", + " name: x.neurons[0][\"idString\"]\n", + " objectOfStudy:\n", + " {\n", + " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", + " type: ObjectOfStudy\n", + " label: Single Cell\n", + " }\n", + " subject:\n", + " {\n", + " type: Subject\n", + " species:\n", + " {\n", + " id: http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " label: Mus musculus\n", + " }\n", + " strain:\n", + " {\n", + " label: x.neurons[0][\"sample\"][\"strain\"]\n", + " }\n", + " }\n", + " version: x.neurons[0][\"annotationSpace\"][\"version\"]\n", + " virus: x.neurons[0][\"label\"][\"virus\"]\n", + "}\n" + ] + } + ], + "source": [ + "print(mapping)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Search and Access data from data source\n", + "\n", + "* Mapping are automatically applied to search results\n", + "* takes a mn for now => working on making it faster " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Type, source or target brain region, \n", + "filters = {\"type\":\"NeuronMorphology\"} # More filters (brain regions, ...) will be added\n", + "#map=True, use_cache=True, # download=True\n", + "resources = forge.search(filters, data_source=\"Mouselight\", limit=2) \n", + "# ADd function for checking datsource health => reqsuire health url from db\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(resources)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/mouselight/AA1544\n", + " type:\n", + " [\n", + " Dataset\n", + " NeuronMorphology\n", + " ]\n", + " brainLocation:\n", + " {\n", + " type: BrainLocation\n", + " brainRegion:\n", + " {\n", + " id: http://api.brain-map.org/api/v2/data/Structure/1021\n", + " label: Secondary motor area layer 6a\n", + " }\n", + " coordinatesInBrainAtlas:\n", + " {\n", + " valueX: 4734.84076775\n", + " valueY: 1791.1994315\n", + " valueZ: 4406.578655875\n", + " }\n", + " }\n", + " contribution:\n", + " {\n", + " type: Contribution\n", + " agent:\n", + " {\n", + " id: https://www.grid.ac/institutes/grid.443970.d\n", + " type: Organization\n", + " label: Janelia Research Campus\n", + " }\n", + " }\n", + " dateCreated: 2018-12-01T05:00:00.000Z\n", + " description: Annotation Space: CCFv2.5 (ML legacy) Axes> Z: Anterior-Posterior; Y: Inferior-Superior; X:Left-Right\n", + " distribution: LazyAction(operation=Store.upload, args=['./mouselight/AA1544.swc', 'application/swc'])\n", + " fluorophore: Immunolabeled with anti-GFP, Alexa-488\n", + " generation:\n", + " {\n", + " type: Generation\n", + " activity:\n", + " {\n", + " type: nsg:NeuronMorphologyReconstruction\n", + " hadProtocol: {}\n", + " }\n", + " }\n", + " identifier: AA1544\n", + " license:\n", + " {\n", + " id: https://mouselight.janelia.org\n", + " type: License\n", + " }\n", + " name: AA1544\n", + " objectOfStudy:\n", + " {\n", + " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", + " type: ObjectOfStudy\n", + " label: Single Cell\n", + " }\n", + " subject:\n", + " {\n", + " type: Subject\n", + " species:\n", + " {\n", + " id: http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " label: Mus musculus\n", + " }\n", + " strain:\n", + " {\n", + " label: Sim1-Cre\n", + " }\n", + " }\n", + " version: 2.5\n", + " virus: PHP-eB-CAG-FRT-rev-3xGFP+PHP-eB-CAG-flex-rev-Flp\n", + "}\n" + ] + } + ], + "source": [ + "print(resources[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Save in BBP KG (Nexus)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 2\n", + " _register_many\n", + " False\n", + " RegistrationError: resource already exists\n" + ] + } + ], + "source": [ + "forge.register(resources)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Access" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set filters" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "_type = \"NeuronMorphology\"\n", + "filters = {\"type\": _type}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run Query" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10 dataset(s) of type NeuronMorphology found\n" + ] + } + ], + "source": [ + "limit = 10 # You can limit the number of results, pass `None` to fetch all the results\n", + "\n", + "data = forge.search(filters, limit=limit)\n", + "\n", + "print(f\"{str(len(data))} dataset(s) of type {_type} found\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Display the results as pandas dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idbrainLocation.brainRegion.idbrainLocation.brainRegion.labelbrainLocation.layercontribution.typecontribution.agent.idcontribution.agent.typedistribution.contentUrldistribution.encodingFormatdistribution.namenamesubject.typesubject.age.periodsubject.age.unitCodesubject.age.valuesubject.identifiersubject.namesubject.sexsubject.speciessubject.strain
0https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...VISp55Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcScnn1a-Tg3-Cre;Ai14-172530.06.01.01SubjectPost-natal322489588Scnn1a-Tg3-Cre;Ai14(GSL)-172530Mus musculusScnn1a-Tg3-Cre
1https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...MTG2Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcH16.06.009.01.01.15.01SubjectPost-natalyrs48528574320H16.06.009FemaleHomo Sapiens
2https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...VISp44Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcScnn1a-Tg3-Cre;Ai14-187849.06.01.01SubjectPost-natal475849748Scnn1a-Tg3-Cre;Ai14(IVSCC)-187849Mus musculusScnn1a-Tg3-Cre
3https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...VISp44Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcRorb-IRES2-Cre-D;Ai14-197330.06.01.01SubjectPost-natal479695183Rorb-IRES2-Cre-D;Ai14-197330Mus musculusRorb-IRES2-Cre
4https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...VISpl44Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcRorb-IRES2-Cre-D;Ai14-230822.04.02.01SubjectPost-natal502081962Rorb-IRES2-Cre-D;Ai14-230822Mus musculusRorb-IRES2-Cre
5https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...AnG2Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcH17.06.004.11.05.04SubjectPost-natalyrs71569008241H17.06.004FemaleHomo Sapiens
6https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...MTG4Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcH17.06.009.11.04.02SubjectPost-natalyrs52595954915H17.06.009MaleHomo Sapiens
7https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...MTG3Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcH16.03.001.01.09.01SubjectPost-natalyrs39518641172H16.03.001MaleHomo Sapiens
8https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...MFG5Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcH17.06.007.11.08.01SubjectPost-natalyrs42576060516H17.06.007FemaleHomo Sapiens
9https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...http://api.brain-map.org/api/v2/data/Structure...VISp55Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://bbp.epfl.ch/nexus/v1/files/dke/kgforge...application/swcreconstruction.swcCux2-CreERT2;Ai14-205530.03.02.01SubjectPost-natal485250100Cux2-CreERT2;Ai14-205530Mus musculusCux2-CreERT2
\n", + "
" + ], + "text/plain": [ + " id \\\n", + "0 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "1 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "2 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "3 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "4 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "5 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "6 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "7 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "8 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "9 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", + "\n", + " brainLocation.brainRegion.id \\\n", + "0 http://api.brain-map.org/api/v2/data/Structure... \n", + "1 http://api.brain-map.org/api/v2/data/Structure... \n", + "2 http://api.brain-map.org/api/v2/data/Structure... \n", + "3 http://api.brain-map.org/api/v2/data/Structure... \n", + "4 http://api.brain-map.org/api/v2/data/Structure... \n", + "5 http://api.brain-map.org/api/v2/data/Structure... \n", + "6 http://api.brain-map.org/api/v2/data/Structure... \n", + "7 http://api.brain-map.org/api/v2/data/Structure... \n", + "8 http://api.brain-map.org/api/v2/data/Structure... \n", + "9 http://api.brain-map.org/api/v2/data/Structure... \n", + "\n", + " brainLocation.brainRegion.label brainLocation.layer contribution.type \\\n", + "0 VISp5 5 Contribution \n", + "1 MTG 2 Contribution \n", + "2 VISp4 4 Contribution \n", + "3 VISp4 4 Contribution \n", + "4 VISpl4 4 Contribution \n", + "5 AnG 2 Contribution \n", + "6 MTG 4 Contribution \n", + "7 MTG 3 Contribution \n", + "8 MFG 5 Contribution \n", + "9 VISp5 5 Contribution \n", + "\n", + " contribution.agent.id contribution.agent.type \\\n", + "0 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "1 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "2 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "3 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "4 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "5 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "6 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "7 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "8 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "9 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "\n", + " distribution.contentUrl \\\n", + "0 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "1 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "2 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "3 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "4 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "5 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "6 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "7 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "8 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "9 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", + "\n", + " distribution.encodingFormat distribution.name \\\n", + "0 application/swc reconstruction.swc \n", + "1 application/swc reconstruction.swc \n", + "2 application/swc reconstruction.swc \n", + "3 application/swc reconstruction.swc \n", + "4 application/swc reconstruction.swc \n", + "5 application/swc reconstruction.swc \n", + "6 application/swc reconstruction.swc \n", + "7 application/swc reconstruction.swc \n", + "8 application/swc reconstruction.swc \n", + "9 application/swc reconstruction.swc \n", + "\n", + " name subject.type subject.age.period \\\n", + "0 Scnn1a-Tg3-Cre;Ai14-172530.06.01.01 Subject Post-natal \n", + "1 H16.06.009.01.01.15.01 Subject Post-natal \n", + "2 Scnn1a-Tg3-Cre;Ai14-187849.06.01.01 Subject Post-natal \n", + "3 Rorb-IRES2-Cre-D;Ai14-197330.06.01.01 Subject Post-natal \n", + "4 Rorb-IRES2-Cre-D;Ai14-230822.04.02.01 Subject Post-natal \n", + "5 H17.06.004.11.05.04 Subject Post-natal \n", + "6 H17.06.009.11.04.02 Subject Post-natal \n", + "7 H16.03.001.01.09.01 Subject Post-natal \n", + "8 H17.06.007.11.08.01 Subject Post-natal \n", + "9 Cux2-CreERT2;Ai14-205530.03.02.01 Subject Post-natal \n", + "\n", + " subject.age.unitCode subject.age.value subject.identifier \\\n", + "0 322489588 \n", + "1 yrs 48 528574320 \n", + "2 475849748 \n", + "3 479695183 \n", + "4 502081962 \n", + "5 yrs 71 569008241 \n", + "6 yrs 52 595954915 \n", + "7 yrs 39 518641172 \n", + "8 yrs 42 576060516 \n", + "9 485250100 \n", + "\n", + " subject.name subject.sex subject.species \\\n", + "0 Scnn1a-Tg3-Cre;Ai14(GSL)-172530 Mus musculus \n", + "1 H16.06.009 Female Homo Sapiens \n", + "2 Scnn1a-Tg3-Cre;Ai14(IVSCC)-187849 Mus musculus \n", + "3 Rorb-IRES2-Cre-D;Ai14-197330 Mus musculus \n", + "4 Rorb-IRES2-Cre-D;Ai14-230822 Mus musculus \n", + "5 H17.06.004 Female Homo Sapiens \n", + "6 H17.06.009 Male Homo Sapiens \n", + "7 H16.03.001 Male Homo Sapiens \n", + "8 H17.06.007 Female Homo Sapiens \n", + "9 Cux2-CreERT2;Ai14-205530 Mus musculus \n", + "\n", + " subject.strain \n", + "0 Scnn1a-Tg3-Cre \n", + "1 \n", + "2 Scnn1a-Tg3-Cre \n", + "3 Rorb-IRES2-Cre \n", + "4 Rorb-IRES2-Cre \n", + "5 \n", + "6 \n", + "7 \n", + "8 \n", + "9 Cux2-CreERT2 " + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "property_to_display = [\"id\",\"name\",\"subject\",\"brainLocation.brainRegion.id\",\"brainLocation.brainRegion.label\",\"brainLocation.layer.id\",\"brainLocation.layer.label\", \"contribution\",\"brainLocation.layer.id\",\"brainLocation.layer.label\",\"distribution.name\",\"distribution.contentUrl\",\"distribution.encodingFormat\"]\n", + "reshaped_data = forge.reshape(data, keep=property_to_display)\n", + "\n", + "forge.as_dataframe(reshaped_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Download" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [], + "source": [ + "dirpath = \"./downloaded/\"\n", + "forge.download(data, \"distribution.contentUrl\", dirpath)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.13 ('kgforge')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "9ac393a5ddd595f2c78ea58b15bf8d269850a4413729cbea5c5fae9013762763" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 0c57eea0..c7439ba6 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -47,6 +47,7 @@ from kgforge.core.wrappings.paths import PathsWrapper, wrap_paths from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping +from kgforge.specializations.resources import DatabaseSource class KnowledgeGraphForge: @@ -514,6 +515,19 @@ def sources(self, pretty: bool = True) -> Optional[List[str]]: """ return self._model.sources(pretty) + @catch + def db_sources(self, pretty: bool = False) -> Optional[List[str]]: + """ + Print(pretty=True) or return (pretty=False) configured data sources. + :param pretty: a boolean + :return: Optional[List[str]] + """ + sources = self._db_sources + if pretty: + print(*["Database sources with managed mappings:", *sources], sep="\n") + else: + return sources + @catch def mappings( self, source: str, pretty: bool = True @@ -964,3 +978,7 @@ def prepare_resolver(config: Dict, store_config: Dict) -> Tuple[str, Resolver]: resolver_name = config.pop("resolver") resolver = import_class(resolver_name, "resolvers") return resolver.__name__, resolver(**config) + +def create_db_sources(config: Optional[Dict[str, Dict[str, str]]]) -> Union[DatabaseSource, List[DatabaseSource]]: + names = config.keys() + return [DatabaseSource(name=name, **config[name]) for name in names] \ No newline at end of file diff --git a/kgforge/specializations/resources/__init__.py b/kgforge/specializations/resources/__init__.py index 46bdaea7..613c0151 100644 --- a/kgforge/specializations/resources/__init__.py +++ b/kgforge/specializations/resources/__init__.py @@ -13,4 +13,5 @@ # along with Blue Brain Nexus Forge. If not, see . from .datasets import Dataset +from .db_sources import DatabaseSource from .entity_linking_candidate import EntityLinkingCandidate \ No newline at end of file diff --git a/kgforge/specializations/resources/datasets.py b/kgforge/specializations/resources/datasets.py index 23375b21..50f0f973 100644 --- a/kgforge/specializations/resources/datasets.py +++ b/kgforge/specializations/resources/datasets.py @@ -12,13 +12,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . -from typing import List, Union +from typing import List, Union, Optional from warnings import warn from kgforge.core import Resource from kgforge.core.commons.actions import LazyAction from kgforge.core.commons.execution import catch, not_supported -from kgforge.core.forge import KnowledgeGraphForge +# Do not 'from kgforge.core import KnowledgeGraphForge' to avoid cyclic dependency # POLICY _set() should be only called as the last statement to ensure atomicity in case of errors. @@ -33,9 +33,9 @@ class Dataset(Resource): # No catching of exceptions so that no incomplete instance is created if an error occurs. # This is a best practice in Python for __init__(). - def __init__(self, forge: KnowledgeGraphForge, type: str = "Dataset", **properties) -> None: + def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Dataset", **properties) -> None: super().__init__(**properties) - self._forge: KnowledgeGraphForge = forge + self._forge: Optional["KnowledgeGraphForge"] = forge self.type: str = type @catch @@ -156,7 +156,7 @@ def download(self, path: str = ".", source: str = "distributions", follow: str = self._forge.download(self, follow, path, overwrite, cross_bucket, content_type) @classmethod - def from_resource(cls, forge: KnowledgeGraphForge, data: Union[Resource, List[Resource]], + def from_resource(cls, forge: Optional["KnowledgeGraphForge"], data: Union[Resource, List[Resource]], store_metadata: bool = False): def _(d): resource_json = forge.as_json(d) diff --git a/kgforge/specializations/resources/db_sources.py b/kgforge/specializations/resources/db_sources.py new file mode 100644 index 00000000..a710937b --- /dev/null +++ b/kgforge/specializations/resources/db_sources.py @@ -0,0 +1,66 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . +import json +from pathlib import Path +from typing import Callable, Optional, Union, Dict, List + +from kgforge.core import Resource +from kgforge.core.archetypes import Mapping + + +class DatabaseSource(Resource): + + def __init__(self, type: str = "Database", **properties) -> None: + """ + The properties defining the Databasesource in YAML are: + + name: + type: Database + origin: <'directory', 'url', or 'store'> + source: + bucket: + endpoint: + token: + bucket: + endpoint: + token: + iri: + """ + super().__init__(**properties) + self.type: str = type + + def _sources(self) -> List[str]: + dirpath = Path(self.source, "mappings") + return [x.stem for x in dirpath.iterdir() if x.is_dir()] + + def _mappings(self, source: str) -> Dict[str, List[str]]: + dirpath = Path(self.source, "mappings", source) + mappings = {} + if dirpath.is_dir(): + for x in dirpath.glob("*/*.hjson"): + mappings.setdefault(x.stem, []).append(x.parent.name) + else: + raise ValueError("unrecognized source") + return mappings + + def mapping(self, entity: str, source: str, type: Callable) -> Mapping: + filename = f"{entity}.hjson" + filepath = Path(self.source, "mappings", source, type.__name__, filename) + if filepath.is_file(): + return type.load(filepath) + else: + raise ValueError("unrecognized entity type or source") + \ No newline at end of file From afd5ba5b1e6e06369a4b845ee6a3f72c5791bab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 4 Oct 2022 14:34:58 +0200 Subject: [PATCH 02/30] Improved DatabaseSource class. Made possible to load mappings from local directory. --- .../DictionaryMapping/NeuronMorphology.hjson | 0 .../NeuronElectrophysiologicalFeature.hjson | 0 .../DictionaryMapping/NeuronMorphology.hjson | 0 .../DictionaryMapping/PatchedCell.hjson | 0 .../DictionaryMapping/Subject.hjson | 0 .../17 - Database-sources.ipynb | 45 ++++---------- kgforge/core/forge.py | 31 ++++++++-- .../specializations/resources/db_sources.py | 59 ++++++++++++++----- 8 files changed, 81 insertions(+), 54 deletions(-) rename examples/database_sources/MouseLight/{ => mappings}/DictionaryMapping/NeuronMorphology.hjson (100%) rename examples/database_sources/UniProt/{ => mappings}/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson (100%) rename examples/database_sources/allen-cell-types-database/{ => mappings}/DictionaryMapping/NeuronMorphology.hjson (100%) rename examples/database_sources/allen-cell-types-database/{ => mappings}/DictionaryMapping/PatchedCell.hjson (100%) rename examples/database_sources/allen-cell-types-database/{ => mappings}/DictionaryMapping/Subject.hjson (100%) diff --git a/examples/database_sources/MouseLight/DictionaryMapping/NeuronMorphology.hjson b/examples/database_sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson similarity index 100% rename from examples/database_sources/MouseLight/DictionaryMapping/NeuronMorphology.hjson rename to examples/database_sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson diff --git a/examples/database_sources/UniProt/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson b/examples/database_sources/UniProt/mappings/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson similarity index 100% rename from examples/database_sources/UniProt/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson rename to examples/database_sources/UniProt/mappings/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson diff --git a/examples/database_sources/allen-cell-types-database/DictionaryMapping/NeuronMorphology.hjson b/examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson similarity index 100% rename from examples/database_sources/allen-cell-types-database/DictionaryMapping/NeuronMorphology.hjson rename to examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson diff --git a/examples/database_sources/allen-cell-types-database/DictionaryMapping/PatchedCell.hjson b/examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson similarity index 100% rename from examples/database_sources/allen-cell-types-database/DictionaryMapping/PatchedCell.hjson rename to examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson diff --git a/examples/database_sources/allen-cell-types-database/DictionaryMapping/Subject.hjson b/examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson similarity index 100% rename from examples/database_sources/allen-cell-types-database/DictionaryMapping/Subject.hjson rename to examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index d4ed5ccd..6fd4369b 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -64,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -72,41 +72,18 @@ "output_type": "stream", "text": [ "Database sources with managed mappings:\n", - "{\n", - " type: Database\n", - " definition:\n", - " {\n", - " iri: UniProt\n", - " origin: directory\n", - " source: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources\n", - " }\n", - " endpoint: https://sparql.uniprot.org/sparql\n", - " name: UniProt\n", - " origin: store\n", - " source: SPARQLStore\n", - "}\n", - "{\n", - " type: Database\n", - " bucket: bbp/neuroelectro\n", - " definition:\n", - " {\n", - " iri: nexus_resource_id\n", - " }\n", - " name: NeuroElectro\n", - " origin: store\n", - " source: BlueBrainNexus\n", - "}\n" + "UniProt\n", + "NeuroElectro\n" ] } ], "source": [ - "sources = forge.db_sources(pretty=True)\n", - "sources" + "sources = forge.db_sources(pretty=True)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -130,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -140,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -164,7 +141,7 @@ " DictionaryMapping: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson\n", " }\n", " }\n", - " name: source_name\n", + " name: MouseLight\n", " origin: store\n", " protocol: https://www.janelia.org/project-team/mouselight/resources\n", " source: MouseLight\n", @@ -179,14 +156,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "None\n" + "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _inner_sync=False, definition={'origin': 'directory', 'source': '/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources', 'iri': 'UniProt'}, endpoint='https://sparql.uniprot.org/sparql', name='UniProt', origin='store', source='SPARQLStore'), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _inner_sync=False, bucket='bbp/neuroelectro', definition={'iri': 'nexus_resource_id'}, name='NeuroElectro', origin='store', source='BlueBrainNexus')}\n" ] } ], @@ -440,7 +417,7 @@ "# Type, source or target brain region, \n", "filters = {\"type\":\"NeuronMorphology\"} # More filters (brain regions, ...) will be added\n", "#map=True, use_cache=True, # download=True\n", - "resources = forge.search(filters, data_source=\"Mouselight\", limit=2) \n", + "resources = forge.search(filters, db_source=\"Mouselight\", limit=2) \n", "# ADd function for checking datsource health => reqsuire health url from db\n" ] }, diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index c7439ba6..4c5a03fe 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -248,6 +248,19 @@ def __init__(self, configuration: Union[str, Dict], **kwargs) -> None: # Formatters. self._formatters: Optional[Dict[str, str]] = config.pop("Formatters", None) +<<<<<<< HEAD +======= + # Database sources. + db_sources_config: Optional[Dict[str, Dict[str, str]]] = config.pop("DatabaseSources", None) + self._db_sources: Union[DatabaseSource, List[DatabaseSource]] = ( + self.create_db_sources(db_sources_config) + if db_sources_config + else None + ) + + # Modeling User Interface. + +>>>>>>> a48cc2b (Improved DatabaseSource class. Made possible to load mappings from local directory.) @catch def prefixes(self, pretty: bool = True) -> Optional[Dict[str, str]]: """ @@ -525,8 +538,13 @@ def db_sources(self, pretty: bool = False) -> Optional[List[str]]: sources = self._db_sources if pretty: print(*["Database sources with managed mappings:", *sources], sep="\n") - else: - return sources + return sources + + def add_db_source(self, db_source: DatabaseSource) -> None: + """ + Add a DatabaseSource to the KG. + """ + self._db_sources[db_source.name] = db_source @catch def mappings( @@ -950,6 +968,11 @@ def get_model_context(self): """Expose the context used in the model.""" return self._model.context() + def create_db_sources(self, config: Optional[Dict[str, Dict[str, str]]]) -> Union[DatabaseSource, List[DatabaseSource]]: + names = config.keys() + return {name: DatabaseSource(self, name=name, from_forge=True, **config[name]) for name in names} + + def prepare_resolvers( config: Dict, store_config: Dict ) -> Dict[str, Dict[str, Resolver]]: @@ -978,7 +1001,3 @@ def prepare_resolver(config: Dict, store_config: Dict) -> Tuple[str, Resolver]: resolver_name = config.pop("resolver") resolver = import_class(resolver_name, "resolvers") return resolver.__name__, resolver(**config) - -def create_db_sources(config: Optional[Dict[str, Dict[str, str]]]) -> Union[DatabaseSource, List[DatabaseSource]]: - names = config.keys() - return [DatabaseSource(name=name, **config[name]) for name in names] \ No newline at end of file diff --git a/kgforge/specializations/resources/db_sources.py b/kgforge/specializations/resources/db_sources.py index a710937b..f606c5c2 100644 --- a/kgforge/specializations/resources/db_sources.py +++ b/kgforge/specializations/resources/db_sources.py @@ -11,43 +11,73 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . +import os import json -from pathlib import Path +from pathlib import Path, PurePath +from re import I from typing import Callable, Optional, Union, Dict, List from kgforge.core import Resource from kgforge.core.archetypes import Mapping +# Get local paths +pathparts = list(Path(__file__).resolve().parts) +KGFORGE_PATH = os.path.join(*pathparts[:-4]) +DBS_PATH = os.path.join(KGFORGE_PATH, "examples", "database_sources") class DatabaseSource(Resource): + """A high-level class for Database-related Resources.""" + + + _DBDIR = Path + _REQUIRED = ("name", "origin", "source", "definition") + _RESERVED = {"_forge", "_from_forge", "_check_properties", "_save_config", "_dirpath", + "_mappings", "mappings"} | Resource._RESERVED - def __init__(self, type: str = "Database", **properties) -> None: + def __init__(self, forge: Optional["KnowledgeGRaphForge"], type: str = "Database", + from_forge: bool = True, **properties) -> None: """ The properties defining the Databasesource in YAML are: - name: - type: Database - origin: <'directory', 'url', or 'store'> - source: + name: - REQUIRED + type: Database - REQUIRED + origin: <'directory', 'url', or 'store'> - REQUIRED + source: - REQUIRED bucket: endpoint: token: bucket: endpoint: token: iri: """ + self._check_properties(**properties) super().__init__(**properties) self.type: str = type + self._forge: Optional["KnowledgeGraphForge"] = forge + self._from_forge = from_forge + if self._from_forge is False: + self._save_config() - def _sources(self) -> List[str]: - dirpath = Path(self.source, "mappings") - return [x.stem for x in dirpath.iterdir() if x.is_dir()] + def _check_properties(self, **info): + properties = info.keys() + for r in self._REQUIRED: + if r not in properties: + raise ValueError(f'Missing {r} from the properties to define the DatabaseResource') - def _mappings(self, source: str) -> Dict[str, List[str]]: - dirpath = Path(self.source, "mappings", source) + def _save_config(self) -> None: + """Save database information inside the kgforge database folder.""" + self._dirpath = os.path.join(DBS_PATH, self.name) + Path(self._dirpath).mkdir(parents=True, exist_ok=True) + # Make mappings directory + Path(self._dirpath, 'mappings').mkdir(parents=True, exist_ok=True) + # Add the source to forge + self._forge.add_db_source(self) + + def _mappings(self) -> Dict[str, List[str]]: + dirpath = Path(self._dirpath, "mappings") mappings = {} if dirpath.is_dir(): for x in dirpath.glob("*/*.hjson"): @@ -56,9 +86,10 @@ def _mappings(self, source: str) -> Dict[str, List[str]]: raise ValueError("unrecognized source") return mappings - def mapping(self, entity: str, source: str, type: Callable) -> Mapping: + def mapping(self, entity: str, type: Callable) -> Mapping: filename = f"{entity}.hjson" - filepath = Path(self.source, "mappings", source, type.__name__, filename) + filepath = Path(self._dirpath, "mappings", type.__name__, filename) + print(filepath) if filepath.is_file(): return type.load(filepath) else: From f03f941e542da20db44b980b8615361d159d6734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 4 Oct 2022 14:56:08 +0200 Subject: [PATCH 03/30] Added option to retrieve mappings from bd_sources also from the forge. --- .../17 - Database-sources.ipynb | 269 +++++++++--------- kgforge/core/forge.py | 10 +- .../specializations/resources/db_sources.py | 11 +- 3 files changed, 147 insertions(+), 143 deletions(-) diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 6fd4369b..b2887afc 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -27,16 +27,6 @@ "from kgforge.specializations.resources import Dataset" ] }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import getpass\n", - "token = getpass.getpass()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -46,13 +36,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"neurosciencegraph/datamodels\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", token=token, bucket=BUCKET)" + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", bucket=BUCKET)" ] }, { @@ -64,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -83,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -97,27 +87,26 @@ " 'license': [{'id': 'https://creativecommons.org/licenses/by-nc/4.0', \n", " 'label': 'CC BY-NC 4.0'}\n", " ],\n", - " 'mappings': {\n", - " 'NeuronMorphology': {\n", - " 'DictionaryMapping': '/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson'\n", - " }\n", + " 'definition': {\n", + " 'origin': 'directory',\n", + " 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'\n", " }\n", " }\n" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from kgforge.specializations.resources import DatabaseSource\n", - "ds = DatabaseSource(name=\"MouseLight\", **data)" + "ds = DatabaseSource(forge, name=\"MouseLight\", from_forge=False, **data)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -126,6 +115,11 @@ "text": [ "{\n", " type: Database\n", + " definition:\n", + " {\n", + " origin: directory\n", + " source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/\n", + " }\n", " description: The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", " license:\n", " [\n", @@ -134,13 +128,6 @@ " label: CC BY-NC 4.0\n", " }\n", " ]\n", - " mappings:\n", - " {\n", - " NeuronMorphology:\n", - " {\n", - " DictionaryMapping: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson\n", - " }\n", - " }\n", " name: MouseLight\n", " origin: store\n", " protocol: https://www.janelia.org/project-team/mouselight/resources\n", @@ -156,14 +143,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _inner_sync=False, definition={'origin': 'directory', 'source': '/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources', 'iri': 'UniProt'}, endpoint='https://sparql.uniprot.org/sparql', name='UniProt', origin='store', source='SPARQLStore'), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _inner_sync=False, bucket='bbp/neuroelectro', definition={'iri': 'nexus_resource_id'}, name='NeuroElectro', origin='store', source='BlueBrainNexus')}\n" + "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _forge=, _from_forge=True, _inner_sync=False, definition={'origin': 'directory', 'source': '/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources', 'iri': 'UniProt'}, endpoint='https://sparql.uniprot.org/sparql', name='UniProt', origin='store', source='SPARQLStore'), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _forge=, _from_forge=True, _inner_sync=False, bucket='bbp/neuroelectro', definition={'iri': 'nexus_resource_id'}, name='NeuroElectro', origin='store', source='BlueBrainNexus'), 'MouseLight': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight', _forge=, _from_forge=False, _inner_sync=False, definition={'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'}, description='The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.', license=[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}], name='MouseLight', origin='store', protocol='https://www.janelia.org/project-team/mouselight/resources', source='MouseLight', url='https://www.janelia.org/project-team/mouselight/neuronbrowser')}\n" ] } ], @@ -188,50 +175,31 @@ "output_type": "stream", "text": [ "{\n", - " type: Datasource\n", - " _store:\n", + " type: Database\n", + " definition:\n", " {\n", - " context: null\n", - " bucket: null\n", - " endpoint: null\n", - " file_mapping: null\n", - " metadata_context: null\n", - " model_context: null\n", - " service:\n", - " {\n", - " archives: {}\n", - " records: {}\n", - " tags: {}\n", - " }\n", - " token: null\n", - " versioned_id_template: null\n", + " origin: directory\n", + " source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/\n", " }\n", - " datatypes:\n", - " [\n", - " NeuronMorphology\n", - " ]\n", " description: The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", " license:\n", - " {\n", - " id: https://creativecommons.org/licenses/by-nc/4.0\n", - " label: CC BY-NC 4.0\n", - " }\n", - " mappings:\n", - " {\n", - " NeuronMorphology:\n", + " [\n", " {\n", - " DictionaryMapping: LazyAction(operation=Mapping.load, args=['/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources/MouseLight/DictionaryMapping/NeuronMorphology.hjson'])\n", + " id: https://creativecommons.org/licenses/by-nc/4.0\n", + " label: CC BY-NC 4.0\n", " }\n", - " }\n", - " name: Mouselight\n", + " ]\n", + " name: MouseLight\n", + " origin: store\n", " protocol: https://www.janelia.org/project-team/mouselight/resources\n", + " source: MouseLight\n", " url: https://www.janelia.org/project-team/mouselight/neuronbrowser\n", "}\n" ] } ], "source": [ - "mouselight= sources[\"Mouselight\"]\n", + "mouselight= sources[\"MouseLight\"]\n", "print(mouselight)" ] }, @@ -251,12 +219,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mouselight\n", + "MouseLight\n", "The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", "https://www.janelia.org/project-team/mouselight/neuronbrowser\n", - "['NeuronMorphology']\n", "https://www.janelia.org/project-team/mouselight/resources\n", - "{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}\n" + "[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}]\n" ] } ], @@ -264,7 +231,6 @@ "print(mouselight.name)\n", "print(mouselight.description)\n", "print(mouselight.url)\n", - "print(mouselight.datatypes)\n", "print(mouselight.protocol)\n", "print(mouselight.license)" ] @@ -281,36 +247,38 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - " not_supported\n", - " NotSupportedError: RdfModel is not supporting _mappings()\n", - "\n" - ] + "data": { + "text/plain": [ + "{'NeuronMorphology': ['DictionaryMapping']}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "forge.mappings(\"Mouselight\", pretty=False)" + "forge.mappings(\"MouseLight\", pretty=False)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "from kgforge.specializations.mappings import DictionaryMapping\n", - "mapping = forge.mapping(\"NeuronMorphology\", \"Mouselight\", type=DictionaryMapping)" + "mapping = forge.mapping(\"NeuronMorphology\", \"MouseLight\", type=DictionaryMapping)\n", + "direct_mapping = mouselight.mapping(\"NeuronMorphology\", type=DictionaryMapping)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -398,60 +366,17 @@ "print(mapping)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Search and Access data from data source\n", - "\n", - "* Mapping are automatically applied to search results\n", - "* takes a mn for now => working on making it faster " - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "# Type, source or target brain region, \n", - "filters = {\"type\":\"NeuronMorphology\"} # More filters (brain regions, ...) will be added\n", - "#map=True, use_cache=True, # download=True\n", - "resources = forge.search(filters, db_source=\"Mouselight\", limit=2) \n", - "# ADd function for checking datsource health => reqsuire health url from db\n" - ] - }, { "cell_type": "code", "execution_count": 14, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(resources)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/mouselight/AA1544\n", + " id: forge.format(\"identifier\", \"neuronmorphologies/mouselight\", x.neurons[0][\"idString\"])\n", " type:\n", " [\n", " Dataset\n", @@ -462,14 +387,14 @@ " type: BrainLocation\n", " brainRegion:\n", " {\n", - " id: http://api.brain-map.org/api/v2/data/Structure/1021\n", - " label: Secondary motor area layer 6a\n", + " id: f\"http://api.brain-map.org/api/v2/data/Structure/{x.neurons[0]['soma']['allenId']}\"\n", + " label: x.neurons[0][\"allenLabel\"]\n", " }\n", " coordinatesInBrainAtlas:\n", " {\n", - " valueX: 4734.84076775\n", - " valueY: 1791.1994315\n", - " valueZ: 4406.578655875\n", + " valueX: x.neurons[0][\"soma\"][\"x\"]\n", + " valueY: x.neurons[0][\"soma\"][\"y\"]\n", + " valueZ: x.neurons[0][\"soma\"][\"z\"]\n", " }\n", " }\n", " contribution:\n", @@ -482,10 +407,10 @@ " label: Janelia Research Campus\n", " }\n", " }\n", - " dateCreated: 2018-12-01T05:00:00.000Z\n", - " description: Annotation Space: CCFv2.5 (ML legacy) Axes> Z: Anterior-Posterior; Y: Inferior-Superior; X:Left-Right\n", - " distribution: LazyAction(operation=Store.upload, args=['./mouselight/AA1544.swc', 'application/swc'])\n", - " fluorophore: Immunolabeled with anti-GFP, Alexa-488\n", + " dateCreated: x.neurons[0][\"sample\"][\"date\"]\n", + " description: x.neurons[0][\"annotationSpace\"][\"description\"]\n", + " distribution: forge.attach(f\"./mouselight/{x.neurons[0]['idString']}.swc\", content_type=\"application/swc\")\n", + " fluorophore: x.neurons[0][\"label\"][\"fluorophore\"]\n", " generation:\n", " {\n", " type: Generation\n", @@ -495,13 +420,13 @@ " hadProtocol: {}\n", " }\n", " }\n", - " identifier: AA1544\n", + " identifier: x.neurons[0][\"idString\"]\n", " license:\n", " {\n", " id: https://mouselight.janelia.org\n", " type: License\n", " }\n", - " name: AA1544\n", + " name: x.neurons[0][\"idString\"]\n", " objectOfStudy:\n", " {\n", " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", @@ -518,15 +443,79 @@ " }\n", " strain:\n", " {\n", - " label: Sim1-Cre\n", + " label: x.neurons[0][\"sample\"][\"strain\"]\n", " }\n", " }\n", - " version: 2.5\n", - " virus: PHP-eB-CAG-FRT-rev-3xGFP+PHP-eB-CAG-flex-rev-Flp\n", + " version: x.neurons[0][\"annotationSpace\"][\"version\"]\n", + " virus: x.neurons[0][\"label\"][\"virus\"]\n", "}\n" ] } ], + "source": [ + "print(direct_mapping)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Search and Access data from data source\n", + "\n", + "* Mapping are automatically applied to search results\n", + "* takes a mn for now => working on making it faster " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Type, source or target brain region, \n", + "filters = {\"type\":\"NeuronMorphology\"} # More filters (brain regions, ...) will be added\n", + "#map=True, use_cache=True, # download=True\n", + "resources = forge.search(filters, db_source=\"MouseLight\", limit=2) \n", + "# ADd function for checking datsource health => reqsuire health url from db\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(resources)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "IndexError", + "evalue": "list index out of range", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_5900/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m: list index out of range" + ] + } + ], "source": [ "print(resources[0])" ] @@ -540,7 +529,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -574,7 +563,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -591,7 +580,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -619,7 +608,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1043,7 +1032,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 4c5a03fe..82790034 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -558,7 +558,10 @@ def mappings( :param pretty: a boolean :return: Optional[Dict[str, List[str]]] """ - return self._model.mappings(source, pretty) + if source in self._db_sources: + return self._db_sources[source].mappings(pretty) + else: + return self._model.mappings(source, pretty) @catch def mapping( @@ -572,7 +575,10 @@ def mapping( :param type: a Mapping class :return: Mapping """ - return self._model.mapping(entity, source, type) + if source in self._db_sources: + return self._db_sources[source].mapping(entity, type) + else: + return self._model.mapping(entity, source, type) @catch def map( diff --git a/kgforge/specializations/resources/db_sources.py b/kgforge/specializations/resources/db_sources.py index f606c5c2..521903a7 100644 --- a/kgforge/specializations/resources/db_sources.py +++ b/kgforge/specializations/resources/db_sources.py @@ -86,10 +86,19 @@ def _mappings(self) -> Dict[str, List[str]]: raise ValueError("unrecognized source") return mappings + def mappings(self, pretty: bool) -> Optional[Dict[str, List[str]]]: + mappings = {k: sorted(v) for k, v in + sorted(self._mappings().items(), key=lambda kv: kv[0])} + if pretty: + print("Managed mappings for the data source per entity type and mapping type:") + for k, v in mappings.items(): + print(*[f" - {k}:", *v], sep="\n * ") + else: + return mappings + def mapping(self, entity: str, type: Callable) -> Mapping: filename = f"{entity}.hjson" filepath = Path(self._dirpath, "mappings", type.__name__, filename) - print(filepath) if filepath.is_file(): return type.load(filepath) else: From 9ace7856394b1b12ef4fc1855e3f313391d59952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 5 Oct 2022 17:38:10 +0200 Subject: [PATCH 04/30] Added option to display types of mappings, a model and a store to each DatabaseSource. --- .../database-sources/prod-nexus-sources.yml | 19 +- ...lectrophysiologicalFeatureAnnotation.hjson | 48 ++ .../ParameterAnnotation.hjson | 51 ++ .../DictionaryMapping/ParameterBody.hjson | 13 + .../DictionaryMapping/ScholarlyArticle.hjson | 21 + .../DictionaryMapping/SeriesBody.hjson | 32 ++ .../NeuroElectro/ontologies/data-types.ttl | 176 ++++++ .../NeuroElectro/ontologies/efeatures.ttl | 527 ++++++++++++++++++ .../17 - Database-sources.ipynb | 243 ++++++-- kgforge/core/forge.py | 48 +- .../specializations/resources/db_sources.py | 68 ++- 11 files changed, 1177 insertions(+), 69 deletions(-) create mode 100644 examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson create mode 100644 examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson create mode 100644 examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson create mode 100644 examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson create mode 100644 examples/database_sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson create mode 100644 examples/database_sources/NeuroElectro/ontologies/data-types.ttl create mode 100644 examples/database_sources/NeuroElectro/ontologies/efeatures.ttl diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml index 6f3b1c3d..1503cd4f 100644 --- a/examples/configurations/database-sources/prod-nexus-sources.yml +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -57,16 +57,13 @@ Formatters: DatabaseSources: UniProt: - origin: store - source: SPARQLStore - endpoint: https://sparql.uniprot.org/sparql - definition: + store: + name: DemoStore + model: + name: DemoModel origin: directory - source: /Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources - iri: UniProt + source: "../../../tests/data/demo-model/" NeuroElectro: - origin: store - source: BlueBrainNexus - bucket: bbp/neuroelectro - definition: - iri: nexus_resource_id \ No newline at end of file + store: + name: BlueBrainNexus + bucket: bbp/neuroelectro \ No newline at end of file diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson new file mode 100644 index 00000000..4b9ba001 --- /dev/null +++ b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson @@ -0,0 +1,48 @@ +{ + "id": forge.format('identifier', "electrophysiologicalfeatureannotations", x.bbpID) + "type": [ + "Entity" + "Annotation" + ] + "description": f"This resource contains Electrophysiological Feature Annotations for {x.NeuronName}" + "hasBody": x.AnnotationBody + "hasTarget": { + "type": "AnnotationTarget" + "hasSelector": x.Selector + "hasSource": { + "id": f"https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/{str(x.ArticleID)}" + "type": "ScholarlyArticle" + } + } + "subject": { + "type": "Subject" + "species": x.resolvedSpecies + "strain": x.resolvedStrain + } + "brainLocation": { + "type": "BrainLocation" + "atlasSpatialReferenceSystem": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "brainRegion": x.resolvedBrainRegion + } + "atlasRelease": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/831a626a-c0ae-4691-8ce8-cfb7491345d9" + "type": [ + "BrainAtlasRelease" + "AtlasRelease" + ] + } + "isRegisteredIn": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "name": f"Annotation for {x.NeuronPrefName}" +} diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson new file mode 100644 index 00000000..99be0646 --- /dev/null +++ b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson @@ -0,0 +1,51 @@ +{ + "id": forge.format('identifier', "protocolannotations", x.expID) + "description": f"This resource contains experimental parameters used to measure the electrophysiological features reported in article_{x.ArticleID}" + "type": [ + "Entity" + "Protocol" + "ParameterAnnotation" + "Annotation" + ] + "hasBody": x.AnnotationBody + "hasTarget": { + "type": "AnnotationTarget" + "hasSource": { + "id": f"https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/{str(x.ArticleID)}" + "type": "ScholarlyArticle" + } + } + "brainLocation": { + "type": "BrainLocation" + "atlasSpatialReferenceSystem": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "brainRegion": x.resolvedBrainRegion + } + "atlasRelease": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/831a626a-c0ae-4691-8ce8-cfb7491345d9" + "type": [ + "BrainAtlasRelease" + "AtlasRelease" + ] + } + "isRegisteredIn": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "subject": { + "type": "Subject" + "species": x.resolvedSpecies + "strain": x.resolvedStrain + "age": x.ageObject + } + "name": f"Experimental parameters from article with PubMed ID {x.Pmid}" + "note": x.note +} diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson new file mode 100644 index 00000000..f65975e2 --- /dev/null +++ b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson @@ -0,0 +1,13 @@ +{ + "type": [ + "AnnotationBody" + "Parameter" + ] + "isMeasurementOf": + { + "label": x.parameter_label + } + "value": x.value + "unitCode": x.units + "comment": x.comment +} \ No newline at end of file diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson new file mode 100644 index 00000000..49cd796a --- /dev/null +++ b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson @@ -0,0 +1,21 @@ +{ + type: [ + Entity + ScholarlyArticle + ] + id: forge.format("identifier", "scholarlyarticles", x.id) + abstract: x.abstract + author: x.authors_shaped + title: x.title + name: f"article_{x.id}" + isPartOf: { + type: Periodical + name: x.journal + publisher: x.publisher + issn: x.issn + } + identifier: x.identifiers + datePublished: x.date_issued + sameAs: x.full_text_link + url: x.full_text_link +} \ No newline at end of file diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson new file mode 100644 index 00000000..392b11e2 --- /dev/null +++ b/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson @@ -0,0 +1,32 @@ +{ + "type": "AnnotationBody" + "isMeasurementOf": { + "id": x.ephys_id + "label": x.resolved_name + } + "value": + { + "series": [ + { + "statistic": "raw" + "unitCode": x.units + "value": x.value['raw'] + } + { + "statistic": "error" + "unitCode": x.units + "value": x.value['err'] + } + { + "statistic": "N" + "unitCode": x.units + "value": x.value['n'] + } + { + "statistic": "standard deviation" + "unitCode": x.units + "value": x.value['sd'] + } + ] + } +} \ No newline at end of file diff --git a/examples/database_sources/NeuroElectro/ontologies/data-types.ttl b/examples/database_sources/NeuroElectro/ontologies/data-types.ttl new file mode 100644 index 00000000..8679b458 --- /dev/null +++ b/examples/database_sources/NeuroElectro/ontologies/data-types.ttl @@ -0,0 +1,176 @@ +@prefix : . +@prefix owl: . +@prefix rdf: . +@prefix xml: . +@prefix xsd: . +@prefix rdfs: . +@base . + + rdf:type owl:Ontology ; + owl:versionInfo "R167"^^xsd:string ; + rdfs:label "Dataset Type Ontology"@en . + +################################################################# +# Annotation properties +################################################################# + +### http://www.w3.org/2004/02/skos/core#altLabel + rdf:type owl:AnnotationProperty . + + +### http://www.w3.org/2004/02/skos/core#prefLabel + rdf:type owl:AnnotationProperty . + +################################################################# +# Object Properties +################################################################# + +### https://neuroshapes.org/defines + rdf:type owl:AnnotationProperty . + + +################################################################# +# Classes +################################################################# + +### http://schema.org/Dataset + rdf:type owl:Class ; + rdfs:label "Dataset"@en ; + "Dataset"@en . + + +### https://bbp.epfl.ch/ontologies/core/bmo/METypeRatio + rdf:type owl:Class ; + rdfs:label "METypeRatio"@en ; + "METype ratio"@en . + + +### https://bbp.epfl.ch/ontologies/core/bmo/MTypeDensity + rdf:type owl:Class ; + rdfs:label "MTypeDensity"@en ; + "Morphology Type Density"@en . + + +### https://neuroshapes.org/BrainParcellationDataLayer + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "BrainParcellationDataLayer"@en ; + "Brain Parcellation Volume"@en . + + +### https://neuroshapes.org/BrainRegionMesh + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "BrainRegionMesh"@en ; + "Parcellation Mesh"@en . + + +### https://neuroshapes.org/BrainTemplateDataLayer + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "BrainTemplateDataLayer"@en ; + "Brain Template Volume"@en . + + +### https://neuroshapes.org/CellDensityDataLayer + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "CellDensityDataLayer"@en ; + "Cell Density Volume"@en . + + +### https://neuroshapes.org/CellPositions + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "CellPositions"@en ; + "Cell Positions"@en ; + "Cell Positions"@en , + "CellPositions"@en . + + + +### https://neuroshapes.org/ElectricalSeries + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "ElectricalSeries"@en ; + "Electrical Series"@en . + + +### https://neuroshapes.org/GeneExpressionVolumetricDataLayer + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "GeneExpressionVolumetricDataLayer"@en ; + "Gene Expression Volume"@en . + + +### https://neuroshapes.org/Literature + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "Literature"@en ; + "Literature"@en . + + +### https://neuroshapes.org/Mesh + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "Mesh"@en ; + "Mesh"@en . + + +### https://neuroshapes.org/NISSLImageDataLayer + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "NISSLImageDataLayer"@en ; + "NISSL Volume"@en . + + +### https://neuroshapes.org/NeuronMorphology + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "NeuronMorphology"@en ; + "Neuron Morphology"@en . + + +### https://neuroshapes.org/PointCloud + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "PointCloud"@en ; + "Point Cloud"@en . + + +### https://neuroshapes.org/ReconstructedNeuronMorphology + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "ReconstructedNeuronMorphology"@en ; + "Reconstructed Neuron Morphology"@en . + + +### https://neuroshapes.org/SynthesizedNeuronMorphology + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "SynthesizedNeuronMorphology"@en ; + "Synthesized Neuron Morphology"@en . + + +### https://neuroshapes.org/TimeSeries + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "TimeSeries"@en ; + "Time Series"@en . + + +### https://neuroshapes.org/Trace + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "Trace"@en ; + "Trace"@en . + + +### https://neuroshapes.org/VolumetricDataLayer + rdf:type owl:Class ; + rdfs:subClassOf ; + rdfs:label "VolumetricDataLayer"@en ; + "Volume"@en . + + +### Generated by the OWL API (version 4.5.10) https://github.com/owlcs/owlapi diff --git a/examples/database_sources/NeuroElectro/ontologies/efeatures.ttl b/examples/database_sources/NeuroElectro/ontologies/efeatures.ttl new file mode 100644 index 00000000..3abb9ba3 --- /dev/null +++ b/examples/database_sources/NeuroElectro/ontologies/efeatures.ttl @@ -0,0 +1,527 @@ +@prefix : . +@prefix owl: . +@prefix rdf: . +@prefix xml: . +@prefix xsd: . +@prefix rdfs: . +@base . + + rdf:type owl:Ontology ; + rdfs:label "Electrophysiological Features Ontology"@en . + +################################################################# +# Annotation properties +################################################################# + +### http://www.w3.org/2004/02/skos/core#altLabel + rdf:type owl:AnnotationProperty . + + +### http://www.w3.org/2004/02/skos/core#definition + rdf:type owl:AnnotationProperty . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/efel_id +:efel_id rdf:type owl:AnnotationProperty ; + rdfs:label "efel_id"@en . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/efel_namespace +:efel_namespace rdf:type owl:AnnotationProperty ; + rdfs:label "efel_namespace"@en . + + +### https://neuroshapes.org/units + rdf:type owl:AnnotationProperty ; + rdfs:label "units"@en . + + +################################################################# +# Classes +################################################################# + +### https://bbp.epfl.ch/ontologies/core/bmo/NeuronElectrophysiologicalFeature + rdf:type owl:Class ; + rdfs:label "Neuron Electrophysiological Feature"@en . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/ADPAmplitude +:ADPAmplitude rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "ADP Amplitude"@en ; + "Amplitude from first AP onset to maximum voltage, typically more depolarized than the resting membrane potential"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/ADPDuration +:ADPDuration rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "ADP Duration"@en ; + "Duration from first AP onset to maximum ADP"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AHPAmplitude +:AHPAmplitude rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "AHP Amplitude"@en ; + "Calculated as the voltage difference between AP threshold and AP trough. Commonly defined using first AP in train at rheobase current."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AHPAmplitudeFromResting +:AHPAmplitudeFromResting rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "AHP Amplitude From Resting"@en ; + "Calculated as the voltage difference between resting or baseline voltages and AP trough. Commonly defined using first AP in train at rheobase current."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AHPDuration +:AHPDuration rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "AHP Duration"@en ; + "Duration of AP after-hyperpolarization, not explicitly referred to by author as either fast or slow"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AHPVoltage +:AHPVoltage rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "AHP Voltage"@en ; + "Calculated as minimum voltage value during an AHP. Commonly defined using first AP in train at rheobase current."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AccessResistance +:AccessResistance rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Access Resistance"@en ; + "Sum of the electrode resistance and the resistance at the electrode-cell junction"@en ; + "M\\ohm" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationPercent1 +:AdaptationPercent1 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Percent1"@en ; + "adaptation percent (first/last ISI)"@en ; + "Percentage of durations between early and late AP inter-spike intervals in an AP train"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationPercent2 +:AdaptationPercent2 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Percent2"@en ; + "adaptation percent (last/first ISI)"@en ; + "Percentage of durations between late and early AP inter-spike intervals in an AP train"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationPercent3 +:AdaptationPercent3 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Percent3"@en ; + "adaptation percent (1 – first/last ISI)"@en ; + "1 minus ratio of durations between early and late AP inter-spike intervals in an AP train, normalized to a percent"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio +:AdaptationRatio rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Ratio"@en ; + "Ratio of durations between early and late AP inter-spike intervals in an AP train"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio1 +:AdaptationRatio1 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Ratio1"@en ; + "adaptation ratio (last/first ISI)"@en ; + "Ratio of durations between late and early AP inter-spike intervals in an AP train"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio2 +:AdaptationRatio2 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Ratio2"@en ; + "adaptation ratio (1 – first/last ISI)"@en ; + "1 minus ratio of durations between early and late AP inter-spike intervals in an AP train"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio3 +:AdaptationRatio3 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Adaptation Ratio3"@en ; + "adaptation ratio (other)"@en ; + "Adaptation ratio, percent, index but not otherwise characterizable to an existing sub-definition"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/CellCapacitance +:CellCapacitance rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Cell Capacitance"@en ; + "Neuron capacitance, typically measured by dividing membrane time constant by membrane resistance"@en ; + "pF" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/CellDiameter +:CellDiameter rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Cell Diameter"@en ; + "Diameter of the cell soma"@en ; + "um" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/CellSurfaceArea +:CellSurfaceArea rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Cell Surface Area"@en ; + "Cross-sectional area of the cell"@en ; + "m" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FISlope +:FISlope rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "FI Slope"@en ; + "Slope of the frequency-current relationship"@en ; + "Hz/nA" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPAmplitude +:FastAHPAmplitude rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Fast AHP Amplitude"@en ; + "Calculated as the difference between AP threshold and AP trough, explictly refered to by author as fast."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPAmplitudeFromResting +:FastAHPAmplitudeFromResting rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Fast AHP Amplitude From Resting"@en ; + "Calculated as the difference between resting or baseline voltages and AP trough, explictly refered to by author as fast."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPDuration +:FastAHPDuration rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Fast AHP Duration"@en ; + "Duration of AP after-hyperpolarization, explictly refered to by author as fast"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPVoltage +:FastAHPVoltage rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Fast AHP Voltage"@en ; + "Calculated as the AHP absolute voltage, explictly refered to by author as fast."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FiringFrequency +:FiringFrequency rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Firing Frequency"@en ; + "AP discharge rate, AP frequency, spike rate, firing rate, spike frequency, mean firing rate, steady state firing rate, average firing rate, steady firing rate"@en ; + "Hz" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/FirstSpikeLatency +:FirstSpikeLatency rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "First Spike Latency"@en ; + "Duration to first AP following a depolarizing current step of fixed amplitude"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/InputResistance +:InputResistance rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Input Resistance"@en ; + "Input resistance measured at steady-state voltage response to current injection"@en ; + "M\\ohm" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/MaximumFiringRate +:MaximumFiringRate rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Maximum Firing Rate"@en ; + "Maximum observed AP discharge rate"@en ; + "Hz" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPAmplitude +:MediumAHPAmplitude rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Medium AHP Amplitude"@en ; + "Calculated as the difference between AP threshold and AP trough, explictly refered to by author as medium"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPAmplitudeFromResting +:MediumAHPAmplitudeFromResting rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Medium AHP Amplitude From Resting"@en ; + "Calculated as the difference between resting or baseline voltages and AP trough, explictly refered to by author as medium"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPDuration +:MediumAHPDuration rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Medium AHP Duration"@en ; + "Duration of AP after-hyperpolarization, explictly refered to by author as medium"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPVoltage +:MediumAHPVoltage rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Medium AHP Voltage"@en ; + "Calculated as the AHP absolute voltage, explictly refered to by author as medium."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/MembraneTimeConstant +:MembraneTimeConstant rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Membrane Time Constant"@en ; + "Time constant for the membrane to repolarize after a small current injection of fixed amplitude and duration"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/NeuroElectroNeuronElectrophysiologicalFeature +:NeuroElectroNeuronElectrophysiologicalFeature rdf:type owl:Class ; + rdfs:label "NeuroElectro Neuron Electrophysiological Feature"@en . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/Other +:Other rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Other"@en ; + "A catch-all property to reflect any properties which are not already explicitly defined"@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/RestingMembranePotential +:RestingMembranePotential rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Resting Membrane Potential"@en ; + "Membrane potential at the onset of whole-cell recording"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/Rheobase +:Rheobase rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Rheobase"@en ; + "Minimum current injected somatically required to fire AP"@en ; + "pA" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SagAmplitude2 +:SagAmplitude2 rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Sag Amplitude2"@en ; + "Absolute difference between the steady state decrease in the voltage and the largest decrease in voltage following a hyperpolarizing current step."^^xsd:string ; + "mV"^^xsd:string . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SagRatio +:SagRatio rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Sag Ratio"@en ; + "Ratio between the steady state decrease in the voltage and the largest decrease in voltage following a hyperpolarizing current step."@en ; + "Unitless" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPAmplitude +:SlowAHPAmplitude rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Slow AHP Amplitude"@en ; + "Calculated as the difference between AP threshold and AP trough, explictly refered to by author as slow"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPAmplitudeFromResting +:SlowAHPAmplitudeFromResting rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Slow AHP Amplitude From Resting"@en ; + "Calculated as the difference between resting or baseline voltages and AP trough, explictly refered to by author as slow"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPDuration +:SlowAHPDuration rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Slow AHP Duration"@en ; + "Duration of AP after-hyperpolarization, explictly refered to by author as slow"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPVoltage +:SlowAHPVoltage rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Slow AHP Voltage"@en ; + "Calculated as the AHP absolute voltage, explictly refered to by author as slow."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeAmplitude +:SpikeAmplitude rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Amplitude"@en ; + "Voltage indicating height of action potential. Usually calculated as the difference between AP peak and AP threshold voltages. Commonly measured using first AP in train at rheobase current."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeAmplitudeFromResting +:SpikeAmplitudeFromResting rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Amplitude From Resting"@en ; + "Voltage indicating height of action potential, calculated as the difference between AP peak and resting or baseline voltages. Commonly measured using first AP in train at rheobase current."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeAmplitudeFromTrough +:SpikeAmplitudeFromTrough rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Amplitude From Trough"@en ; + "Voltage indicating height of action potential, calculated as the difference between AP peak and AHP trough voltages. Commonly measured using first AP in train at rheobase current."@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeDecayTime +:SpikeDecayTime rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Decay Time"@en ; + "Time for spike to fall from peak to threshold, usually calculated as 10-90% decay time"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeHalf-Width +:SpikeHalf-Width rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Half-Width"@en ; + "Calculated as the AP duration at the membrane voltage halfway between AP threshold and AP peak. Most commonly calculated using first AP in train at rheobase. current."@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeMaxDecaySlope +:SpikeMaxDecaySlope rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Max Decay Slope"@en ; + "Maximum rate of rise of membrane voltage during spike falling phase"@en ; + "mV/ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeMaxRiseSlope +:SpikeMaxRiseSlope rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Max Rise Slope"@en ; + "Maximum rate of rise of membrane voltage during spike rising phase"@en ; + "mV/ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikePeak +:SpikePeak rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Peak"@en ; + "Maximum voltage reached during AP"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeRiseTime +:SpikeRiseTime rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Rise Time"@en ; + "Time for spike to rise from threshold to peak, usually calculated as 10-90% rise time"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeThreshold +:SpikeThreshold rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Threshold"@en ; + "Voltage at which AP is initiated (as assessed by measuring rising slope of membrane voltage)"@en ; + "mV" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeWidth +:SpikeWidth rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spike Width"@en ; + "Duration of AP, not explictly refered to as half-width"@en ; + "ms" . + + +### https://bbp.epfl.ch/ontologies/core/efeatures/SpontaneousFiringRate +:SpontaneousFiringRate rdf:type owl:Class ; + rdfs:subClassOf , + :NeuroElectroNeuronElectrophysiologicalFeature ; + rdfs:label "Spontaneous Firing Rate"@en ; + "AP discharge rate in the absence of current injection or a stimulus"@en ; + "Hz" . + + +### Generated by the OWL API (version 4.5.9.2019-02-01T07:24:44Z) https://github.com/owlcs/owlapi diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index b2887afc..ba0a8627 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -38,11 +38,24 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "myname UniProt\n", + "store_config {'name': 'DemoStore'}\n", + "model_config {'origin': 'directory', 'source': '../../../tests/data/demo-model/'}\n", + "myname NeuroElectro\n", + "store_config {'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'}\n", + "model_config {'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}\n" + ] + } + ], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"neurosciencegraph/datamodels\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", bucket=BUCKET)" + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", endpoint=endpoint, bucket=BUCKET)" ] }, { @@ -56,33 +69,43 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [], + "source": [ + "sources = forge.db_sources(pretty=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Database sources with managed mappings:\n", - "UniProt\n", - "NeuroElectro\n" - ] + "data": { + "text/plain": [ + "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, _from_forge=True, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='UniProt', store={'name': 'DemoStore'}),\n", + " 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', _forge=, _from_forge=True, _inner_sync=False, _model=RdfModel(service=, source='BlueBrainNexus'), _store=BlueBrainNexus(context=, bucket='neurosciencegraph/datamodels', endpoint='https://staging.nise.bbp.epfl.ch/nexus/v1', file_mapping=DictionaryMapping(rules=OrderedDict([('type', 'DataDownload'), ('contentSize', OrderedDict([('unitCode', 'f\"bytes\"'), ('value', 'x._bytes')])), ('digest', OrderedDict([('algorithm', 'x._digest._algorithm'), ('value', 'x._digest._value')])), ('encodingFormat', 'x._mediaType'), ('name', 'x._filename'), ('contentUrl', 'x._self'), ('atLocation', OrderedDict([('type', 'Location'), ('store', OrderedDict([('id', 'x._storage[\"@id\"]'), ('type', 'x._storage[\"@type\"] if \\'@type\\' in x._storage else None'), ('_rev', 'x._storage[\"_rev\"] if \\'_rev\\' in x._storage else None')])), ('location', \"x._location if '_location' in x else None\")]))])), metadata_context=, model_context=, organisation='neurosciencegraph', project='datamodels', service=, token='eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', versioned_id_template='{x.id}?rev={x._store_metadata._rev}'), model={'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}, name='NeuroElectro', store={'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'})}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "sources = forge.db_sources(pretty=True)" + "sources" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "\n", "data = {\n", - " 'origin': 'store',\n", - " 'source': 'MouseLight', \n", - " 'description': 'The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.',\n", - " 'url': 'https://www.janelia.org/project-team/mouselight/neuronbrowser',\n", + " 'store':{\n", + " 'name': 'DemoStore'\n", + " },\n", " 'protocol': 'https://www.janelia.org/project-team/mouselight/resources', \n", " 'license': [{'id': 'https://creativecommons.org/licenses/by-nc/4.0', \n", " 'label': 'CC BY-NC 4.0'}\n", @@ -90,15 +113,30 @@ " 'definition': {\n", " 'origin': 'directory',\n", " 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'\n", + " },\n", + " 'model': { \n", + " 'name': 'DemoModel',\n", + " 'origin': 'directory',\n", + " 'source': \"../../../tests/data/demo-model/\" \n", " }\n", - " }\n" + "}\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "myname MouseLight\n", + "store_config {'name': 'DemoStore'}\n", + "model_config {'origin': 'directory', 'source': '../../../tests/data/demo-model/'}\n" + ] + } + ], "source": [ "from kgforge.specializations.resources import DatabaseSource\n", "ds = DatabaseSource(forge, name=\"MouseLight\", from_forge=False, **data)" @@ -106,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -115,12 +153,28 @@ "text": [ "{\n", " type: Database\n", + " _store:\n", + " {\n", + " context: null\n", + " bucket: null\n", + " endpoint: null\n", + " file_mapping: null\n", + " metadata_context: null\n", + " model_context: null\n", + " service:\n", + " {\n", + " archives: {}\n", + " records: {}\n", + " tags: {}\n", + " }\n", + " token: null\n", + " versioned_id_template: null\n", + " }\n", " definition:\n", " {\n", " origin: directory\n", " source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/\n", " }\n", - " description: The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", " license:\n", " [\n", " {\n", @@ -128,11 +182,17 @@ " label: CC BY-NC 4.0\n", " }\n", " ]\n", + " model:\n", + " {\n", + " origin: directory\n", + " source: ../../../tests/data/demo-model/\n", + " }\n", " name: MouseLight\n", - " origin: store\n", " protocol: https://www.janelia.org/project-team/mouselight/resources\n", - " source: MouseLight\n", - " url: https://www.janelia.org/project-team/mouselight/neuronbrowser\n", + " store:\n", + " {\n", + " name: DemoStore\n", + " }\n", "}\n" ] } @@ -143,14 +203,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _forge=, _from_forge=True, _inner_sync=False, definition={'origin': 'directory', 'source': '/Users/mfsy/dev/apps/forked/nexus-forge/examples/notebooks/use-cases/datasources', 'iri': 'UniProt'}, endpoint='https://sparql.uniprot.org/sparql', name='UniProt', origin='store', source='SPARQLStore'), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _forge=, _from_forge=True, _inner_sync=False, bucket='bbp/neuroelectro', definition={'iri': 'nexus_resource_id'}, name='NeuroElectro', origin='store', source='BlueBrainNexus'), 'MouseLight': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight', _forge=, _from_forge=False, _inner_sync=False, definition={'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'}, description='The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.', license=[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}], name='MouseLight', origin='store', protocol='https://www.janelia.org/project-team/mouselight/resources', source='MouseLight', url='https://www.janelia.org/project-team/mouselight/neuronbrowser')}\n" + "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, _from_forge=True, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='UniProt', store={'name': 'DemoStore'}), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', _forge=, _from_forge=True, _inner_sync=False, _model=RdfModel(service=, source='BlueBrainNexus'), _store=BlueBrainNexus(context=, bucket='neurosciencegraph/datamodels', endpoint='https://staging.nise.bbp.epfl.ch/nexus/v1', file_mapping=DictionaryMapping(rules=OrderedDict([('type', 'DataDownload'), ('contentSize', OrderedDict([('unitCode', 'f\"bytes\"'), ('value', 'x._bytes')])), ('digest', OrderedDict([('algorithm', 'x._digest._algorithm'), ('value', 'x._digest._value')])), ('encodingFormat', 'x._mediaType'), ('name', 'x._filename'), ('contentUrl', 'x._self'), ('atLocation', OrderedDict([('type', 'Location'), ('store', OrderedDict([('id', 'x._storage[\"@id\"]'), ('type', 'x._storage[\"@type\"] if \\'@type\\' in x._storage else None'), ('_rev', 'x._storage[\"_rev\"] if \\'_rev\\' in x._storage else None')])), ('location', \"x._location if '_location' in x else None\")]))])), metadata_context=, model_context=, organisation='neurosciencegraph', project='datamodels', service=, token='eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', versioned_id_template='{x.id}?rev={x._store_metadata._rev}'), model={'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}, name='NeuroElectro', store={'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'}), 'MouseLight': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight', _forge=, _from_forge=False, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), definition={'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'}, license=[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}], model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='MouseLight', protocol='https://www.janelia.org/project-team/mouselight/resources', store={'name': 'DemoStore'})}\n" ] } ], @@ -167,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -176,12 +236,28 @@ "text": [ "{\n", " type: Database\n", + " _store:\n", + " {\n", + " context: null\n", + " bucket: null\n", + " endpoint: null\n", + " file_mapping: null\n", + " metadata_context: null\n", + " model_context: null\n", + " service:\n", + " {\n", + " archives: {}\n", + " records: {}\n", + " tags: {}\n", + " }\n", + " token: null\n", + " versioned_id_template: null\n", + " }\n", " definition:\n", " {\n", " origin: directory\n", " source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/\n", " }\n", - " description: The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", " license:\n", " [\n", " {\n", @@ -189,11 +265,17 @@ " label: CC BY-NC 4.0\n", " }\n", " ]\n", + " model:\n", + " {\n", + " origin: directory\n", + " source: ../../../tests/data/demo-model/\n", + " }\n", " name: MouseLight\n", - " origin: store\n", " protocol: https://www.janelia.org/project-team/mouselight/resources\n", - " source: MouseLight\n", - " url: https://www.janelia.org/project-team/mouselight/neuronbrowser\n", + " store:\n", + " {\n", + " name: DemoStore\n", + " }\n", "}\n" ] } @@ -212,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -220,8 +302,6 @@ "output_type": "stream", "text": [ "MouseLight\n", - "The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain.\n", - "https://www.janelia.org/project-team/mouselight/neuronbrowser\n", "https://www.janelia.org/project-team/mouselight/resources\n", "[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}]\n" ] @@ -229,8 +309,6 @@ ], "source": [ "print(mouselight.name)\n", - "print(mouselight.description)\n", - "print(mouselight.url)\n", "print(mouselight.protocol)\n", "print(mouselight.license)" ] @@ -247,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -256,7 +334,7 @@ "{'NeuronMorphology': ['DictionaryMapping']}" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -265,10 +343,56 @@ "forge.mappings(\"MouseLight\", pretty=False)" ] }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Managed mappings for the data source per entity type and mapping type:\n", + " - NeuronElectrophysiologicalFeature:\n", + " * DictionaryMapping\n" + ] + } + ], + "source": [ + "forge.mappings('UniProt', pretty=True)" + ] + }, { "cell_type": "code", "execution_count": 13, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Managed mappings for the data source per entity type and mapping type:\n", + " - ElectrophysiologicalFeatureAnnotation:\n", + " * DictionaryMapping\n", + " - ParameterAnnotation:\n", + " * DictionaryMapping\n", + " - ParameterBody:\n", + " * DictionaryMapping\n", + " - ScholarlyArticle:\n", + " * DictionaryMapping\n", + " - SeriesBody:\n", + " * DictionaryMapping\n" + ] + } + ], + "source": [ + "forge.mappings('NeuroElectro', pretty=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, "outputs": [], "source": [ "from kgforge.specializations.mappings import DictionaryMapping\n", @@ -278,7 +402,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -368,7 +492,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -456,6 +580,43 @@ "print(direct_mapping)" ] }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available Database sources:\n", + "MouseLight\n" + ] + } + ], + "source": [ + "forge.db_sources(with_datatype='NeuronMorphology', pretty=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "props = {'origin': 'store', 'source': 'BlueBrainNexus', 'definition':{'iri': 'some_address'}}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# props = {'origin': 'store', 'source': 'BlueBrainNexus', 'definition':{'iri':}}\n", + "# new_db = DatabaseSource(forge, from_forge=False, name='new_db')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -490,7 +651,7 @@ "0" ] }, - "execution_count": 14, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -511,7 +672,7 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_5900/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_29724/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mIndexError\u001b[0m: list index out of range" ] } diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 82790034..65956b61 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -228,6 +228,7 @@ def __init__(self, configuration: Union[str, Dict], **kwargs) -> None: model_name = model_config.pop("name") model = import_class(model_name, "models") self._model: Model = model(**model_config) + model_config.update(name=model_name) # Store. store_config.update(model_context=self._model.context()) @@ -252,8 +253,9 @@ def __init__(self, configuration: Union[str, Dict], **kwargs) -> None: ======= # Database sources. db_sources_config: Optional[Dict[str, Dict[str, str]]] = config.pop("DatabaseSources", None) + self._db_sources: Union[DatabaseSource, List[DatabaseSource]] = ( - self.create_db_sources(db_sources_config) + self.create_db_sources(db_sources_config, store_config, model_config) if db_sources_config else None ) @@ -529,16 +531,32 @@ def sources(self, pretty: bool = True) -> Optional[List[str]]: return self._model.sources(pretty) @catch - def db_sources(self, pretty: bool = False) -> Optional[List[str]]: + def db_sources(self, with_datatype: Optional[List[str]] = None, + pretty: bool = False) -> Optional[List[str]]: """ Print(pretty=True) or return (pretty=False) configured data sources. :param pretty: a boolean :return: Optional[List[str]] """ - sources = self._db_sources + if with_datatype is not None: + sources = {} + if isinstance(with_datatype, list): + for type in with_datatype: + for db in self._db_sources: + if type in self._db_sources[db].datatypes(): + sources[db] = self._db_sources[db] + else: + for db in self._db_sources: + if with_datatype in self._db_sources[db].datatypes(): + sources[db] = self._db_sources[db] + if not sources: + print("No Database sources were found for the given datatype(s)") + else: + sources = self._db_sources if pretty: - print(*["Database sources with managed mappings:", *sources], sep="\n") - return sources + print(*["Available Database sources:", *sources], sep="\n") + else: + return sources def add_db_source(self, db_source: DatabaseSource) -> None: """ @@ -974,9 +992,23 @@ def get_model_context(self): """Expose the context used in the model.""" return self._model.context() - def create_db_sources(self, config: Optional[Dict[str, Dict[str, str]]]) -> Union[DatabaseSource, List[DatabaseSource]]: - names = config.keys() - return {name: DatabaseSource(self, name=name, from_forge=True, **config[name]) for name in names} + def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], + store_config : Optional[Dict[str, Dict[str, str]]], + model_config : Optional[Dict[str, Dict[str, str]]] + ) -> Union[DatabaseSource, List[DatabaseSource]]: + names = all_config.keys() + dbs = {} + for name in names: + config = all_config[name] + # Provide store and model configuration to the database sources + if "model" not in config: + config.update(model=model_config) + + # Complete configuration of the db store in case is the same + if config['store']['name'] == store_config['name']: + config['store'] = store_config + dbs[name] = DatabaseSource(self, name=name, from_forge=True, **config) + return dbs def prepare_resolvers( diff --git a/kgforge/specializations/resources/db_sources.py b/kgforge/specializations/resources/db_sources.py index 521903a7..435ee873 100644 --- a/kgforge/specializations/resources/db_sources.py +++ b/kgforge/specializations/resources/db_sources.py @@ -15,26 +15,30 @@ import json from pathlib import Path, PurePath from re import I +import copy from typing import Callable, Optional, Union, Dict, List from kgforge.core import Resource -from kgforge.core.archetypes import Mapping +from kgforge.core.archetypes import Mapping, Store, Model +from kgforge.core.commons.execution import not_supported +from kgforge.core.commons.dictionaries import with_defaults +from kgforge.core.commons.imports import import_class # Get local paths pathparts = list(Path(__file__).resolve().parts) KGFORGE_PATH = os.path.join(*pathparts[:-4]) -DBS_PATH = os.path.join(KGFORGE_PATH, "examples", "database_sources") +DBS_PATH = Path(KGFORGE_PATH, "examples", "database_sources") class DatabaseSource(Resource): """A high-level class for Database-related Resources.""" _DBDIR = Path - _REQUIRED = ("name", "origin", "source", "definition") + _REQUIRED = ("name", "store", "model") _RESERVED = {"_forge", "_from_forge", "_check_properties", "_save_config", "_dirpath", - "_mappings", "mappings"} | Resource._RESERVED + "health", "_mappings", "mappings", "mapping", "dump_config", "_model", "._store"} | Resource._RESERVED - def __init__(self, forge: Optional["KnowledgeGRaphForge"], type: str = "Database", + def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Database", from_forge: bool = True, **properties) -> None: """ The properties defining the Databasesource in YAML are: @@ -46,7 +50,7 @@ def __init__(self, forge: Optional["KnowledgeGRaphForge"], type: str = "Database bucket: endpoint: token: bucket: endpoint: @@ -57,9 +61,41 @@ def __init__(self, forge: Optional["KnowledgeGRaphForge"], type: str = "Database super().__init__(**properties) self.type: str = type self._forge: Optional["KnowledgeGraphForge"] = forge + + + store_config = properties.pop('store') + + # Model + model_config = properties.pop("model") + # Assume that the model is a store + # TODO: get configuration from forge._store when using BlueBrainNexus + if model_config["origin"] == "store": + with_defaults( + model_config, + store_config, + "source", + "name", + ["endpoint", "token", "bucket", "vocabulary"], + ) + model_name = model_config.pop("name") + model = import_class(model_name, "models") + self._model: Model = model(**model_config) + + # Store. + store_name = store_config.pop("name") + store = import_class(store_name, "stores") + self._store: Store = store(**store_config) + store_config.update(name=store_name) + self._from_forge = from_forge + self._dirpath = os.path.join(DBS_PATH, self.name) if self._from_forge is False: + # Save in directory and add it to forge instance self._save_config() + else: + if not Path(self._dirpath).is_dir(): + raise ValueError(f"Database directory for {self.name} was not found.\ + To create a new database use from_forge=False") def _check_properties(self, **info): properties = info.keys() @@ -69,13 +105,19 @@ def _check_properties(self, **info): def _save_config(self) -> None: """Save database information inside the kgforge database folder.""" - self._dirpath = os.path.join(DBS_PATH, self.name) - Path(self._dirpath).mkdir(parents=True, exist_ok=True) # Make mappings directory + Path(self._dirpath).mkdir(parents=True, exist_ok=True) Path(self._dirpath, 'mappings').mkdir(parents=True, exist_ok=True) # Add the source to forge self._forge.add_db_source(self) + def datatypes(self): + # TODO: add other datatypes used, for instance, inside the mappings + return self.mappings(pretty=False).keys() + + def _model(self) -> None: + not_supported() + def _mappings(self) -> Dict[str, List[str]]: dirpath = Path(self._dirpath, "mappings") mappings = {} @@ -103,4 +145,12 @@ def mapping(self, entity: str, type: Callable) -> Mapping: return type.load(filepath) else: raise ValueError("unrecognized entity type or source") - \ No newline at end of file + + def dump_config(self) -> None: + filename = "config.json" + filepath = Path(self._dirpath, filename) + with open(filepath, 'w') as mfile: + return mfile.write(json.dumps(self._forge.as_json(self), indent=4)) + + def health(self): + not_supported() \ No newline at end of file From 1da88838855b0081121b49304a1c4ec2c17dca3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 5 Oct 2022 17:40:46 +0200 Subject: [PATCH 05/30] Updated notebook --- .../17 - Database-sources.ipynb | 90 +++++-------------- 1 file changed, 23 insertions(+), 67 deletions(-) diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index ba0a8627..df807847 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -38,20 +38,7 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "myname UniProt\n", - "store_config {'name': 'DemoStore'}\n", - "model_config {'origin': 'directory', 'source': '../../../tests/data/demo-model/'}\n", - "myname NeuroElectro\n", - "store_config {'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'}\n", - "model_config {'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}\n" - ] - } - ], + "outputs": [], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"neurosciencegraph/datamodels\"\n", @@ -78,27 +65,6 @@ "cell_type": "code", "execution_count": 4, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, _from_forge=True, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='UniProt', store={'name': 'DemoStore'}),\n", - " 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', _forge=, _from_forge=True, _inner_sync=False, _model=RdfModel(service=, source='BlueBrainNexus'), _store=BlueBrainNexus(context=, bucket='neurosciencegraph/datamodels', endpoint='https://staging.nise.bbp.epfl.ch/nexus/v1', file_mapping=DictionaryMapping(rules=OrderedDict([('type', 'DataDownload'), ('contentSize', OrderedDict([('unitCode', 'f\"bytes\"'), ('value', 'x._bytes')])), ('digest', OrderedDict([('algorithm', 'x._digest._algorithm'), ('value', 'x._digest._value')])), ('encodingFormat', 'x._mediaType'), ('name', 'x._filename'), ('contentUrl', 'x._self'), ('atLocation', OrderedDict([('type', 'Location'), ('store', OrderedDict([('id', 'x._storage[\"@id\"]'), ('type', 'x._storage[\"@type\"] if \\'@type\\' in x._storage else None'), ('_rev', 'x._storage[\"_rev\"] if \\'_rev\\' in x._storage else None')])), ('location', \"x._location if '_location' in x else None\")]))])), metadata_context=, model_context=, organisation='neurosciencegraph', project='datamodels', service=, token='eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', versioned_id_template='{x.id}?rev={x._store_metadata._rev}'), model={'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}, name='NeuroElectro', store={'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'})}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sources" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, "outputs": [], "source": [ "\n", @@ -124,19 +90,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "myname MouseLight\n", - "store_config {'name': 'DemoStore'}\n", - "model_config {'origin': 'directory', 'source': '../../../tests/data/demo-model/'}\n" - ] - } - ], + "outputs": [], "source": [ "from kgforge.specializations.resources import DatabaseSource\n", "ds = DatabaseSource(forge, name=\"MouseLight\", from_forge=False, **data)" @@ -144,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -203,14 +159,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, _from_forge=True, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='UniProt', store={'name': 'DemoStore'}), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', _forge=, _from_forge=True, _inner_sync=False, _model=RdfModel(service=, source='BlueBrainNexus'), _store=BlueBrainNexus(context=, bucket='neurosciencegraph/datamodels', endpoint='https://staging.nise.bbp.epfl.ch/nexus/v1', file_mapping=DictionaryMapping(rules=OrderedDict([('type', 'DataDownload'), ('contentSize', OrderedDict([('unitCode', 'f\"bytes\"'), ('value', 'x._bytes')])), ('digest', OrderedDict([('algorithm', 'x._digest._algorithm'), ('value', 'x._digest._value')])), ('encodingFormat', 'x._mediaType'), ('name', 'x._filename'), ('contentUrl', 'x._self'), ('atLocation', OrderedDict([('type', 'Location'), ('store', OrderedDict([('id', 'x._storage[\"@id\"]'), ('type', 'x._storage[\"@type\"] if \\'@type\\' in x._storage else None'), ('_rev', 'x._storage[\"_rev\"] if \\'_rev\\' in x._storage else None')])), ('location', \"x._location if '_location' in x else None\")]))])), metadata_context=, model_context=, organisation='neurosciencegraph', project='datamodels', service=, token='eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', versioned_id_template='{x.id}?rev={x._store_metadata._rev}'), model={'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}, name='NeuroElectro', store={'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjQ5ODE0MDEsImlhdCI6MTY2NDk2OTU3OCwiYXV0aF90aW1lIjoxNjY0OTUyNjAxLCJqdGkiOiIyZGRhOWRiZS1mNWUyLTQyNmMtYTRhMi02ZTMwYTNkN2JjNTEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwNmJlYTU4MTczYzQ2YTdiYzI0NjM1YjAwMzZkYTczIiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1c2VyIiwiZGVmYXVsdC1yb2xlcy1iYnAiXX0sInJlc291cmNlX2FjY2VzcyI6eyJodHRwczovL3NsYWNrLmNvbSI6eyJyb2xlcyI6WyJzbGFja191c2VyX3JvbGUiXX19LCJzY29wZSI6Im9wZW5pZCBuZXh1cyBwcm9maWxlIGxvY2F0aW9uIGVtYWlsIiwic2lkIjoiNzZkNWE0NzEtZTVjMi00ZWZmLWFhNWYtZmRjMjM0M2Y2Y2Y4IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXogRXNwaW5vemEiLCJncm91cHMiOlsiL2JicC1kZXYtcHJvajEwOSIsIi9iYnAtZGtlLWRldiIsIi9iYnAtZGV2LXByb2o2NCIsIi9iYnAtZGV2LXByb2oxMTYiLCIvYmJwLW91LWRrZSIsIi9iYnAtZGV2LXByb2o4NCIsIi9iYnAtZGtlLXN0YWdpbmciLCIvYmJwLXVzZXItcHJvajM5IiwiL2JicC11c2VyLXByb2o2NCIsIi9iYnAtb3UtbmV1cm9pbmZvcm1hdGljcyIsIi9iYnAtdXNlci1yZWxlYXNlLWwyIiwiL2JicC1vdS1uZXh1cyIsIi9iYnAtdXNlci1jb3JlYmx1cm9uIiwiL2JicC1kZXYtcHJvajE0MSIsIi9iYnAtZGV2LXByb2oxMTUiLCIvYmJwLW91LXNpbXVsYXRpb25uZXVyb3NjaWVuY2UiLCIvYmJwLWRzLWV4cGVyaW1lbnQiLCIvYmJwLXN2Yy1seHZpeiIsIi9iYnAtdXNlci1wcm9qMTA2IiwiL2JicC1zdmMtdml6IiwiL2JicC11c2VyLXJlbGVhc2UtbDEiLCIvYmJwLWRldi1wcm9qNTUiLCIvYmJwLWRldi1wcm9qMTM0IiwiL2JicC1mdWxsIiwiL2JicC1zdmMtZ2l0bGFiIiwiL2JicC1jb2xsYWItbGlua2FibGUtYWNjb3VudHMiLCIvYmJwLWRldi1yZWZpbmVtZW50cHJvcCIsIi9iYnAtZGV2LXByb2o4MyIsIi9iYnAtaXRjLWRhdGFpbnRlZ3JhdGlvbiIsIi9iYnAtZGV2LXByb2oxMDUiLCIvYmJwLXN2Yy1iZzQiLCIvYmJwLWRldi1wcm9qODIiLCIvYmJwLXNydi1saW5zcnYyIiwiL2JicC11c2VyLXByb2o5NCIsIi9iYnAtZGV2LXByb2o3MiIsIi9iYnAtdXNlci1wcm9qMTIzIiwiL2JicC1kcy1ocGMiLCIvYmJwLWRrZS1wcm9kdWN0aW9uIiwiL2JicC11c2VyLXByb2oxMTYiLCIvYmJwLWRldi1wcm9qMTM2IiwiL2JicC11c2VyLXJlbGVhc2UtbDAiLCIvYmJwX2NvbmZsdWVuY2UiLCIvYmJwLW91LWVwZmwiLCIvYmJwLXVzZXItcHJvajExNCIsIi9iYnAtc3ZjLWhwYyIsIi9iYnAtdXNlci1wcm9qNDIiLCIvYmJwLXN2Yy1jb2RlIiwiL2JicC1kZXYtcHJvajEzMiIsIi9iYnAtc3ZjLXNsYWNrIiwiL2JicC1uZXh1cy1kZXYiLCIvYmJwLXN0YWZmIiwiL2JicC1kZXYtaGJwdml6IiwiL2JicC1kZXYtcHJvamVjdHByb3AiLCIvYmJwLXN2Yy1rdWJlcm5ldGVzIiwiL2JicC1zdmMtc3RhdHVzIiwiL2JicC1wdWJsaWNhdGlvbnMiLCIvYmJwLXVzZXItcHJvajMiXSwibG9jYXRpb24iOiJCMSAzIDI4NC4wNTIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJjZ29uemFsZSIsImdpdmVuX25hbWUiOiJDcmlzdGluYSBFbGl6YWJldGggR29uemFsZXoiLCJmYW1pbHlfbmFtZSI6IkVzcGlub3phIiwiZW1haWwiOiJjcmlzdGluYS5nb256YWxlemVzcGlub3phQGVwZmwuY2gifQ.EF7dOKxyJwlUTLW6A8rIZlmo-NEObynMXJFE_46LwBEzwU1a4zXezCqi9sKchLucBa16c_V4OI2KiwK6Ac0BKBWQfEriVnF2oCdD60PhIn7NMt24LYXhjTbGf3QJLBx8uM0QUkP__PozCTr6UBKCQOTbxivvEO4j3bBQnHfhWiHJPjBCTZg5OrCxN6pwExJkswkrSFLCKS6uhLLKwB043UXNHSG_GTbeDMWuXW0WT4WLFI9DyEKYbYa-4ZRAXg8hidhBHeh6kENFeY5GXUiAeFTvT7zZ_tzAmotDXBr_KUlIHj5Wuvo2X-atCRaW6UVwj-UVtA2v4NeSpRHn_Zz1RQ', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'}), 'MouseLight': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight', _forge=, _from_forge=False, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), definition={'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'}, license=[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}], model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='MouseLight', protocol='https://www.janelia.org/project-team/mouselight/resources', store={'name': 'DemoStore'})}\n" + "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, _from_forge=True, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='UniProt', store={'name': 'DemoStore'}), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', _forge=, _from_forge=True, _inner_sync=False, _model=RdfModel(service=, source='BlueBrainNexus'), _store=BlueBrainNexus(context=, bucket='neurosciencegraph/datamodels', endpoint='https://staging.nise.bbp.epfl.ch/nexus/v1', file_mapping=DictionaryMapping(rules=OrderedDict([('type', 'DataDownload'), ('contentSize', OrderedDict([('unitCode', 'f\"bytes\"'), ('value', 'x._bytes')])), ('digest', OrderedDict([('algorithm', 'x._digest._algorithm'), ('value', 'x._digest._value')])), ('encodingFormat', 'x._mediaType'), ('name', 'x._filename'), ('contentUrl', 'x._self'), ('atLocation', OrderedDict([('type', 'Location'), ('store', OrderedDict([('id', 'x._storage[\"@id\"]'), ('type', 'x._storage[\"@type\"] if \\'@type\\' in x._storage else None'), ('_rev', 'x._storage[\"_rev\"] if \\'_rev\\' in x._storage else None')])), ('location', \"x._location if '_location' in x else None\")]))])), metadata_context=, model_context=, organisation='neurosciencegraph', project='datamodels', service=, token='eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjUwMTE2NjEsImlhdCI6MTY2NDk4NDEwMywiYXV0aF90aW1lIjoxNjY0OTgyODYxLCJqdGkiOiI5NzJiMGEwNi01ZGQ1LTRmMzAtYWE1OC04ZjhhY2EwOTViMTYiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6ImYwYzgyYzJiNDBlODQ4NzdiNzE3YjQyMDQzOWE2YjA2Iiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.KNaQQ6jTSddkHPoNyNra-4K7ZWErsQ6oP8E1XyGEKmlIjTVgkHlDM4kKNvd3x21w3TaANFlrGHG-UUUazcRZHphDUuS66nPNjHvqvuxzvWA9qSi4nNoxlp8nfAjfYoWO18y-uVy4shcIkcsMPNU3NG6ojVZuWoyceoy0mHnVHwGqr3JMWBfezTND2-5HUehHcFVHu-D4Ajzyw8NB96-9QUpc2Di1_HM_oc4_42VZtLSKjp6qne0KzYNHHUm6cUUENmreLZ-PzkkhNqribXF1vK6cFE30Zj_PiPDdRDwnUibYp2iqlqrdMY_8f2NjlrdD0sHKaP3ppiGPVsJE7YJpUg', versioned_id_template='{x.id}?rev={x._store_metadata._rev}'), model={'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjUwMTE2NjEsImlhdCI6MTY2NDk4NDEwMywiYXV0aF90aW1lIjoxNjY0OTgyODYxLCJqdGkiOiI5NzJiMGEwNi01ZGQ1LTRmMzAtYWE1OC04ZjhhY2EwOTViMTYiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6ImYwYzgyYzJiNDBlODQ4NzdiNzE3YjQyMDQzOWE2YjA2Iiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.KNaQQ6jTSddkHPoNyNra-4K7ZWErsQ6oP8E1XyGEKmlIjTVgkHlDM4kKNvd3x21w3TaANFlrGHG-UUUazcRZHphDUuS66nPNjHvqvuxzvWA9qSi4nNoxlp8nfAjfYoWO18y-uVy4shcIkcsMPNU3NG6ojVZuWoyceoy0mHnVHwGqr3JMWBfezTND2-5HUehHcFVHu-D4Ajzyw8NB96-9QUpc2Di1_HM_oc4_42VZtLSKjp6qne0KzYNHHUm6cUUENmreLZ-PzkkhNqribXF1vK6cFE30Zj_PiPDdRDwnUibYp2iqlqrdMY_8f2NjlrdD0sHKaP3ppiGPVsJE7YJpUg', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}, name='NeuroElectro', store={'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjUwMTE2NjEsImlhdCI6MTY2NDk4NDEwMywiYXV0aF90aW1lIjoxNjY0OTgyODYxLCJqdGkiOiI5NzJiMGEwNi01ZGQ1LTRmMzAtYWE1OC04ZjhhY2EwOTViMTYiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6ImYwYzgyYzJiNDBlODQ4NzdiNzE3YjQyMDQzOWE2YjA2Iiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.KNaQQ6jTSddkHPoNyNra-4K7ZWErsQ6oP8E1XyGEKmlIjTVgkHlDM4kKNvd3x21w3TaANFlrGHG-UUUazcRZHphDUuS66nPNjHvqvuxzvWA9qSi4nNoxlp8nfAjfYoWO18y-uVy4shcIkcsMPNU3NG6ojVZuWoyceoy0mHnVHwGqr3JMWBfezTND2-5HUehHcFVHu-D4Ajzyw8NB96-9QUpc2Di1_HM_oc4_42VZtLSKjp6qne0KzYNHHUm6cUUENmreLZ-PzkkhNqribXF1vK6cFE30Zj_PiPDdRDwnUibYp2iqlqrdMY_8f2NjlrdD0sHKaP3ppiGPVsJE7YJpUg', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'}), 'MouseLight': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight', _forge=, _from_forge=False, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), definition={'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'}, license=[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}], model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='MouseLight', protocol='https://www.janelia.org/project-team/mouselight/resources', store={'name': 'DemoStore'})}\n" ] } ], @@ -227,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -294,7 +250,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -325,7 +281,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -334,7 +290,7 @@ "{'NeuronMorphology': ['DictionaryMapping']}" ] }, - "execution_count": 11, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -345,7 +301,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -364,7 +320,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -391,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -402,7 +358,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -492,7 +448,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -582,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -600,7 +556,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -609,7 +565,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -629,7 +585,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -642,7 +598,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -651,7 +607,7 @@ "0" ] }, - "execution_count": 29, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -662,7 +618,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -672,7 +628,7 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_29724/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_39476/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mIndexError\u001b[0m: list index out of range" ] } From 9b3c88ad52669341982de6106abd4ec18fa4f0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 5 Oct 2022 17:45:05 +0200 Subject: [PATCH 06/30] Did a little clean-up of the notebook --- .../17 - Database-sources.ipynb | 68 ++++++++++++------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index df807847..7a1db016 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -56,6 +56,25 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available Database sources:\n", + "UniProt\n", + "NeuroElectro\n" + ] + } + ], + "source": [ + "forge.db_sources(pretty=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, "outputs": [], "source": [ "sources = forge.db_sources(pretty=False)" @@ -63,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -100,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -159,19 +178,22 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'UniProt': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, _from_forge=True, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='UniProt', store={'name': 'DemoStore'}), 'NeuroElectro': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', _forge=, _from_forge=True, _inner_sync=False, _model=RdfModel(service=, source='BlueBrainNexus'), _store=BlueBrainNexus(context=, bucket='neurosciencegraph/datamodels', endpoint='https://staging.nise.bbp.epfl.ch/nexus/v1', file_mapping=DictionaryMapping(rules=OrderedDict([('type', 'DataDownload'), ('contentSize', OrderedDict([('unitCode', 'f\"bytes\"'), ('value', 'x._bytes')])), ('digest', OrderedDict([('algorithm', 'x._digest._algorithm'), ('value', 'x._digest._value')])), ('encodingFormat', 'x._mediaType'), ('name', 'x._filename'), ('contentUrl', 'x._self'), ('atLocation', OrderedDict([('type', 'Location'), ('store', OrderedDict([('id', 'x._storage[\"@id\"]'), ('type', 'x._storage[\"@type\"] if \\'@type\\' in x._storage else None'), ('_rev', 'x._storage[\"_rev\"] if \\'_rev\\' in x._storage else None')])), ('location', \"x._location if '_location' in x else None\")]))])), metadata_context=, model_context=, organisation='neurosciencegraph', project='datamodels', service=, token='eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjUwMTE2NjEsImlhdCI6MTY2NDk4NDEwMywiYXV0aF90aW1lIjoxNjY0OTgyODYxLCJqdGkiOiI5NzJiMGEwNi01ZGQ1LTRmMzAtYWE1OC04ZjhhY2EwOTViMTYiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6ImYwYzgyYzJiNDBlODQ4NzdiNzE3YjQyMDQzOWE2YjA2Iiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.KNaQQ6jTSddkHPoNyNra-4K7ZWErsQ6oP8E1XyGEKmlIjTVgkHlDM4kKNvd3x21w3TaANFlrGHG-UUUazcRZHphDUuS66nPNjHvqvuxzvWA9qSi4nNoxlp8nfAjfYoWO18y-uVy4shcIkcsMPNU3NG6ojVZuWoyceoy0mHnVHwGqr3JMWBfezTND2-5HUehHcFVHu-D4Ajzyw8NB96-9QUpc2Di1_HM_oc4_42VZtLSKjp6qne0KzYNHHUm6cUUENmreLZ-PzkkhNqribXF1vK6cFE30Zj_PiPDdRDwnUibYp2iqlqrdMY_8f2NjlrdD0sHKaP3ppiGPVsJE7YJpUg', versioned_id_template='{x.id}?rev={x._store_metadata._rev}'), model={'origin': 'store', 'source': 'BlueBrainNexus', 'context': {'iri': 'https://bbp.neuroshapes.org', 'bucket': 'neurosciencegraph/datamodels'}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjUwMTE2NjEsImlhdCI6MTY2NDk4NDEwMywiYXV0aF90aW1lIjoxNjY0OTgyODYxLCJqdGkiOiI5NzJiMGEwNi01ZGQ1LTRmMzAtYWE1OC04ZjhhY2EwOTViMTYiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6ImYwYzgyYzJiNDBlODQ4NzdiNzE3YjQyMDQzOWE2YjA2Iiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.KNaQQ6jTSddkHPoNyNra-4K7ZWErsQ6oP8E1XyGEKmlIjTVgkHlDM4kKNvd3x21w3TaANFlrGHG-UUUazcRZHphDUuS66nPNjHvqvuxzvWA9qSi4nNoxlp8nfAjfYoWO18y-uVy4shcIkcsMPNU3NG6ojVZuWoyceoy0mHnVHwGqr3JMWBfezTND2-5HUehHcFVHu-D4Ajzyw8NB96-9QUpc2Di1_HM_oc4_42VZtLSKjp6qne0KzYNHHUm6cUUENmreLZ-PzkkhNqribXF1vK6cFE30Zj_PiPDdRDwnUibYp2iqlqrdMY_8f2NjlrdD0sHKaP3ppiGPVsJE7YJpUg', 'bucket': 'neurosciencegraph/datamodels', 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}}, name='NeuroElectro', store={'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjUwMTE2NjEsImlhdCI6MTY2NDk4NDEwMywiYXV0aF90aW1lIjoxNjY0OTgyODYxLCJqdGkiOiI5NzJiMGEwNi01ZGQ1LTRmMzAtYWE1OC04ZjhhY2EwOTViMTYiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6ImYwYzgyYzJiNDBlODQ4NzdiNzE3YjQyMDQzOWE2YjA2Iiwic2Vzc2lvbl9zdGF0ZSI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6Ijc2ZDVhNDcxLWU1YzItNGVmZi1hYTVmLWZkYzIzNDNmNmNmOCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.KNaQQ6jTSddkHPoNyNra-4K7ZWErsQ6oP8E1XyGEKmlIjTVgkHlDM4kKNvd3x21w3TaANFlrGHG-UUUazcRZHphDUuS66nPNjHvqvuxzvWA9qSi4nNoxlp8nfAjfYoWO18y-uVy4shcIkcsMPNU3NG6ojVZuWoyceoy0mHnVHwGqr3JMWBfezTND2-5HUehHcFVHu-D4Ajzyw8NB96-9QUpc2Di1_HM_oc4_42VZtLSKjp6qne0KzYNHHUm6cUUENmreLZ-PzkkhNqribXF1vK6cFE30Zj_PiPDdRDwnUibYp2iqlqrdMY_8f2NjlrdD0sHKaP3ppiGPVsJE7YJpUg', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'bucket': 'neurosciencegraph/datamodels', 'model_context': , 'name': 'BlueBrainNexus'}), 'MouseLight': DatabaseSource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight', _forge=, _from_forge=False, _inner_sync=False, _model=DemoModel(service=, source='../../../tests/data/demo-model/'), _store=DemoStore(context=None, bucket=None, endpoint=None, file_mapping=None, metadata_context=None, model_context=None, service=, token=None, versioned_id_template=None), definition={'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'}, license=[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}], model={'origin': 'directory', 'source': '../../../tests/data/demo-model/'}, name='MouseLight', protocol='https://www.janelia.org/project-team/mouselight/resources', store={'name': 'DemoStore'})}\n" + "Available Database sources:\n", + "UniProt\n", + "NeuroElectro\n", + "MouseLight\n" ] } ], "source": [ - "print(sources)" + "forge.db_sources(pretty=True)" ] }, { @@ -183,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -250,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -281,7 +303,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -290,7 +312,7 @@ "{'NeuronMorphology': ['DictionaryMapping']}" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -301,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -320,7 +342,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -347,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -358,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -448,7 +470,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -538,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -556,7 +578,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -565,7 +587,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -585,7 +607,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -598,7 +620,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -607,7 +629,7 @@ "0" ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -618,7 +640,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -628,7 +650,7 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_39476/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_39689/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mIndexError\u001b[0m: list index out of range" ] } From a3e4d74b38f1ce13060486b7dfcf832ebcdf2167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 10 Oct 2022 17:28:49 +0200 Subject: [PATCH 07/30] Change SPARQLStore to Specilized UniProtStore. Added Service and spaql query. Ongoing work. --- .../database-sources/prod-nexus-sources.yml | 12 +- .../NeuroElectro/ontologies/data-types.ttl | 176 ------ .../NeuroElectro/ontologies/efeatures.ttl | 527 ---------------- .../UniProt/jsonld_context.json | 24 + .../17 - Database-sources.ipynb | 583 +++++------------- kgforge/core/forge.py | 25 +- kgforge/specializations/stores/__init__.py | 1 + .../stores/uniprot/__init__.py | 15 + .../specializations/stores/uniprot/service.py | 271 ++++++++ .../specializations/stores/uniprot_store.py | 357 +++++++++++ 10 files changed, 860 insertions(+), 1131 deletions(-) delete mode 100644 examples/database_sources/NeuroElectro/ontologies/data-types.ttl delete mode 100644 examples/database_sources/NeuroElectro/ontologies/efeatures.ttl create mode 100644 kgforge/specializations/stores/uniprot/__init__.py create mode 100644 kgforge/specializations/stores/uniprot/service.py create mode 100644 kgforge/specializations/stores/uniprot_store.py diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml index 1503cd4f..7942e420 100644 --- a/examples/configurations/database-sources/prod-nexus-sources.yml +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -57,12 +57,12 @@ Formatters: DatabaseSources: UniProt: - store: - name: DemoStore - model: - name: DemoModel - origin: directory - source: "../../../tests/data/demo-model/" + store: + name: UniProtStore + endpoint: "https://www.uniprot.org/" + searchendpoints: + sparql: + endpoint: "https://sparql.uniprot.org/sparql" NeuroElectro: store: name: BlueBrainNexus diff --git a/examples/database_sources/NeuroElectro/ontologies/data-types.ttl b/examples/database_sources/NeuroElectro/ontologies/data-types.ttl deleted file mode 100644 index 8679b458..00000000 --- a/examples/database_sources/NeuroElectro/ontologies/data-types.ttl +++ /dev/null @@ -1,176 +0,0 @@ -@prefix : . -@prefix owl: . -@prefix rdf: . -@prefix xml: . -@prefix xsd: . -@prefix rdfs: . -@base . - - rdf:type owl:Ontology ; - owl:versionInfo "R167"^^xsd:string ; - rdfs:label "Dataset Type Ontology"@en . - -################################################################# -# Annotation properties -################################################################# - -### http://www.w3.org/2004/02/skos/core#altLabel - rdf:type owl:AnnotationProperty . - - -### http://www.w3.org/2004/02/skos/core#prefLabel - rdf:type owl:AnnotationProperty . - -################################################################# -# Object Properties -################################################################# - -### https://neuroshapes.org/defines - rdf:type owl:AnnotationProperty . - - -################################################################# -# Classes -################################################################# - -### http://schema.org/Dataset - rdf:type owl:Class ; - rdfs:label "Dataset"@en ; - "Dataset"@en . - - -### https://bbp.epfl.ch/ontologies/core/bmo/METypeRatio - rdf:type owl:Class ; - rdfs:label "METypeRatio"@en ; - "METype ratio"@en . - - -### https://bbp.epfl.ch/ontologies/core/bmo/MTypeDensity - rdf:type owl:Class ; - rdfs:label "MTypeDensity"@en ; - "Morphology Type Density"@en . - - -### https://neuroshapes.org/BrainParcellationDataLayer - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "BrainParcellationDataLayer"@en ; - "Brain Parcellation Volume"@en . - - -### https://neuroshapes.org/BrainRegionMesh - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "BrainRegionMesh"@en ; - "Parcellation Mesh"@en . - - -### https://neuroshapes.org/BrainTemplateDataLayer - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "BrainTemplateDataLayer"@en ; - "Brain Template Volume"@en . - - -### https://neuroshapes.org/CellDensityDataLayer - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "CellDensityDataLayer"@en ; - "Cell Density Volume"@en . - - -### https://neuroshapes.org/CellPositions - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "CellPositions"@en ; - "Cell Positions"@en ; - "Cell Positions"@en , - "CellPositions"@en . - - - -### https://neuroshapes.org/ElectricalSeries - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "ElectricalSeries"@en ; - "Electrical Series"@en . - - -### https://neuroshapes.org/GeneExpressionVolumetricDataLayer - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "GeneExpressionVolumetricDataLayer"@en ; - "Gene Expression Volume"@en . - - -### https://neuroshapes.org/Literature - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "Literature"@en ; - "Literature"@en . - - -### https://neuroshapes.org/Mesh - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "Mesh"@en ; - "Mesh"@en . - - -### https://neuroshapes.org/NISSLImageDataLayer - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "NISSLImageDataLayer"@en ; - "NISSL Volume"@en . - - -### https://neuroshapes.org/NeuronMorphology - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "NeuronMorphology"@en ; - "Neuron Morphology"@en . - - -### https://neuroshapes.org/PointCloud - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "PointCloud"@en ; - "Point Cloud"@en . - - -### https://neuroshapes.org/ReconstructedNeuronMorphology - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "ReconstructedNeuronMorphology"@en ; - "Reconstructed Neuron Morphology"@en . - - -### https://neuroshapes.org/SynthesizedNeuronMorphology - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "SynthesizedNeuronMorphology"@en ; - "Synthesized Neuron Morphology"@en . - - -### https://neuroshapes.org/TimeSeries - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "TimeSeries"@en ; - "Time Series"@en . - - -### https://neuroshapes.org/Trace - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "Trace"@en ; - "Trace"@en . - - -### https://neuroshapes.org/VolumetricDataLayer - rdf:type owl:Class ; - rdfs:subClassOf ; - rdfs:label "VolumetricDataLayer"@en ; - "Volume"@en . - - -### Generated by the OWL API (version 4.5.10) https://github.com/owlcs/owlapi diff --git a/examples/database_sources/NeuroElectro/ontologies/efeatures.ttl b/examples/database_sources/NeuroElectro/ontologies/efeatures.ttl deleted file mode 100644 index 3abb9ba3..00000000 --- a/examples/database_sources/NeuroElectro/ontologies/efeatures.ttl +++ /dev/null @@ -1,527 +0,0 @@ -@prefix : . -@prefix owl: . -@prefix rdf: . -@prefix xml: . -@prefix xsd: . -@prefix rdfs: . -@base . - - rdf:type owl:Ontology ; - rdfs:label "Electrophysiological Features Ontology"@en . - -################################################################# -# Annotation properties -################################################################# - -### http://www.w3.org/2004/02/skos/core#altLabel - rdf:type owl:AnnotationProperty . - - -### http://www.w3.org/2004/02/skos/core#definition - rdf:type owl:AnnotationProperty . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/efel_id -:efel_id rdf:type owl:AnnotationProperty ; - rdfs:label "efel_id"@en . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/efel_namespace -:efel_namespace rdf:type owl:AnnotationProperty ; - rdfs:label "efel_namespace"@en . - - -### https://neuroshapes.org/units - rdf:type owl:AnnotationProperty ; - rdfs:label "units"@en . - - -################################################################# -# Classes -################################################################# - -### https://bbp.epfl.ch/ontologies/core/bmo/NeuronElectrophysiologicalFeature - rdf:type owl:Class ; - rdfs:label "Neuron Electrophysiological Feature"@en . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/ADPAmplitude -:ADPAmplitude rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "ADP Amplitude"@en ; - "Amplitude from first AP onset to maximum voltage, typically more depolarized than the resting membrane potential"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/ADPDuration -:ADPDuration rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "ADP Duration"@en ; - "Duration from first AP onset to maximum ADP"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AHPAmplitude -:AHPAmplitude rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "AHP Amplitude"@en ; - "Calculated as the voltage difference between AP threshold and AP trough. Commonly defined using first AP in train at rheobase current."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AHPAmplitudeFromResting -:AHPAmplitudeFromResting rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "AHP Amplitude From Resting"@en ; - "Calculated as the voltage difference between resting or baseline voltages and AP trough. Commonly defined using first AP in train at rheobase current."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AHPDuration -:AHPDuration rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "AHP Duration"@en ; - "Duration of AP after-hyperpolarization, not explicitly referred to by author as either fast or slow"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AHPVoltage -:AHPVoltage rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "AHP Voltage"@en ; - "Calculated as minimum voltage value during an AHP. Commonly defined using first AP in train at rheobase current."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AccessResistance -:AccessResistance rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Access Resistance"@en ; - "Sum of the electrode resistance and the resistance at the electrode-cell junction"@en ; - "M\\ohm" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationPercent1 -:AdaptationPercent1 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Percent1"@en ; - "adaptation percent (first/last ISI)"@en ; - "Percentage of durations between early and late AP inter-spike intervals in an AP train"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationPercent2 -:AdaptationPercent2 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Percent2"@en ; - "adaptation percent (last/first ISI)"@en ; - "Percentage of durations between late and early AP inter-spike intervals in an AP train"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationPercent3 -:AdaptationPercent3 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Percent3"@en ; - "adaptation percent (1 – first/last ISI)"@en ; - "1 minus ratio of durations between early and late AP inter-spike intervals in an AP train, normalized to a percent"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio -:AdaptationRatio rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Ratio"@en ; - "Ratio of durations between early and late AP inter-spike intervals in an AP train"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio1 -:AdaptationRatio1 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Ratio1"@en ; - "adaptation ratio (last/first ISI)"@en ; - "Ratio of durations between late and early AP inter-spike intervals in an AP train"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio2 -:AdaptationRatio2 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Ratio2"@en ; - "adaptation ratio (1 – first/last ISI)"@en ; - "1 minus ratio of durations between early and late AP inter-spike intervals in an AP train"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/AdaptationRatio3 -:AdaptationRatio3 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Adaptation Ratio3"@en ; - "adaptation ratio (other)"@en ; - "Adaptation ratio, percent, index but not otherwise characterizable to an existing sub-definition"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/CellCapacitance -:CellCapacitance rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Cell Capacitance"@en ; - "Neuron capacitance, typically measured by dividing membrane time constant by membrane resistance"@en ; - "pF" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/CellDiameter -:CellDiameter rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Cell Diameter"@en ; - "Diameter of the cell soma"@en ; - "um" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/CellSurfaceArea -:CellSurfaceArea rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Cell Surface Area"@en ; - "Cross-sectional area of the cell"@en ; - "m" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FISlope -:FISlope rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "FI Slope"@en ; - "Slope of the frequency-current relationship"@en ; - "Hz/nA" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPAmplitude -:FastAHPAmplitude rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Fast AHP Amplitude"@en ; - "Calculated as the difference between AP threshold and AP trough, explictly refered to by author as fast."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPAmplitudeFromResting -:FastAHPAmplitudeFromResting rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Fast AHP Amplitude From Resting"@en ; - "Calculated as the difference between resting or baseline voltages and AP trough, explictly refered to by author as fast."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPDuration -:FastAHPDuration rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Fast AHP Duration"@en ; - "Duration of AP after-hyperpolarization, explictly refered to by author as fast"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FastAHPVoltage -:FastAHPVoltage rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Fast AHP Voltage"@en ; - "Calculated as the AHP absolute voltage, explictly refered to by author as fast."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FiringFrequency -:FiringFrequency rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Firing Frequency"@en ; - "AP discharge rate, AP frequency, spike rate, firing rate, spike frequency, mean firing rate, steady state firing rate, average firing rate, steady firing rate"@en ; - "Hz" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/FirstSpikeLatency -:FirstSpikeLatency rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "First Spike Latency"@en ; - "Duration to first AP following a depolarizing current step of fixed amplitude"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/InputResistance -:InputResistance rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Input Resistance"@en ; - "Input resistance measured at steady-state voltage response to current injection"@en ; - "M\\ohm" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/MaximumFiringRate -:MaximumFiringRate rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Maximum Firing Rate"@en ; - "Maximum observed AP discharge rate"@en ; - "Hz" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPAmplitude -:MediumAHPAmplitude rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Medium AHP Amplitude"@en ; - "Calculated as the difference between AP threshold and AP trough, explictly refered to by author as medium"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPAmplitudeFromResting -:MediumAHPAmplitudeFromResting rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Medium AHP Amplitude From Resting"@en ; - "Calculated as the difference between resting or baseline voltages and AP trough, explictly refered to by author as medium"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPDuration -:MediumAHPDuration rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Medium AHP Duration"@en ; - "Duration of AP after-hyperpolarization, explictly refered to by author as medium"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/MediumAHPVoltage -:MediumAHPVoltage rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Medium AHP Voltage"@en ; - "Calculated as the AHP absolute voltage, explictly refered to by author as medium."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/MembraneTimeConstant -:MembraneTimeConstant rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Membrane Time Constant"@en ; - "Time constant for the membrane to repolarize after a small current injection of fixed amplitude and duration"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/NeuroElectroNeuronElectrophysiologicalFeature -:NeuroElectroNeuronElectrophysiologicalFeature rdf:type owl:Class ; - rdfs:label "NeuroElectro Neuron Electrophysiological Feature"@en . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/Other -:Other rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Other"@en ; - "A catch-all property to reflect any properties which are not already explicitly defined"@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/RestingMembranePotential -:RestingMembranePotential rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Resting Membrane Potential"@en ; - "Membrane potential at the onset of whole-cell recording"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/Rheobase -:Rheobase rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Rheobase"@en ; - "Minimum current injected somatically required to fire AP"@en ; - "pA" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SagAmplitude2 -:SagAmplitude2 rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Sag Amplitude2"@en ; - "Absolute difference between the steady state decrease in the voltage and the largest decrease in voltage following a hyperpolarizing current step."^^xsd:string ; - "mV"^^xsd:string . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SagRatio -:SagRatio rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Sag Ratio"@en ; - "Ratio between the steady state decrease in the voltage and the largest decrease in voltage following a hyperpolarizing current step."@en ; - "Unitless" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPAmplitude -:SlowAHPAmplitude rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Slow AHP Amplitude"@en ; - "Calculated as the difference between AP threshold and AP trough, explictly refered to by author as slow"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPAmplitudeFromResting -:SlowAHPAmplitudeFromResting rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Slow AHP Amplitude From Resting"@en ; - "Calculated as the difference between resting or baseline voltages and AP trough, explictly refered to by author as slow"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPDuration -:SlowAHPDuration rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Slow AHP Duration"@en ; - "Duration of AP after-hyperpolarization, explictly refered to by author as slow"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SlowAHPVoltage -:SlowAHPVoltage rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Slow AHP Voltage"@en ; - "Calculated as the AHP absolute voltage, explictly refered to by author as slow."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeAmplitude -:SpikeAmplitude rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Amplitude"@en ; - "Voltage indicating height of action potential. Usually calculated as the difference between AP peak and AP threshold voltages. Commonly measured using first AP in train at rheobase current."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeAmplitudeFromResting -:SpikeAmplitudeFromResting rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Amplitude From Resting"@en ; - "Voltage indicating height of action potential, calculated as the difference between AP peak and resting or baseline voltages. Commonly measured using first AP in train at rheobase current."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeAmplitudeFromTrough -:SpikeAmplitudeFromTrough rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Amplitude From Trough"@en ; - "Voltage indicating height of action potential, calculated as the difference between AP peak and AHP trough voltages. Commonly measured using first AP in train at rheobase current."@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeDecayTime -:SpikeDecayTime rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Decay Time"@en ; - "Time for spike to fall from peak to threshold, usually calculated as 10-90% decay time"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeHalf-Width -:SpikeHalf-Width rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Half-Width"@en ; - "Calculated as the AP duration at the membrane voltage halfway between AP threshold and AP peak. Most commonly calculated using first AP in train at rheobase. current."@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeMaxDecaySlope -:SpikeMaxDecaySlope rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Max Decay Slope"@en ; - "Maximum rate of rise of membrane voltage during spike falling phase"@en ; - "mV/ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeMaxRiseSlope -:SpikeMaxRiseSlope rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Max Rise Slope"@en ; - "Maximum rate of rise of membrane voltage during spike rising phase"@en ; - "mV/ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikePeak -:SpikePeak rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Peak"@en ; - "Maximum voltage reached during AP"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeRiseTime -:SpikeRiseTime rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Rise Time"@en ; - "Time for spike to rise from threshold to peak, usually calculated as 10-90% rise time"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeThreshold -:SpikeThreshold rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Threshold"@en ; - "Voltage at which AP is initiated (as assessed by measuring rising slope of membrane voltage)"@en ; - "mV" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpikeWidth -:SpikeWidth rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spike Width"@en ; - "Duration of AP, not explictly refered to as half-width"@en ; - "ms" . - - -### https://bbp.epfl.ch/ontologies/core/efeatures/SpontaneousFiringRate -:SpontaneousFiringRate rdf:type owl:Class ; - rdfs:subClassOf , - :NeuroElectroNeuronElectrophysiologicalFeature ; - rdfs:label "Spontaneous Firing Rate"@en ; - "AP discharge rate in the absence of current injection or a stimulus"@en ; - "Hz" . - - -### Generated by the OWL API (version 4.5.9.2019-02-01T07:24:44Z) https://github.com/owlcs/owlapi diff --git a/examples/database_sources/UniProt/jsonld_context.json b/examples/database_sources/UniProt/jsonld_context.json index e69de29b..8518b0e8 100644 --- a/examples/database_sources/UniProt/jsonld_context.json +++ b/examples/database_sources/UniProt/jsonld_context.json @@ -0,0 +1,24 @@ +{ + "@context": { + "up": "http://purl.uniprot.org/core/", + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "core": "http://purl.uniprot.org/core/", + "dc11": "http://purl.org/dc/terms/", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "Protein": { + "@id": "up:Protein" + }, + "Gene": { + "@id": "up:Gene" + } + }, + "@id": "https://bbp.epfl.ch/jsonldcontext/db/uniprot" +} \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 7a1db016..f03e045e 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -23,17 +23,12 @@ }, "outputs": [], "source": [ + "import json\n", + "\n", "from kgforge.core import KnowledgeGraphForge\n", "from kgforge.specializations.resources import Dataset" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A configuration file is needed in order to create a KnowledgeGraphForge session. A configuration can be generated using the notebook [00-Initialization.ipynb](00%20-%20Initialization.ipynb)." - ] - }, { "cell_type": "code", "execution_count": 2, @@ -582,17 +577,7 @@ "metadata": {}, "outputs": [], "source": [ - "props = {'origin': 'store', 'source': 'BlueBrainNexus', 'definition':{'iri': 'some_address'}}" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "# props = {'origin': 'store', 'source': 'BlueBrainNexus', 'definition':{'iri':}}\n", - "# new_db = DatabaseSource(forge, from_forge=False, name='new_db')" + "ne = sources['NeuroElectro']" ] }, { @@ -607,29 +592,29 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "# Type, source or target brain region, \n", - "filters = {\"type\":\"NeuronMorphology\"} # More filters (brain regions, ...) will be added\n", + "filters = {\"type\":\"ScholarlyArticle\"}\n", "#map=True, use_cache=True, # download=True\n", - "resources = forge.search(filters, db_source=\"MouseLight\", limit=2) \n", + "resources = forge.search(filters, db_source=\"NeuroElectro\", limit=2) \n", "# ADd function for checking datsource health => reqsuire health url from db\n" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0" + "2" ] }, - "execution_count": 21, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -640,18 +625,78 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [ { - "ename": "IndexError", - "evalue": "list index out of range", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_39689/2183119560.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m: list index out of range" + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " context: https://bbp.neuroshapes.org\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", + " type:\n", + " [\n", + " Entity\n", + " ScholarlyArticle\n", + " ]\n", + " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", + " author:\n", + " [\n", + " {\n", + " type: Person\n", + " familyName: Henderson\n", + " givenName: Z\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Morris\n", + " givenName: N P\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Grimwood\n", + " givenName: P\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Fiddler\n", + " givenName: G\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Yang\n", + " givenName: H W\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Appenteng\n", + " givenName: K\n", + " }\n", + " ]\n", + " datePublished: 2001-2-12\n", + " identifier:\n", + " [\n", + " {\n", + " propertyID: PMID\n", + " value: 11169477\n", + " }\n", + " {\n", + " propertyID: doi\n", + " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", + " }\n", + " ]\n", + " isPartOf:\n", + " {\n", + " type: Periodical\n", + " issn: 0021-9967\n", + " name: The Journal of comparative neurology\n", + " }\n", + " name: article_91941\n", + " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", + " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + "}\n" ] } ], @@ -660,30 +705,90 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 22, "metadata": {}, + "outputs": [], "source": [ - "# Save in BBP KG (Nexus)" + "uquery = \"\"\"\n", + "PREFIX up: \n", + "SELECT ?protein\n", + "WHERE {\n", + " ?protein a up:Protein .\n", + " ?protein up:reviewed true .\n", + "}\n", + "\"\"\"" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " 2\n", - " _register_many\n", - " False\n", - " RegistrationError: resource already exists\n" + "Submitted query:\n", + " \n", + " PREFIX up: \n", + " SELECT ?protein\n", + " WHERE {\n", + " ?protein a up:Protein .\n", + " ?protein up:reviewed true .\n", + " }\n", + " LIMIT 10\n", + "\n", + "query \n", + "PREFIX up: \n", + "SELECT ?protein\n", + "WHERE {\n", + " ?protein a up:Protein .\n", + " ?protein up:reviewed true .\n", + "}\n", + " LIMIT 10\n", + "self.service.sparql_endpoint {'endpoint': 'https://sparql.uniprot.org/sparql', 'type': 'sparql'}\n", + "self.service.headers_sparql {'Content-Type': 'text/plain', 'Accept': 'application/sparql-results+json'}\n" ] + }, + { + "data": { + "text/plain": [ + "[Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/A0A131MCZ8'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/D0PX85'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/G5EF51'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/I0DF35'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P0C645'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P53500'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIL8'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIL9'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIM0'),\n", + " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIM1')]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "forge.register(resources)" + "forge.sparql(query=uquery, db_source='UniProt', limit=10, debug=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Save in BBP KG (Nexus)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "# forge.register(resources)" ] }, { @@ -702,7 +807,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -719,14 +824,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "10 dataset(s) of type NeuronMorphology found\n" + "0 dataset(s) of type NeuronMorphology found\n" ] } ], @@ -747,7 +852,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -771,386 +876,20 @@ " \n", " \n", " \n", - " id\n", - " brainLocation.brainRegion.id\n", - " brainLocation.brainRegion.label\n", - " brainLocation.layer\n", - " contribution.type\n", - " contribution.agent.id\n", - " contribution.agent.type\n", - " distribution.contentUrl\n", - " distribution.encodingFormat\n", - " distribution.name\n", - " name\n", - " subject.type\n", - " subject.age.period\n", - " subject.age.unitCode\n", - " subject.age.value\n", - " subject.identifier\n", - " subject.name\n", - " subject.sex\n", - " subject.species\n", - " subject.strain\n", " \n", " \n", " \n", - " \n", - " 0\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISp5\n", - " 5\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " Scnn1a-Tg3-Cre;Ai14-172530.06.01.01\n", - " Subject\n", - " Post-natal\n", - " \n", - " \n", - " 322489588\n", - " Scnn1a-Tg3-Cre;Ai14(GSL)-172530\n", - " \n", - " Mus musculus\n", - " Scnn1a-Tg3-Cre\n", - " \n", - " \n", - " 1\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " MTG\n", - " 2\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " H16.06.009.01.01.15.01\n", - " Subject\n", - " Post-natal\n", - " yrs\n", - " 48\n", - " 528574320\n", - " H16.06.009\n", - " Female\n", - " Homo Sapiens\n", - " \n", - " \n", - " \n", - " 2\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISp4\n", - " 4\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " Scnn1a-Tg3-Cre;Ai14-187849.06.01.01\n", - " Subject\n", - " Post-natal\n", - " \n", - " \n", - " 475849748\n", - " Scnn1a-Tg3-Cre;Ai14(IVSCC)-187849\n", - " \n", - " Mus musculus\n", - " Scnn1a-Tg3-Cre\n", - " \n", - " \n", - " 3\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISp4\n", - " 4\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " Rorb-IRES2-Cre-D;Ai14-197330.06.01.01\n", - " Subject\n", - " Post-natal\n", - " \n", - " \n", - " 479695183\n", - " Rorb-IRES2-Cre-D;Ai14-197330\n", - " \n", - " Mus musculus\n", - " Rorb-IRES2-Cre\n", - " \n", - " \n", - " 4\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISpl4\n", - " 4\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " Rorb-IRES2-Cre-D;Ai14-230822.04.02.01\n", - " Subject\n", - " Post-natal\n", - " \n", - " \n", - " 502081962\n", - " Rorb-IRES2-Cre-D;Ai14-230822\n", - " \n", - " Mus musculus\n", - " Rorb-IRES2-Cre\n", - " \n", - " \n", - " 5\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " AnG\n", - " 2\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " H17.06.004.11.05.04\n", - " Subject\n", - " Post-natal\n", - " yrs\n", - " 71\n", - " 569008241\n", - " H17.06.004\n", - " Female\n", - " Homo Sapiens\n", - " \n", - " \n", - " \n", - " 6\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " MTG\n", - " 4\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " H17.06.009.11.04.02\n", - " Subject\n", - " Post-natal\n", - " yrs\n", - " 52\n", - " 595954915\n", - " H17.06.009\n", - " Male\n", - " Homo Sapiens\n", - " \n", - " \n", - " \n", - " 7\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " MTG\n", - " 3\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " H16.03.001.01.09.01\n", - " Subject\n", - " Post-natal\n", - " yrs\n", - " 39\n", - " 518641172\n", - " H16.03.001\n", - " Male\n", - " Homo Sapiens\n", - " \n", - " \n", - " \n", - " 8\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " MFG\n", - " 5\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " H17.06.007.11.08.01\n", - " Subject\n", - " Post-natal\n", - " yrs\n", - " 42\n", - " 576060516\n", - " H17.06.007\n", - " Female\n", - " Homo Sapiens\n", - " \n", - " \n", - " \n", - " 9\n", - " https://bbp.epfl.ch/nexus/v1/resources/dke/kgf...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISp5\n", - " 5\n", - " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", - " Organization\n", - " https://bbp.epfl.ch/nexus/v1/files/dke/kgforge...\n", - " application/swc\n", - " reconstruction.swc\n", - " Cux2-CreERT2;Ai14-205530.03.02.01\n", - " Subject\n", - " Post-natal\n", - " \n", - " \n", - " 485250100\n", - " Cux2-CreERT2;Ai14-205530\n", - " \n", - " Mus musculus\n", - " Cux2-CreERT2\n", - " \n", " \n", "\n", "" ], "text/plain": [ - " id \\\n", - "0 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "1 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "2 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "3 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "4 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "5 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "6 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "7 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "8 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "9 https://bbp.epfl.ch/nexus/v1/resources/dke/kgf... \n", - "\n", - " brainLocation.brainRegion.id \\\n", - "0 http://api.brain-map.org/api/v2/data/Structure... \n", - "1 http://api.brain-map.org/api/v2/data/Structure... \n", - "2 http://api.brain-map.org/api/v2/data/Structure... \n", - "3 http://api.brain-map.org/api/v2/data/Structure... \n", - "4 http://api.brain-map.org/api/v2/data/Structure... \n", - "5 http://api.brain-map.org/api/v2/data/Structure... \n", - "6 http://api.brain-map.org/api/v2/data/Structure... \n", - "7 http://api.brain-map.org/api/v2/data/Structure... \n", - "8 http://api.brain-map.org/api/v2/data/Structure... \n", - "9 http://api.brain-map.org/api/v2/data/Structure... \n", - "\n", - " brainLocation.brainRegion.label brainLocation.layer contribution.type \\\n", - "0 VISp5 5 Contribution \n", - "1 MTG 2 Contribution \n", - "2 VISp4 4 Contribution \n", - "3 VISp4 4 Contribution \n", - "4 VISpl4 4 Contribution \n", - "5 AnG 2 Contribution \n", - "6 MTG 4 Contribution \n", - "7 MTG 3 Contribution \n", - "8 MFG 5 Contribution \n", - "9 VISp5 5 Contribution \n", - "\n", - " contribution.agent.id contribution.agent.type \\\n", - "0 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "1 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "2 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "3 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "4 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "5 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "6 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "7 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "8 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "9 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "\n", - " distribution.contentUrl \\\n", - "0 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "1 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "2 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "3 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "4 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "5 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "6 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "7 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "8 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "9 https://bbp.epfl.ch/nexus/v1/files/dke/kgforge... \n", - "\n", - " distribution.encodingFormat distribution.name \\\n", - "0 application/swc reconstruction.swc \n", - "1 application/swc reconstruction.swc \n", - "2 application/swc reconstruction.swc \n", - "3 application/swc reconstruction.swc \n", - "4 application/swc reconstruction.swc \n", - "5 application/swc reconstruction.swc \n", - "6 application/swc reconstruction.swc \n", - "7 application/swc reconstruction.swc \n", - "8 application/swc reconstruction.swc \n", - "9 application/swc reconstruction.swc \n", - "\n", - " name subject.type subject.age.period \\\n", - "0 Scnn1a-Tg3-Cre;Ai14-172530.06.01.01 Subject Post-natal \n", - "1 H16.06.009.01.01.15.01 Subject Post-natal \n", - "2 Scnn1a-Tg3-Cre;Ai14-187849.06.01.01 Subject Post-natal \n", - "3 Rorb-IRES2-Cre-D;Ai14-197330.06.01.01 Subject Post-natal \n", - "4 Rorb-IRES2-Cre-D;Ai14-230822.04.02.01 Subject Post-natal \n", - "5 H17.06.004.11.05.04 Subject Post-natal \n", - "6 H17.06.009.11.04.02 Subject Post-natal \n", - "7 H16.03.001.01.09.01 Subject Post-natal \n", - "8 H17.06.007.11.08.01 Subject Post-natal \n", - "9 Cux2-CreERT2;Ai14-205530.03.02.01 Subject Post-natal \n", - "\n", - " subject.age.unitCode subject.age.value subject.identifier \\\n", - "0 322489588 \n", - "1 yrs 48 528574320 \n", - "2 475849748 \n", - "3 479695183 \n", - "4 502081962 \n", - "5 yrs 71 569008241 \n", - "6 yrs 52 595954915 \n", - "7 yrs 39 518641172 \n", - "8 yrs 42 576060516 \n", - "9 485250100 \n", - "\n", - " subject.name subject.sex subject.species \\\n", - "0 Scnn1a-Tg3-Cre;Ai14(GSL)-172530 Mus musculus \n", - "1 H16.06.009 Female Homo Sapiens \n", - "2 Scnn1a-Tg3-Cre;Ai14(IVSCC)-187849 Mus musculus \n", - "3 Rorb-IRES2-Cre-D;Ai14-197330 Mus musculus \n", - "4 Rorb-IRES2-Cre-D;Ai14-230822 Mus musculus \n", - "5 H17.06.004 Female Homo Sapiens \n", - "6 H17.06.009 Male Homo Sapiens \n", - "7 H16.03.001 Male Homo Sapiens \n", - "8 H17.06.007 Female Homo Sapiens \n", - "9 Cux2-CreERT2;Ai14-205530 Mus musculus \n", - "\n", - " subject.strain \n", - "0 Scnn1a-Tg3-Cre \n", - "1 \n", - "2 Scnn1a-Tg3-Cre \n", - "3 Rorb-IRES2-Cre \n", - "4 Rorb-IRES2-Cre \n", - "5 \n", - "6 \n", - "7 \n", - "8 \n", - "9 Cux2-CreERT2 " + "Empty DataFrame\n", + "Columns: []\n", + "Index: []" ] }, - "execution_count": 54, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1171,9 +910,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " download\n", + " DownloadingError: path to follow 'distribution.contentUrl' was not found in any provided resource.\n", + "\n" + ] + } + ], "source": [ "dirpath = \"./downloaded/\"\n", "forge.download(data, \"distribution.contentUrl\", dirpath)" diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 65956b61..fa21e38e 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -684,7 +684,11 @@ def search(self, *filters, **params) -> List[Resource]: resolvers = ( list(self._resolvers.values()) if self._resolvers is not None else None ) - return self._store.search(resolvers, *filters, **params) + db_source = params.pop('db_source', None) + if db_source: + return self._db_sources[db_source]._store.search(resolvers, *filters, **params) + else: + return self._store.search(resolvers, *filters, **params) @catch def sparql( @@ -705,7 +709,11 @@ def sparql( :param params: a dictionary of parameters. Supported params are: rewrite (whether to rewrite the sparql query or run it as is) :return: List[Resource] """ - return self._store.sparql(query, debug, limit, offset, **params) + db_source = params.pop('db_source', None) + if db_source: + return self._db_sources[db_source]._store.sparql(query, debug, limit, offset, **params) + else: + return self._store.sparql(query, debug, limit, offset, **params) @catch def elastic( @@ -714,6 +722,7 @@ def elastic( debug: bool = False, limit: Optional[int] = None, offset: Optional[int] = None, + db_source: Optional[str] = None, ) -> List[Resource]: """ Search for resources using an ElasticSearch DSL query. See ElasticSearch DSL docs: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html. @@ -724,7 +733,10 @@ def elastic( :param offset: how many results to skip from the first one :return: List[Resource] """ - return self._store.elastic(query, debug, limit, offset) + if db_source: + return self._db_sources[db_source]._store.elastic(query, debug, limit, offset) + else: + return self._store.elastic(query, debug, limit, offset) @catch def download( @@ -1002,11 +1014,14 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], config = all_config[name] # Provide store and model configuration to the database sources if "model" not in config: - config.update(model=model_config) + config.update(model=deepcopy(model_config)) # Complete configuration of the db store in case is the same if config['store']['name'] == store_config['name']: - config['store'] = store_config + store_copy = deepcopy(store_config) + with_defaults(config['store'], store_copy, + "name", "name", + store_copy.keys()) dbs[name] = DatabaseSource(self, name=name, from_forge=True, **config) return dbs diff --git a/kgforge/specializations/stores/__init__.py b/kgforge/specializations/stores/__init__.py index 8354b932..9a89dfe5 100644 --- a/kgforge/specializations/stores/__init__.py +++ b/kgforge/specializations/stores/__init__.py @@ -14,3 +14,4 @@ from .bluebrain_nexus import BlueBrainNexus from .demo_store import DemoStore +from .uniprot_store import UniProtStore diff --git a/kgforge/specializations/stores/uniprot/__init__.py b/kgforge/specializations/stores/uniprot/__init__.py new file mode 100644 index 00000000..711555f6 --- /dev/null +++ b/kgforge/specializations/stores/uniprot/__init__.py @@ -0,0 +1,15 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +from .service import Service diff --git a/kgforge/specializations/stores/uniprot/service.py b/kgforge/specializations/stores/uniprot/service.py new file mode 100644 index 00000000..d42bfc27 --- /dev/null +++ b/kgforge/specializations/stores/uniprot/service.py @@ -0,0 +1,271 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import asyncio +import copy +from difflib import context_diff +import json +import re +from pathlib import Path +from asyncio import Task +from collections import namedtuple +from copy import deepcopy +from enum import Enum +from typing import Callable, Dict, List, Optional, Union, Tuple +from urllib.error import URLError +from urllib.parse import quote_plus, urlparse + +import nest_asyncio +import nexussdk as nexus +import requests +from aiohttp import ClientSession, hdrs +from numpy import nan +from requests import HTTPError + +from kgforge.core import Resource +from kgforge.core.commons.actions import ( + Action, + collect_lazy_actions, + execute_lazy_actions, + LazyAction, +) +from kgforge.core.commons.context import Context +from kgforge.core.conversions.rdf import ( + _from_jsonld_one, + _remove_ld_keys, + as_jsonld, + recursive_resolve, +) +from kgforge.core.wrappings.dict import wrap_dict +from kgforge.specializations.resources.db_sources import DBS_PATH + + +class Service: + + UNIPROT_CONTEXT_PATH = Path(DBS_PATH, 'UniProt', 'jsonld_context.json') + UNIPROT_NAMESPACE = 'http://purl.uniprot.org/core/' + + def __init__( + self, + endpoint: str, + model_context: Context, + max_connection: int, + searchendpoints: Dict, + content_type: str, + accept: str, + **params, + ): + + self.endpoint = endpoint + self.model_context = model_context + self.context_cache: Dict = dict() + self.max_connection = max_connection + self.params = copy.deepcopy(params) + self.namespace = self.UNIPROT_NAMESPACE + + self.headers = {"Content-Type": content_type, "Accept": accept} + + sparql_config = ( + searchendpoints["sparql"] + if searchendpoints and "sparql" in searchendpoints + else None + ) + + self.headers_sparql = { + "Content-Type": sparql_config["Content-Type"] + if sparql_config and "Content-Type" in sparql_config + else "text/plain", + "Accept": sparql_config["Accept"] + if sparql_config and "Accept" in sparql_config + else "application/sparql-results+json", + } + + # load context from file + with open(self.UNIPROT_CONTEXT_PATH) as jfile: + uniprot_context = json.load(jfile) + self.context = Context(uniprot_context) + + self.metadata_context = Context(uniprot_context) + + self.sparql_endpoint = dict() + self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] + self.sparql_endpoint["type"] = "sparql" + + # The following code is for async to work on jupyter notebooks + try: + asyncio.get_event_loop() + nest_asyncio.apply() + except RuntimeError: + pass + + def resolve_context(self, iri: str) -> Dict: + if iri in self.context_cache: + return self.context_cache[iri] + context_to_resolve = iri + try: + context = Context(context_to_resolve) + except URLError: + raise ValueError(f"{context_to_resolve} is not resolvable") + else: + document = context.document["@context"] + self.context_cache.update({context_to_resolve: document}) + return document + + async def request(method, session, url, resource, payload, params, exception): + async with session.request( + method, + url, + headers=self.headers, + data=json.dumps(payload), + params=params, + ) as response: + content = await response.json() + if response.status < 400: + return (resource, content) + else: + msg = " ".join(re.findall("[A-Z][^A-Z]*", content["@type"])).lower() + error = exception(msg) + return (resource, error) + + def sync_metadata(self, resource: Resource, result: Dict) -> None: + metadata = ( + {"id": resource.id} + if hasattr(resource, "id") + else ( + {"id": resource.__getattribute__("@id")} + if hasattr(resource, "@id") + else dict() + ) + ) + keys = sorted(self.metadata_context.terms.keys()) + keys.extend(["_index", "_score", "id", "@id"]) + only_meta = {k: v for k, v in result.items() if k in keys} + metadata.update(_remove_ld_keys(only_meta, self.metadata_context, False)) + if not hasattr(resource, "id") and not hasattr(resource, "@id"): + resource.id= result.get("id", result.get("@id",None)) + resource._store_metadata = wrap_dict(metadata) + + def synchronize_resource( + self, + resource: Resource, + response: Union[Exception, Dict], + action_name: str, + succeeded: bool, + synchronized: bool, + ) -> None: + if succeeded: + action = Action(action_name, succeeded, None) + self.sync_metadata(resource, response) + else: + action = Action(action_name, succeeded, response) + resource._last_action = action + resource._synchronized = synchronized + + def default_callback(self, fun_name: str) -> Callable: + def callback(task: Task): + result = task.result() + if isinstance(result.response, Exception): + self.synchronize_resource( + result.resource, result.response, fun_name, False, False + ) + else: + self.synchronize_resource( + result.resource, result.response, fun_name, True, True + ) + + return callback + + def verify( + self, + resources: List[Resource], + function_name, + exception: Callable, + id_required: bool, + required_synchronized: bool, + execute_actions: bool, + ) -> List[Resource]: + valid = list() + for resource in resources: + if id_required and not hasattr(resource, "id"): + error = exception("resource should have an id") + self.synchronize_resource(resource, error, function_name, False, False) + continue + if required_synchronized is not None: + synchronized = resource._synchronized + if synchronized is not required_synchronized: + be_or_not_be = "be" if required_synchronized is True else "not be" + error = exception(f"resource should {be_or_not_be} synchronized") + self.synchronize_resource( + resource, error, function_name, False, False + ) + continue + if execute_actions: + lazy_actions = collect_lazy_actions(resource) + if lazy_actions is not None: + try: + execute_lazy_actions(resource, lazy_actions) + except Exception as e: + self.synchronize_resource( + resource, exception(e), function_name, False, False + ) + continue + valid.append(resource) + return valid + + def to_resource( + self, payload: Dict, sync_metadata: bool = True, **kwargs + ) -> Resource: + # Use JSONLD context defined in Model if no context is retrieved from payload + # Todo: BlueBrainNexus store is not indexing in ES the JSONLD context, user provided context can be changed to Model defined one + data_context = deepcopy(payload.get("@context", self.model_context.iri if self.model_context else None)) + if not isinstance(data_context, list): + data_context = [data_context] + if self.store_context in data_context: + data_context.remove(self.store_context) + data_context = data_context[0] if len(data_context) == 1 else data_context + metadata = dict() + data = dict() + for k, v in payload.items(): + if k in self.metadata_context.terms.keys(): + metadata[k] = v + else: + data[k] = v + + if ( + self.model_context + and data_context is not None + and data_context == self.model_context.iri + ): + resolved_ctx = self.model_context.document["@context"] + elif data_context is not None: + resolved_ctx = recursive_resolve( + data_context, + self.resolve_context, + already_loaded=[self.store_local_context, self.store_context], + ) + else: + resolved_ctx = None + if resolved_ctx: + data["@context"] = resolved_ctx + resource = _from_jsonld_one(data) + resource.context = data_context + else: + resource = Resource.from_json(data) + + if len(metadata) > 0 and sync_metadata: + metadata.update(kwargs) + self.sync_metadata(resource, metadata) + if not hasattr(resource, "id") and kwargs and 'id' in kwargs.keys(): + resource.id = kwargs.get("id") + return resource diff --git a/kgforge/specializations/stores/uniprot_store.py b/kgforge/specializations/stores/uniprot_store.py new file mode 100644 index 00000000..74dbec67 --- /dev/null +++ b/kgforge/specializations/stores/uniprot_store.py @@ -0,0 +1,357 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import json +from pyld import jsonld +from copy import deepcopy +from pathlib import Path +from typing import Dict, List, Optional, Union, Any +from uuid import uuid4 +import requests +from rdflib import Graph +from rdflib.plugins.sparql.parser import Query +from requests import HTTPError +from SPARQLWrapper import SPARQLWrapper, JSON + +from kgforge.core import Resource +from kgforge.core.archetypes import Resolver, Store +from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql +# from kgforge.specializations.stores.bluebrain_nexus import ( +# CategoryDataType, +# build_sparql_query_statements, +# format_type, +# _create_select_query +# ) +from kgforge.core.commons.context import Context +from kgforge.core.wrappings.dict import DictWrapper +from kgforge.specializations.stores.uniprot.service import Service +from kgforge.core.commons.exceptions import DownloadingError, QueryingError +from kgforge.core.commons.execution import not_supported +from kgforge.core.conversions.rdf import as_jsonld, from_jsonld +# from kgforge.core.conversions.json import as_json, from_json +# from kgforge.core.wrappings.dict import wrap_dict +# from kgforge.core.wrappings.paths import create_filters_from_dict +# from urllib.parse import quote_plus, unquote, urlparse, parse_qs + + +class UniProtStore(Store): + """A Store specialized for SPARQL queries, supporting only Reading (searching) methods.""" + + def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, + token: Optional[str] = None, versioned_id_template: Optional[str] = None, + file_resource_mapping: Optional[str] = None, + model_context: Optional[Context] = None, + searchendpoints: Optional[Dict] = None,) -> None: + super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, + model_context, searchendpoints) + + + # [C]RUD + + def register(self, data: Union[Resource, List[Resource]], schema_id: str = None + ) -> None: + # Replace None by self._register_many to switch to optimized bulk registration. + not_supported() + + def _register_one(self, resource: Resource, schema_id: str) -> None: + not_supported() + + def upload(self, path: str, content_type: str) -> Union[Resource, List[Resource]]: + not_supported() + + def _upload(path: Path, content_type: str) -> Union[Any, List[Any]]: + not_supported() + + def _upload_many(self, paths: List[Path], content_type: str) -> List[Any]: + not_supported() + + def _download_one( + self, + url: str, + path: str, + store_metadata: Optional[DictWrapper], + cross_bucket: bool, + ) -> None: + # path: FilePath. + # TODO define dowloading method + # POLICY Should notify of failures with exception DownloadingError including a message. + not_supported() + + # C[R]UD + + def retrieve( + self, id: str, version: Optional[Union[int, str]], cross_bucket: bool, **params + ) -> Resource: + not_supported() + + def _retrieve_filename(self, id: str) -> str: + not_supported() + + # CR[U]D. + + def update( + self, data: Union[Resource, List[Resource]], schema_id: Optional[str] + ) -> None: + # Replace None by self._update_many to switch to optimized bulk update. + not_supported() + + def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: + not_supported() + + def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + # Replace None by self._tag_many to switch to optimized bulk tagging. + # POLICY If tagging modify the resource, run() should have status='_synchronized'. + not_supported() + + # CRU[D]. + + def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + # Replace None by self._deprecate_many to switch to optimized bulk deprecation. + not_supported() + + # Querying. + + def search( + self, resolvers: Optional[List["Resolver"]], *filters, **params + ) -> List[Resource]: + # Positional arguments in 'filters' are instances of type Filter from wrappings/paths.py + # A dictionary can be provided for filters: + # - {'key1': 'val', 'key2': {'key3': 'val'}} will be translated to + # - [Filter(operator='__eq__', path=['key1'], value='val'), Filter(operator='__eq__', path=['key2', 'key3'], value='val')] + # Keyword arguments in 'params' could be: + # - debug: bool, + # - limit: int, + # - offset: int, + # - deprecated: bool, + # - resolving: str, with values in ('exact', 'fuzzy'), + # - lookup: str, with values in ('current', 'children'). + # POLICY Should use sparql() and contain 'search_endpoint'. + # POLICY Resource _store_metadata should be set using wrappers.dict.wrap_dict(). + # POLICY Resource _synchronized should be set to True. + pass + # if self.model_context is None: + # raise ValueError("context model missing") + + # debug = params.get("debug", False) + # limit = params.get("limit", 100) + # offset = params.get("offset", None) + # deprecated = params.get("deprecated", False) + # distinct = params.get("distinct", False) + # includes = params.get("includes", None) + # excludes = params.get("excludes", None) + # retrieve_source = params.get("retrieve_source") + # search_endpoint = params.get( + # "search_endpoint", self.service.sparql_endpoint["type"] + # ) + # if search_endpoint not in [ + # self.service.sparql_endpoint["type"], + # ]: + # raise ValueError( + # f"The provided search_endpoint value '{search_endpoint}' is not supported, only 'sparql'" + # ) + # if "filters" in params: + # raise ValueError("A 'filters' key was provided as params. Filters should be provided as iterable to be unpacked.") + + + # if filters and isinstance(filters[0], dict): + # filters = create_filters_from_dict(filters[0]) + # filters = list(filters) if not isinstance(filters, list) else filters + + # if includes or excludes: + # raise ValueError( + # "Field inclusion and exclusion are not supported when using SPARQL" + # ) + + # query_statements, query_filters = build_sparql_query_statements( + # self.model_context, filters + # ) + # store_metadata_statements = [] + # if retrieve_source: + # _vars = ["?id"] + # for i, k in enumerate(self.service.store_metadata_keys): + # _vars.append(f"?{k}") + # store_metadata_statements.insert(i+2, f"<{self.metadata_context.terms[k].id}> ?{k}") + # deprecated_filter = f"Filter (?_deprecated = {format_type[CategoryDataType.BOOLEAN](deprecated)})" + # query_filters.append(deprecated_filter) + # else: + # _vars = ["?id", "?_project", "?_rev"] + # store_metadata_statements.append(f"<{self.service.revision_property}> ?_rev") + # store_metadata_statements.append(f"<{self.service.project_property}> ?_project") + # query_statements.append( + # f"<{self.service.deprecated_property}> {format_type[CategoryDataType.BOOLEAN](deprecated)}", + # ) + # query_statements.extend(store_metadata_statements) + # statements = ";\n ".join(query_statements) + # _filters = "\n".join(".\n ".join(query_filters)) + # query = _create_select_query( + # _vars, f"?id {statements} . \n {_filters}", distinct + # ) + # # support @id and @type + # resources = self.sparql(query, debug=debug, limit=limit, offset=offset) + # params_retrieve = deepcopy(self.service.params.get("retrieve", {})) + # params_retrieve['retrieve_source'] = retrieve_source + # results = self.service.batch_request( + # resources, BatchAction.FETCH, None, QueryingError, params=params_retrieve + # ) + # resources = list() + # for result in results: + # resource = result.resource + # if retrieve_source: + # store_metadata_response = as_json(result.resource, expanded=False, store_metadata=False, + # model_context=None, + # metadata_context=None, + # context_resolver=None) # store_metadata is obtained from SPARQL (resource) and not from server (response) because of retrieve_source==True + # else: + # store_metadata_response = result.response # dict + # try: + # resource = self.service.to_resource(result.response) + # except Exception as e: + # self.service.synchronize_resource( + # resource, store_metadata_response, self.search.__name__, False, False + # ) + # raise ValueError(e) + # finally: + # self.service.synchronize_resource( + # resource, store_metadata_response, self.search.__name__, True, False + # ) + # resources.append(resource) + # return resources + + def sparql( + self, query: str, debug: bool, limit: int = None, offset: int = None, **params + ) -> List[Resource]: + rewrite = params.get("rewrite", True) + qr = ( + rewrite_sparql(query, self.model_context, self.service.metadata_context) + if self.model_context is not None and rewrite + else query + ) + qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") + qr = _replace_in_sparql(qr, "OFFSET", offset, 0, r" OFFSET \d+") + if debug: + self._debug_query(qr) + return self._sparql(qr, limit, offset, **params) + + def _sparql(self, query: str, limit: int, offset: int = None, **params) -> List[Resource]: + print('query', query) + print('self.service.sparql_endpoint', self.service.sparql_endpoint) + print('self.service.headers_sparql', self.service.headers_sparql) + try: + wrapper = SPARQLWrapper(self.service.sparql_endpoint["endpoint"]) + wrapper.setQuery(query) + wrapper.setReturnFormat(JSON) + response = wrapper.query() + except Exception as e: + raise QueryingError(e) + else: + data = response.convert() + # FIXME workaround to parse a CONSTRUCT query, this fix depends on + # https://github.com/BlueBrain/nexus/issues/1155 + _, q_comp = Query.parseString(query) + if q_comp.name == "ConstructQuery": + subject_triples = {} + for r in data["results"]["bindings"]: + subject = r["subject"]["value"] + s = f"<{r['subject']['value']}>" + p = f"<{r['predicate']['value']}>" + if r["object"]["type"] == "uri": + o = f"<{r['object']['value']}>" + else: + if "datatype" in r["object"]: + o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" + else: + o = f"\"{r['object']['value']}\"" + if subject in subject_triples: + subject_triples[subject] += f"\n{s} {p} {o} . " + else: + subject_triples[subject] = f"{s} {p} {o} . " + + def triples_to_resource(iri, triples): + graph = Graph().parse(data=triples, format="nt") + data_expanded = json.loads(graph.serialize(format="json-ld")) + data_expanded = json.loads(graph.serialize(format="json-ld")) + frame = {"@id": iri} + data_framed = jsonld.frame(data_expanded, frame) + context = self.model_context or self.context + compacted = jsonld.compact(data_framed, context.document) + resource = from_jsonld(compacted) + resource.context = ( + context.iri + if context.is_http_iri() + else context.document["@context"] + ) + return resource + + return [triples_to_resource(s, t) for s, t in subject_triples.items()] + + else: + # SELECT QUERY + results = data["results"]["bindings"] + return [ + Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') + else (int(v["value"]) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer') + else v["value"] + ) + for k, v in x.items()} ) + for x in results + ] + + def elastic( + self, query: str, debug: bool, limit: int, offset: int + ) -> List[Resource]: + not_supported() + + # Versioning. + + def freeze(self, data: Union[Resource, List[Resource]]) -> None: + not_supported() + + def _freeze_one(self, resource: Resource) -> None: + not_supported() + + # Utils. + + def _initialize_service( + self, + endpoint: Optional[str], + bucket: Optional[str], + token: Optional[str], + searchendpoints: Optional[Dict] = None, + **store_config, + ) -> Any: + try: + max_connection = store_config.pop("max_connection", 50) + if max_connection <= 0: + raise ValueError( + f"max_connection value should be great than 0 but {max_connection} is provided" + ) + store_context_config = store_config.pop("vocabulary", {}) + content_type = store_config.pop("Content-Type", "application/ld+json") + accept = store_config.pop("Accept", "application/ld+json") + params = store_config.pop("params", {}) + except Exception as ve: + raise ValueError(f"Store configuration error: {ve}") + else: + return Service(endpoint=endpoint, model_context=self.model_context, max_connection=max_connection, + searchendpoints=searchendpoints, content_type=content_type, accept=accept, **params) + + @staticmethod + def _debug_query(query): + if isinstance(query, Dict): + print("Submitted query:", query) + else: + print(*["Submitted query:", *query.splitlines()], sep="\n ") + print() \ No newline at end of file From 3e002cd28116b7d8a330c8927979cb0f5d123874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 10 Oct 2022 17:39:42 +0200 Subject: [PATCH 08/30] Added SPARQLWrapper to dependency list. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a1c9fe10..4a09e7ed 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,8 @@ "nest-asyncio>=1.5.1", "pyparsing>=2.0.2", "owlrl>=5.2.3", - "elasticsearch_dsl==7.4.0" + "elasticsearch_dsl==7.4.0", + "SPARQLWrapper" ], extras_require={ "dev": [ From 47f8b0c458187bee5101eefde76d32aebf2c9c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 11 Oct 2022 14:57:27 +0200 Subject: [PATCH 09/30] Recreated the SPARQLStore, and added the specialized UniProtStore. Still missing to define the search method. --- .../17 - Database-sources.ipynb | 2020 ++++++++++++++++- kgforge/core/conversions/rdf.py | 2 +- kgforge/specializations/stores/__init__.py | 1 + kgforge/specializations/stores/sparql.py | 348 +++ .../specializations/stores/uniprot_store.py | 335 +-- 5 files changed, 2341 insertions(+), 365 deletions(-) create mode 100644 kgforge/specializations/stores/sparql.py diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index f03e045e..21441a95 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -634,68 +634,49 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/35463\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", + " abstract: Rationally, an increased intrinsic excitability of dorsal horn neurons could be a factor contributing to alter the gain of the nociceptive system during central sensitization, however direct evidence is scarce. Here we have examined this hypothesis using current and voltage-clamp recordings from dorsal horn neurons in the spinal cord in vitro preparation obtained from mice pups of either sex. Cords were extracted from carrageenan-pretreated and control animals to allow for comparison. Dorsal horn neurons from treated animals showed significantly larger and faster synaptic responses. Synaptic changes started developing shortly after inflammation (1 h) and developed further after a longer-term inflammation (20 h). However, these neurons showed biphasic changes in membrane excitability with an increase shortly after inflammation and a decrease in the longer term. Concomitant changes were observed in transient (I(A)) and sustained potassium currents (I(DR)). Prolonged superfusion of naive spinal cords with NMDA led to a decreased neuronal excitability and to increased potassium currents. Results suggest that excitability plays a role more complex than expected during the process of central sensitization of dorsal horn neurons and that modulation of potassium currents may contribute to shape the changing states of excitability. The decreased excitability observed after long-term inflammation is interpreted as a homeostatic correction to an abnormal state of synaptic activity.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Henderson\n", - " givenName: Z\n", + " familyName: Rivera-Arconada\n", + " givenName: Ivan\n", " }\n", " {\n", " type: Person\n", - " familyName: Morris\n", - " givenName: N P\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Grimwood\n", - " givenName: P\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Fiddler\n", - " givenName: G\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Yang\n", - " givenName: H W\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Appenteng\n", - " givenName: K\n", + " familyName: Lopez-Garcia\n", + " givenName: Jose A\n", " }\n", " ]\n", - " datePublished: 2001-2-12\n", + " datePublished: 2010-4-14\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 11169477\n", + " value: 20392959\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", + " value: 10.1523/JNEUROSCI.4359-09.2010\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0021-9967\n", - " name: The Journal of comparative neurology\n", + " issn: 0270-6474\n", + " name: The Journal of neuroscience : the official journal of the Society for Neuroscience\n", + " publisher: Society for Neuroscience\n", " }\n", - " name: article_91941\n", - " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", - " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", - " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " name: article_35463\n", + " sameAs: http://www.jneurosci.org/content/30/15/5376.long\n", + " title: Changes in membrane excitability and potassium currents in sensitized dorsal horn neurons of mice pups.\n", + " url: http://www.jneurosci.org/content/30/15/5376.long\n", "}\n" ] } @@ -738,41 +719,1952 @@ " ?protein up:reviewed true .\n", " }\n", " LIMIT 10\n", - "\n", - "query \n", - "PREFIX up: \n", - "SELECT ?protein\n", - "WHERE {\n", - " ?protein a up:Protein .\n", - " ?protein up:reviewed true .\n", - "}\n", - " LIMIT 10\n", - "self.service.sparql_endpoint {'endpoint': 'https://sparql.uniprot.org/sparql', 'type': 'sparql'}\n", - "self.service.headers_sparql {'Content-Type': 'text/plain', 'Accept': 'application/sparql-results+json'}\n" + "\n" + ] + } + ], + "source": [ + "uresources = forge.sparql(query=uquery, db_source='UniProt', limit=10, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "https://bbp.neuroshapes.org/SUBCELLULAR LOCATION does not look like a valid URI, trying to serialize this will break.\n", + "https://bbp.neuroshapes.org/ALTERNATIVE PRODUCTS does not look like a valid URI, trying to serialize this will break.\n", + "https://bbp.neuroshapes.org/TISSUE SPECIFICITY does not look like a valid URI, trying to serialize this will break.\n", + "https://bbp.neuroshapes.org/DISRUPTION PHENOTYPE does not look like a valid URI, trying to serialize this will break.\n", + "https://bbp.neuroshapes.org/Topological domain does not look like a valid URI, trying to serialize this will break.\n", + "https://bbp.neuroshapes.org/Alternative sequence does not look like a valid URI, trying to serialize this will break.\n" ] }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"protein\": {\n", + " \"entryType\": \"UniProtKB reviewed (Swiss-Prot)\",\n", + " \"primaryAccession\": \"A0A131MCZ8\",\n", + " \"secondaryAccessions\": [\n", + " \"A0A131MBV5\",\n", + " \"A0A131MD56\",\n", + " \"Q21469\"\n", + " ],\n", + " \"uniProtkbId\": \"CNNM3_CAEEL\",\n", + " \"entryAudit\": {\n", + " \"firstPublicDate\": \"2016-11-30\",\n", + " \"lastAnnotationUpdateDate\": \"2022-08-03\",\n", + " \"lastSequenceUpdateDate\": \"2016-05-11\",\n", + " \"entryVersion\": 37,\n", + " \"sequenceVersion\": 1\n", + " },\n", + " \"annotationScore\": 5.0,\n", + " \"organism\": {\n", + " \"scientificName\": \"Caenorhabditis elegans\",\n", + " \"taxonId\": 6239,\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"Proteomes\",\n", + " \"@id\": \"UP000001940\"\n", + " }\n", + " ],\n", + " \"lineage\": [\n", + " \"Eukaryota\",\n", + " \"Metazoa\",\n", + " \"Ecdysozoa\",\n", + " \"Nematoda\",\n", + " \"Chromadorea\",\n", + " \"Rhabditida\",\n", + " \"Rhabditina\",\n", + " \"Rhabditomorpha\",\n", + " \"Rhabditoidea\",\n", + " \"Rhabditidae\",\n", + " \"Peloderinae\",\n", + " \"Caenorhabditis\"\n", + " ]\n", + " },\n", + " \"proteinExistence\": \"2: Evidence at transcript level\",\n", + " \"proteinDescription\": {\n", + " \"recommendedName\": {\n", + " \"fullName\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ],\n", + " \"value\": \"Metal transporter cnnm-3\"\n", + " }\n", + " },\n", + " \"alternativeNames\": [\n", + " {\n", + " \"fullName\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2a\"\n", + " }\n", + " ],\n", + " \"value\": \"CNNM family homolog 3\"\n", + " }\n", + " }\n", + " ],\n", + " \"flag\": \"Precursor\"\n", + " },\n", + " \"genes\": [\n", + " {\n", + " \"geneName\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2a\"\n", + " }\n", + " ],\n", + " \"value\": \"cnnm-3\"\n", + " },\n", + " \"orfNames\": [\n", + " {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2a\"\n", + " }\n", + " ],\n", + " \"value\": \"C33D12.2\"\n", + " }\n", + " ]\n", + " }\n", + " ],\n", + " \"comments\": [\n", + " {\n", + " \"texts\": [\n", + " {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000269\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ],\n", + " \"value\": \"Probable metal transporter. Probably acts redundantly with the other metal transport proteins cnnm-1, cnnm-2, cnnm-4 and cnnm-5 to regulate Mg(2+) homeostasis. Promotes postembryonic gonad development by regulating Mg(2+) levels, probably via AMPK signaling\"\n", + " }\n", + " ],\n", + " \"commentType\": \"FUNCTION\"\n", + " },\n", + " {\n", + " \"commentType\": \"SUBCELLULAR LOCATION\",\n", + " \"subcellularLocations\": [\n", + " {\n", + " \"location\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000269\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ],\n", + " \"value\": \"Basolateral cell membrane\",\n", + " \"@id\": \"SL-0026\"\n", + " },\n", + " \"topology\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\"\n", + " }\n", + " ],\n", + " \"value\": \"Multi-pass membrane protein\",\n", + " \"@id\": \"SL-9909\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"commentType\": \"ALTERNATIVE PRODUCTS\",\n", + " \"events\": [\n", + " \"Alternative splicing\"\n", + " ],\n", + " \"isoforms\": [\n", + " {\n", + " \"name\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2a\"\n", + " }\n", + " ],\n", + " \"value\": \"a\"\n", + " },\n", + " \"isoformIds\": [\n", + " \"A0A131MCZ8-1\"\n", + " ],\n", + " \"isoformSequenceStatus\": \"Displayed\"\n", + " },\n", + " {\n", + " \"name\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2b\"\n", + " }\n", + " ],\n", + " \"value\": \"b\"\n", + " },\n", + " \"isoformIds\": [\n", + " \"A0A131MCZ8-2\"\n", + " ],\n", + " \"sequenceIds\": [\n", + " \"VSP_058662\"\n", + " ],\n", + " \"isoformSequenceStatus\": \"Described\"\n", + " },\n", + " {\n", + " \"name\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2c\"\n", + " }\n", + " ],\n", + " \"value\": \"c\"\n", + " },\n", + " \"isoformIds\": [\n", + " \"A0A131MCZ8-3\"\n", + " ],\n", + " \"sequenceIds\": [\n", + " \"VSP_058663\",\n", + " \"VSP_058664\"\n", + " ],\n", + " \"isoformSequenceStatus\": \"Described\"\n", + " },\n", + " {\n", + " \"name\": {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"WormBase\",\n", + " \"@id\": \"C33D12.2d\"\n", + " }\n", + " ],\n", + " \"value\": \"d\"\n", + " },\n", + " \"isoformIds\": [\n", + " \"A0A131MCZ8-4\"\n", + " ],\n", + " \"sequenceIds\": [\n", + " \"VSP_058662\",\n", + " \"VSP_058663\",\n", + " \"VSP_058664\"\n", + " ],\n", + " \"isoformSequenceStatus\": \"Described\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"texts\": [\n", + " {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000269\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ],\n", + " \"value\": \"Highly expressed in the intestine and in neurons, but it is also expressed in a variety of tissues including the pharynx, hypodermis, rectum and in muscles\"\n", + " }\n", + " ],\n", + " \"commentType\": \"TISSUE SPECIFICITY\"\n", + " },\n", + " {\n", + " \"texts\": [\n", + " {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000269\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ],\n", + " \"value\": \"No visible phenotype. Double knockout with cnnm-1 results in increased levels of intestinal Mg(2+) and reduced levels in other tissues. This Mg(2+) deficiency in tissues leads to a reduced lifespan, 100% sterility, and smaller animals that exhibit a developmental delay with defective gonad development and which therefore do not produce oocytes or form vulva. In addition, the gonad development defect in the cnnm-1 and cnnm-3 double knockout is rescued when the AMPK alpha subunit aak-2 is also knocked out. Double knockout with cnnm-2 results in 22% sterility. Quintuple knockout with cnnm-1, cnnm-2, cnnm-4 and cnnm-5 results in a reduced lifespan and 100% sterility\"\n", + " }\n", + " ],\n", + " \"commentType\": \"DISRUPTION PHENOTYPE\"\n", + " },\n", + " {\n", + " \"texts\": [\n", + " {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ],\n", + " \"value\": \"Belongs to the ACDP family\"\n", + " }\n", + " ],\n", + " \"commentType\": \"SIMILARITY\"\n", + " }\n", + " ],\n", + " \"features\": [\n", + " {\n", + " \"@type\": \"Signal\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 1,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 23,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Chain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 24,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 797,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Metal transporter cnnm-3\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ],\n", + " \"featureId\": \"PRO_5007283697\"\n", + " },\n", + " {\n", + " \"@type\": \"Topological domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 24,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 200,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Extracellular\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Transmembrane\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 201,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 221,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Helical\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Topological domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 222,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 255,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Cytoplasmic\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Transmembrane\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 256,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 276,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Helical\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Topological domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 277,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 280,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Extracellular\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Transmembrane\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 281,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 301,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Helical\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Topological domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 302,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 322,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Cytoplasmic\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Transmembrane\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 323,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 343,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Helical\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Topological domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 344,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 797,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"Extracellular\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 193,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 373,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"CNNM transmembrane\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU01193\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 393,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 454,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"CBS 1\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00703\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Domain\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 461,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 527,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"CBS 2\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00703\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 31,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 31,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 42,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 42,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 676,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 676,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 692,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 692,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 724,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 724,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 731,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 731,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Glycosylation\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 761,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 761,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000255\",\n", + " \"source\": \"PROSITE-ProRule\",\n", + " \"@id\": \"PRU00498\"\n", + " }\n", + " ],\n", + " \"featureId\": \"\"\n", + " },\n", + " {\n", + " \"@type\": \"Alternative sequence\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 1,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 547,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"in isoform b and isoform d\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ],\n", + " \"featureId\": \"VSP_058662\",\n", + " \"alternativeSequence\": {}\n", + " },\n", + " {\n", + " \"@type\": \"Alternative sequence\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 753,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 766,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"in isoform c and isoform d\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ],\n", + " \"featureId\": \"VSP_058663\",\n", + " \"alternativeSequence\": {\n", + " \"originalSequence\": \"DAVSTPIRNGSVKL\",\n", + " \"alternativeSequences\": [\n", + " \"KYLIGRWKRRGTYV\"\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"@type\": \"Alternative sequence\",\n", + " \"location\": {\n", + " \"start\": {\n", + " \"value\": 767,\n", + " \"modifier\": \"EXACT\"\n", + " },\n", + " \"end\": {\n", + " \"value\": 797,\n", + " \"modifier\": \"EXACT\"\n", + " }\n", + " },\n", + " \"description\": \"in isoform c and isoform d\",\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ],\n", + " \"featureId\": \"VSP_058664\",\n", + " \"alternativeSequence\": {}\n", + " }\n", + " ],\n", + " \"keywords\": [\n", + " {\n", + " \"@id\": \"KW-0025\",\n", + " \"category\": \"Coding sequence diversity\",\n", + " \"name\": \"Alternative splicing\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0129\",\n", + " \"category\": \"Domain\",\n", + " \"name\": \"CBS domain\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-1003\",\n", + " \"category\": \"Cellular component\",\n", + " \"name\": \"Cell membrane\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0325\",\n", + " \"category\": \"PTM\",\n", + " \"name\": \"Glycoprotein\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0406\",\n", + " \"category\": \"Biological process\",\n", + " \"name\": \"Ion transport\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0472\",\n", + " \"category\": \"Cellular component\",\n", + " \"name\": \"Membrane\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-1185\",\n", + " \"category\": \"Technical term\",\n", + " \"name\": \"Reference proteome\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0677\",\n", + " \"category\": \"Domain\",\n", + " \"name\": \"Repeat\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0732\",\n", + " \"category\": \"Domain\",\n", + " \"name\": \"Signal\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0812\",\n", + " \"category\": \"Domain\",\n", + " \"name\": \"Transmembrane\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-1133\",\n", + " \"category\": \"Domain\",\n", + " \"name\": \"Transmembrane helix\"\n", + " },\n", + " {\n", + " \"@id\": \"KW-0813\",\n", + " \"category\": \"Biological process\",\n", + " \"name\": \"Transport\"\n", + " }\n", + " ],\n", + " \"references\": [\n", + " {\n", + " \"citation\": {\n", + " \"@id\": \"9851916\",\n", + " \"citationType\": \"journal article\",\n", + " \"authoringGroup\": [\n", + " \"The C. elegans sequencing consortium\"\n", + " ],\n", + " \"citationCrossReferences\": [\n", + " {\n", + " \"database\": \"PubMed\",\n", + " \"@id\": \"9851916\"\n", + " },\n", + " {\n", + " \"database\": \"DOI\",\n", + " \"@id\": \"10.1126/science.282.5396.2012\"\n", + " }\n", + " ],\n", + " \"title\": \"Genome sequence of the nematode C. elegans: a platform for investigating biology.\",\n", + " \"publicationDate\": \"1998\",\n", + " \"journal\": \"Science\",\n", + " \"firstPage\": \"2012\",\n", + " \"lastPage\": \"2018\",\n", + " \"volume\": \"282\"\n", + " },\n", + " \"referencePositions\": [\n", + " \"NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]\"\n", + " ],\n", + " \"referenceComments\": [\n", + " {\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"Proteomes\",\n", + " \"@id\": \"UP000001940\"\n", + " }\n", + " ],\n", + " \"value\": \"Bristol N2\",\n", + " \"@type\": \"STRAIN\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000312\",\n", + " \"source\": \"Proteomes\",\n", + " \"@id\": \"UP000001940\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"citation\": {\n", + " \"@id\": \"27564576\",\n", + " \"citationType\": \"journal article\",\n", + " \"authors\": [\n", + " \"Ishii T.\",\n", + " \"Funato Y.\",\n", + " \"Hashizume O.\",\n", + " \"Yamazaki D.\",\n", + " \"Hirata Y.\",\n", + " \"Nishiwaki K.\",\n", + " \"Kono N.\",\n", + " \"Arai H.\",\n", + " \"Miki H.\"\n", + " ],\n", + " \"citationCrossReferences\": [\n", + " {\n", + " \"database\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " },\n", + " {\n", + " \"database\": \"DOI\",\n", + " \"@id\": \"10.1371/journal.pgen.1006276\"\n", + " }\n", + " ],\n", + " \"title\": \"Mg2+ extrusion from intestinal epithelia by CNNM proteins is essential for gonadogenesis via AMPK-TORC1 signaling in Caenorhabditis elegans.\",\n", + " \"publicationDate\": \"2016\",\n", + " \"journal\": \"PLoS Genet.\",\n", + " \"firstPage\": \"E1006276\",\n", + " \"lastPage\": \"E1006276\",\n", + " \"volume\": \"12\"\n", + " },\n", + " \"referencePositions\": [\n", + " \"FUNCTION\",\n", + " \"SUBCELLULAR LOCATION\",\n", + " \"TISSUE SPECIFICITY\",\n", + " \"DISRUPTION PHENOTYPE\"\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000305\"\n", + " }\n", + " ]\n", + " }\n", + " ],\n", + " \"uniProtKBCrossReferences\": [\n", + " {\n", + " \"database\": \"EMBL\",\n", + " \"@id\": \"BX284606\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CCD66487.1\"\n", + " },\n", + " {\n", + " \"key\": \"Status\",\n", + " \"value\": \"-\"\n", + " },\n", + " {\n", + " \"key\": \"MoleculeType\",\n", + " \"value\": \"Genomic_DNA\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"EMBL\",\n", + " \"@id\": \"BX284606\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CZR14596.1\"\n", + " },\n", + " {\n", + " \"key\": \"Status\",\n", + " \"value\": \"-\"\n", + " },\n", + " {\n", + " \"key\": \"MoleculeType\",\n", + " \"value\": \"Genomic_DNA\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"EMBL\",\n", + " \"@id\": \"BX284606\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CZR14597.1\"\n", + " },\n", + " {\n", + " \"key\": \"Status\",\n", + " \"value\": \"-\"\n", + " },\n", + " {\n", + " \"key\": \"MoleculeType\",\n", + " \"value\": \"Genomic_DNA\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"EMBL\",\n", + " \"@id\": \"BX284606\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CZR14608.1\"\n", + " },\n", + " {\n", + " \"key\": \"Status\",\n", + " \"value\": \"-\"\n", + " },\n", + " {\n", + " \"key\": \"MoleculeType\",\n", + " \"value\": \"Genomic_DNA\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"PIR\",\n", + " \"@id\": \"T16631\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"T16631\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"RefSeq\",\n", + " \"@id\": \"NP_001309670.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"NucleotideSequenceId\",\n", + " \"value\": \"NM_001322613.1\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-1\"\n", + " },\n", + " {\n", + " \"database\": \"RefSeq\",\n", + " \"@id\": \"NP_001309671.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"NucleotideSequenceId\",\n", + " \"value\": \"NM_001322614.1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"RefSeq\",\n", + " \"@id\": \"NP_001309682.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"NucleotideSequenceId\",\n", + " \"value\": \"NM_001322615.1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"RefSeq\",\n", + " \"@id\": \"NP_508520.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"NucleotideSequenceId\",\n", + " \"value\": \"NM_076119.3\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"AlphaFoldDB\",\n", + " \"@id\": \"A0A131MCZ8\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"SMR\",\n", + " \"@id\": \"A0A131MCZ8\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"IntAct\",\n", + " \"@id\": \"A0A131MCZ8\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Interactions\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"STRING\",\n", + " \"@id\": \"6239.C33D12.2\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"PaxDb\",\n", + " \"@id\": \"A0A131MCZ8\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2a.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2a.1\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-1\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2b.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2b.1\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2b.2\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2b.2\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2b.3\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2b.3\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2b.4\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2b.4\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2b.5\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2b.5\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2b.6\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2b.6\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2c.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2c.1\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-3\"\n", + " },\n", + " {\n", + " \"database\": \"EnsemblMetazoa\",\n", + " \"@id\": \"C33D12.2d.1\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"C33D12.2d.1\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-4\"\n", + " },\n", + " {\n", + " \"database\": \"GeneID\",\n", + " \"@id\": \"180591\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"KEGG\",\n", + " \"@id\": \"cel:CELE_C33D12.2\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"UCSC\",\n", + " \"@id\": \"M02F4.3\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"OrganismName\",\n", + " \"value\": \"c. elegans\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"CTD\",\n", + " \"@id\": \"180591\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"WormBase\",\n", + " \"@id\": \"C33D12.2a\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CE51453\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " },\n", + " {\n", + " \"key\": \"GeneName\",\n", + " \"value\": \"cnnm-3\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-1\"\n", + " },\n", + " {\n", + " \"database\": \"WormBase\",\n", + " \"@id\": \"C33D12.2b\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CE04764\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " },\n", + " {\n", + " \"key\": \"GeneName\",\n", + " \"value\": \"cnnm-3\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-2\"\n", + " },\n", + " {\n", + " \"database\": \"WormBase\",\n", + " \"@id\": \"C33D12.2c\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CE51420\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " },\n", + " {\n", + " \"key\": \"GeneName\",\n", + " \"value\": \"cnnm-3\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-3\"\n", + " },\n", + " {\n", + " \"database\": \"WormBase\",\n", + " \"@id\": \"C33D12.2d\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ProteinId\",\n", + " \"value\": \"CE51314\"\n", + " },\n", + " {\n", + " \"key\": \"GeneId\",\n", + " \"value\": \"WBGene00016343\"\n", + " },\n", + " {\n", + " \"key\": \"GeneName\",\n", + " \"value\": \"cnnm-3\"\n", + " }\n", + " ],\n", + " \"isoformId\": \"A0A131MCZ8-4\"\n", + " },\n", + " {\n", + " \"database\": \"eggNOG\",\n", + " \"@id\": \"KOG2118\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ToxonomicScope\",\n", + " \"value\": \"Eukaryota\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GeneTree\",\n", + " \"@id\": \"ENSGT00940000169533\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"OMA\",\n", + " \"@id\": \"MSTCHKI\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Fingerprint\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"OrthoDB\",\n", + " \"@id\": \"1446644at2759\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"PRO\",\n", + " \"@id\": \"PR:A0A131MCZ8\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Description\",\n", + " \"value\": \"-\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"Proteomes\",\n", + " \"@id\": \"UP000001940\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"Component\",\n", + " \"value\": \"Chromosome X\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"Bgee\",\n", + " \"@id\": \"WBGene00016343\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"ExpressionPatterns\",\n", + " \"value\": \"Expressed in larva and 3 other tissues\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0016323\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"C:basolateral plasma membrane\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IDA:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000314\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0016021\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"C:integral component of membrane\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IEA:UniProtKB-KW\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0043231\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"C:intracellular membrane-bounded organelle\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IBA:GO_Central\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000318\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"21873635\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0005886\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"C:plasma membrane\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IBA:GO_Central\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000318\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"21873635\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0022857\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"F:transmembrane transporter activity\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IBA:GO_Central\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000318\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"21873635\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0008340\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:determination of adult lifespan\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IMP:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000315\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0045087\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:innate immune response\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"HEP:WormBase\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0007007\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"16968778\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0010960\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:magnesium ion homeostasis\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IGI:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0015693\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:magnesium ion transport\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IGI:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:1905941\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:positive regulation of gonad development\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IGI:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " },\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " },\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0040018\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:positive regulation of multicellular organism growth\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IGI:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0040026\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:positive regulation of vulval development\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IGI:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"GO\",\n", + " \"@id\": \"GO:0032026\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"GoTerm\",\n", + " \"value\": \"P:response to magnesium ion\"\n", + " },\n", + " {\n", + " \"key\": \"GoEvidenceType\",\n", + " \"value\": \"IGI:UniProtKB\"\n", + " }\n", + " ],\n", + " \"evidences\": [\n", + " {\n", + " \"evidenceCode\": \"ECO:0000316\",\n", + " \"source\": \"PubMed\",\n", + " \"@id\": \"27564576\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"CDD\",\n", + " \"@id\": \"cd04590\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"CBS_pair_CorC_HlyC_assoc\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"Gene3D\",\n", + " \"@id\": \"3.10.580.10\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"-\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"InterPro\",\n", + " \"@id\": \"IPR045095\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"ACDP\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"InterPro\",\n", + " \"@id\": \"IPR000644\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"CBS_dom\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"InterPro\",\n", + " \"@id\": \"IPR046342\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"CBS_dom_sf\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"InterPro\",\n", + " \"@id\": \"IPR002550\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"CNNM\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"InterPro\",\n", + " \"@id\": \"IPR044751\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"Ion_transp-like_CBS\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"PANTHER\",\n", + " \"@id\": \"PTHR12064\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"PTHR12064\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"Pfam\",\n", + " \"@id\": \"PF01595\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"DUF21\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"SUPFAM\",\n", + " \"@id\": \"SSF54631\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"SSF54631\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"PROSITE\",\n", + " \"@id\": \"PS51371\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"CBS\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"2\"\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"database\": \"PROSITE\",\n", + " \"@id\": \"PS51846\",\n", + " \"properties\": [\n", + " {\n", + " \"key\": \"EntryName\",\n", + " \"value\": \"CNNM\"\n", + " },\n", + " {\n", + " \"key\": \"MatchStatus\",\n", + " \"value\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " ],\n", + " \"sequence\": {\n", + " \"value\": \"MSKTPWALGLLIFLLTFTSPLSSSPVRSTDNSTSSKGLLNVNSSVILEPSILPSSASKPESLHLSKVRVSGLRLEAHASSTENIVLGHNKKHNVVVVPNKNVRVVLFGQNFQDIGALTFTADGSCKDLAHFFEADFSSMTPIRVVVEMSFPKTTESKDSFKLCVSEKFYANPQFVIVEDPFTMVTTEIPPVDEYMPKWLSWICLLILLCFSGLFSGLNLGLMTLSPYELQLYIASGTEQEKRDAGRILPIRKKGNQLLCTLLIGNVVVNVGVSLLMDQLVGSGFAVLVAATSCIVVFGEIIPQALCVKLGLPIGARTIPITQVLLFLMYPLTWPISKVLDIFLKEELTRSLERNKLVEMLKLSEKSIIGGQSDEFKMVLGALELYDKTVAHAMTRYEDIFMLPHTLTLGAGMVTQILDMGYTRIPIYENDRKNIVALLFVKDLALLDPDDNHNVMKIASIYNHEVRRVLVDMPLRNMLEEFKRGEYHMALVERLVEQEDKDPIYELCGLITLEDIIEEIIQCEIIDETDAVCDNVHRKKRQRKRNHDMSQIVNTAHAKCAINIQMLAVTIQVMSTCHKIFSSNYILPTILEKLIRKNCKKVETTQFSCLKEVGVVQPKPAVLFTKGEFSNKFIMILSGRAVVTIGKEEMRLEAGAWHSFGTEVLDAMAEAIERSLNQSTSRSTVSLNTEITNNSIGFIPDFDTVILYECVFCEITAADLLLAYNSSQIMQNNTKMQVVRSNSRISLIEEIPKDAVSTPIRNGSVKLRTVSEGETVHLLPKNMECHFNKQEKYEEEEE\",\n", + " \"length\": 797,\n", + " \"molWeight\": 89288,\n", + " \"crc64\": \"25AD3EA350EB4E85\",\n", + " \"md5\": \"CDB9455358FB9B5496A1D047EF2A79FD\"\n", + " },\n", + " \"extraAttributes\": {\n", + " \"countByCommentType\": {\n", + " \"FUNCTION\": 1,\n", + " \"SUBCELLULAR LOCATION\": 1,\n", + " \"ALTERNATIVE PRODUCTS\": 4,\n", + " \"TISSUE SPECIFICITY\": 1,\n", + " \"DISRUPTION PHENOTYPE\": 1,\n", + " \"SIMILARITY\": 1\n", + " },\n", + " \"countByFeatureType\": {\n", + " \"Signal\": 1,\n", + " \"Chain\": 1,\n", + " \"Topological domain\": 5,\n", + " \"Transmembrane\": 4,\n", + " \"Domain\": 3,\n", + " \"Glycosylation\": 7,\n", + " \"Alternative sequence\": 3\n", + " },\n", + " \"uniParcId\": \"UPI00077FE610\"\n", + " },\n", + " \"@id\": \"http://purl.uniprot.org/uniprot/A0A131MCZ8\"\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(forge.as_jsonld(uresources[0]), indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "import requests" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post('https://sparql.uniprot.org/sparql', data=uquery,\n", + " headers={'Content-Type': 'text/plain', 'Accept': 'application/sparql-results+json'},\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ { "data": { "text/plain": [ - "[Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/A0A131MCZ8'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/D0PX85'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/G5EF51'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/I0DF35'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P0C645'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P53500'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIL8'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIL9'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIM0'),\n", - " Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/P9WIM1')]" + "" ] }, - "execution_count": 23, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "forge.sparql(query=uquery, db_source='UniProt', limit=10, debug=True)" + "response" ] }, { @@ -784,7 +2676,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -807,7 +2699,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -824,7 +2716,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -852,7 +2744,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -889,7 +2781,7 @@ "Index: []" ] }, - "execution_count": 27, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -910,7 +2802,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 32, "metadata": {}, "outputs": [ { diff --git a/kgforge/core/conversions/rdf.py b/kgforge/core/conversions/rdf.py index 3d662c8c..bd48955e 100644 --- a/kgforge/core/conversions/rdf.py +++ b/kgforge/core/conversions/rdf.py @@ -405,7 +405,7 @@ def _unpack_from_list(data): def _add_ld_keys( - rsc: [Resource, Dict], + rsc: Union[Resource, Dict], context: Optional[Union[Dict, List, str]], base: Optional[str], ) -> Union[Dict, List[str]]: diff --git a/kgforge/specializations/stores/__init__.py b/kgforge/specializations/stores/__init__.py index 9a89dfe5..def3492d 100644 --- a/kgforge/specializations/stores/__init__.py +++ b/kgforge/specializations/stores/__init__.py @@ -14,4 +14,5 @@ from .bluebrain_nexus import BlueBrainNexus from .demo_store import DemoStore +from .sparql import SPARQLStore from .uniprot_store import UniProtStore diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py new file mode 100644 index 00000000..079e605e --- /dev/null +++ b/kgforge/specializations/stores/sparql.py @@ -0,0 +1,348 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import json +from pyld import jsonld +from copy import deepcopy +from pathlib import Path +from typing import Dict, List, Optional, Union, Any +from uuid import uuid4 +import requests +from rdflib import Graph +from rdflib.plugins.sparql.parser import Query +from requests import HTTPError +from SPARQLWrapper import SPARQLWrapper, JSON + +from kgforge.core import Resource +from kgforge.core.archetypes import Resolver, Store +from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql +from kgforge.core.commons.context import Context +from kgforge.core.wrappings.dict import DictWrapper +from kgforge.specializations.stores.uniprot.service import Service +from kgforge.core.commons.exceptions import DownloadingError, QueryingError +from kgforge.core.commons.execution import not_supported +from kgforge.core.conversions.rdf import as_jsonld, from_jsonld + + +class SPARQLStore(Store): + """A Store specialized for SPARQL queries, supporting only Reading (searching) methods.""" + + def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, + token: Optional[str] = None, versioned_id_template: Optional[str] = None, + file_resource_mapping: Optional[str] = None, + model_context: Optional[Context] = None, + searchendpoints: Optional[Dict] = None,) -> None: + super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, + model_context, searchendpoints) + + + # [C]RUD + + def register(self, data: Union[Resource, List[Resource]], schema_id: str = None + ) -> None: + # Replace None by self._register_many to switch to optimized bulk registration. + not_supported() + + def _register_one(self, resource: Resource, schema_id: str) -> None: + not_supported() + + def upload(self, path: str, content_type: str) -> Union[Resource, List[Resource]]: + not_supported() + + def _upload(path: Path, content_type: str) -> Union[Any, List[Any]]: + not_supported() + + def _upload_many(self, paths: List[Path], content_type: str) -> List[Any]: + not_supported() + + def _download_one( + self, + url: str, + path: str, + store_metadata: Optional[DictWrapper], + cross_bucket: bool, + ) -> None: + # path: FilePath. + # TODO define dowloading method + # POLICY Should notify of failures with exception DownloadingError including a message. + not_supported() + + # C[R]UD + + def retrieve( + self, id: str, version: Optional[Union[int, str]], cross_bucket: bool, **params + ) -> Resource: + not_supported() + + def _retrieve_filename(self, id: str) -> str: + not_supported() + + # CR[U]D. + + def update( + self, data: Union[Resource, List[Resource]], schema_id: Optional[str] + ) -> None: + # Replace None by self._update_many to switch to optimized bulk update. + not_supported() + + def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: + not_supported() + + def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + # Replace None by self._tag_many to switch to optimized bulk tagging. + # POLICY If tagging modify the resource, run() should have status='_synchronized'. + not_supported() + + # CRU[D]. + + def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + # Replace None by self._deprecate_many to switch to optimized bulk deprecation. + not_supported() + + # Querying. + + def search( + self, resolvers: Optional[List["Resolver"]], *filters, **params + ) -> List[Resource]: + # Positional arguments in 'filters' are instances of type Filter from wrappings/paths.py + # A dictionary can be provided for filters: + # - {'key1': 'val', 'key2': {'key3': 'val'}} will be translated to + # - [Filter(operator='__eq__', path=['key1'], value='val'), Filter(operator='__eq__', path=['key2', 'key3'], value='val')] + # Keyword arguments in 'params' could be: + # - debug: bool, + # - limit: int, + # - offset: int, + # - deprecated: bool, + # - resolving: str, with values in ('exact', 'fuzzy'), + # - lookup: str, with values in ('current', 'children'). + # POLICY Should use sparql() and contain 'search_endpoint'. + # POLICY Resource _store_metadata should be set using wrappers.dict.wrap_dict(). + # POLICY Resource _synchronized should be set to True. + pass + # if self.model_context is None: + # raise ValueError("context model missing") + + # debug = params.get("debug", False) + # limit = params.get("limit", 100) + # offset = params.get("offset", None) + # deprecated = params.get("deprecated", False) + # distinct = params.get("distinct", False) + # includes = params.get("includes", None) + # excludes = params.get("excludes", None) + # retrieve_source = params.get("retrieve_source") + # search_endpoint = params.get( + # "search_endpoint", self.service.sparql_endpoint["type"] + # ) + # if search_endpoint not in [ + # self.service.sparql_endpoint["type"], + # ]: + # raise ValueError( + # f"The provided search_endpoint value '{search_endpoint}' is not supported, only 'sparql'" + # ) + # if "filters" in params: + # raise ValueError("A 'filters' key was provided as params. Filters should be provided as iterable to be unpacked.") + + + # if filters and isinstance(filters[0], dict): + # filters = create_filters_from_dict(filters[0]) + # filters = list(filters) if not isinstance(filters, list) else filters + + # if includes or excludes: + # raise ValueError( + # "Field inclusion and exclusion are not supported when using SPARQL" + # ) + + # query_statements, query_filters = build_sparql_query_statements( + # self.model_context, filters + # ) + # store_metadata_statements = [] + # if retrieve_source: + # _vars = ["?id"] + # for i, k in enumerate(self.service.store_metadata_keys): + # _vars.append(f"?{k}") + # store_metadata_statements.insert(i+2, f"<{self.metadata_context.terms[k].id}> ?{k}") + # deprecated_filter = f"Filter (?_deprecated = {format_type[CategoryDataType.BOOLEAN](deprecated)})" + # query_filters.append(deprecated_filter) + # else: + # _vars = ["?id", "?_project", "?_rev"] + # store_metadata_statements.append(f"<{self.service.revision_property}> ?_rev") + # store_metadata_statements.append(f"<{self.service.project_property}> ?_project") + # query_statements.append( + # f"<{self.service.deprecated_property}> {format_type[CategoryDataType.BOOLEAN](deprecated)}", + # ) + # query_statements.extend(store_metadata_statements) + # statements = ";\n ".join(query_statements) + # _filters = "\n".join(".\n ".join(query_filters)) + # query = _create_select_query( + # _vars, f"?id {statements} . \n {_filters}", distinct + # ) + # # support @id and @type + # resources = self.sparql(query, debug=debug, limit=limit, offset=offset) + # params_retrieve = deepcopy(self.service.params.get("retrieve", {})) + # params_retrieve['retrieve_source'] = retrieve_source + # results = self.service.batch_request( + # resources, BatchAction.FETCH, None, QueryingError, params=params_retrieve + # ) + # resources = list() + # for result in results: + # resource = result.resource + # if retrieve_source: + # store_metadata_response = as_json(result.resource, expanded=False, store_metadata=False, + # model_context=None, + # metadata_context=None, + # context_resolver=None) # store_metadata is obtained from SPARQL (resource) and not from server (response) because of retrieve_source==True + # else: + # store_metadata_response = result.response # dict + # try: + # resource = self.service.to_resource(result.response) + # except Exception as e: + # self.service.synchronize_resource( + # resource, store_metadata_response, self.search.__name__, False, False + # ) + # raise ValueError(e) + # finally: + # self.service.synchronize_resource( + # resource, store_metadata_response, self.search.__name__, True, False + # ) + # resources.append(resource) + # return resources + + def sparql( + self, query: str, debug: bool, limit: int = None, offset: int = None, **params + ) -> List[Resource]: + rewrite = params.get("rewrite", True) + qr = ( + rewrite_sparql(query, self.model_context, self.service.metadata_context) + if self.model_context is not None and rewrite + else query + ) + qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") + qr = _replace_in_sparql(qr, "OFFSET", offset, 0, r" OFFSET \d+") + if debug: + self._debug_query(qr) + return self._sparql(qr, limit, offset, **params) + + def _sparql(self, query: str, limit: int, offset: int = None, **params) -> List[Resource]: + try: + wrapper = SPARQLWrapper(self.service.sparql_endpoint["endpoint"]) + wrapper.setQuery(query) + wrapper.setReturnFormat(JSON) + response = wrapper.query() + except Exception as e: + raise QueryingError(e) + else: + data = response.convert() + # FIXME workaround to parse a CONSTRUCT query, this fix depends on + # https://github.com/BlueBrain/nexus/issues/1155 + _, q_comp = Query.parseString(query) + if q_comp.name == "ConstructQuery": + subject_triples = {} + for r in data["results"]["bindings"]: + subject = r["subject"]["value"] + s = f"<{r['subject']['value']}>" + p = f"<{r['predicate']['value']}>" + if r["object"]["type"] == "uri": + o = f"<{r['object']['value']}>" + else: + if "datatype" in r["object"]: + o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" + else: + o = f"\"{r['object']['value']}\"" + if subject in subject_triples: + subject_triples[subject] += f"\n{s} {p} {o} . " + else: + subject_triples[subject] = f"{s} {p} {o} . " + + def triples_to_resource(iri, triples): + graph = Graph().parse(data=triples, format="nt") + data_expanded = json.loads(graph.serialize(format="json-ld")) + data_expanded = json.loads(graph.serialize(format="json-ld")) + frame = {"@id": iri} + data_framed = jsonld.frame(data_expanded, frame) + context = self.model_context or self.context + compacted = jsonld.compact(data_framed, context.document) + resource = from_jsonld(compacted) + resource.context = ( + context.iri + if context.is_http_iri() + else context.document["@context"] + ) + return resource + + return [triples_to_resource(s, t) for s, t in subject_triples.items()] + + else: + # SELECT QUERY + results = data["results"]["bindings"] + return self.resources_from_results(results) + + @staticmethod + def resources_from_results(results): + return [ + Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') + else (int(v["value"]) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer') + else v["value"] + ) + for k, v in x.items()} ) + for x in results + ] + + def elastic( + self, query: str, debug: bool, limit: int, offset: int + ) -> List[Resource]: + not_supported() + + # Versioning. + + def freeze(self, data: Union[Resource, List[Resource]]) -> None: + not_supported() + + def _freeze_one(self, resource: Resource) -> None: + not_supported() + + # Utils. + + def _initialize_service( + self, + endpoint: Optional[str], + bucket: Optional[str], + token: Optional[str], + searchendpoints: Optional[Dict] = None, + **store_config, + ) -> Any: + try: + max_connection = store_config.pop("max_connection", 50) + if max_connection <= 0: + raise ValueError( + f"max_connection value should be great than 0 but {max_connection} is provided" + ) + store_context_config = store_config.pop("vocabulary", {}) + content_type = store_config.pop("Content-Type", "application/ld+json") + accept = store_config.pop("Accept", "application/ld+json") + params = store_config.pop("params", {}) + except Exception as ve: + raise ValueError(f"Store configuration error: {ve}") + else: + return Service(endpoint=endpoint, model_context=self.model_context, max_connection=max_connection, + searchendpoints=searchendpoints, content_type=content_type, accept=accept, **params) + + @staticmethod + def _debug_query(query): + if isinstance(query, Dict): + print("Submitted query:", query) + else: + print(*["Submitted query:", *query.splitlines()], sep="\n ") + print() \ No newline at end of file diff --git a/kgforge/specializations/stores/uniprot_store.py b/kgforge/specializations/stores/uniprot_store.py index 74dbec67..36ccfe31 100644 --- a/kgforge/specializations/stores/uniprot_store.py +++ b/kgforge/specializations/stores/uniprot_store.py @@ -33,19 +33,25 @@ # format_type, # _create_select_query # ) +from kgforge.specializations.stores import SPARQLStore from kgforge.core.commons.context import Context from kgforge.core.wrappings.dict import DictWrapper from kgforge.specializations.stores.uniprot.service import Service -from kgforge.core.commons.exceptions import DownloadingError, QueryingError +from kgforge.core.commons.exceptions import QueryingError from kgforge.core.commons.execution import not_supported from kgforge.core.conversions.rdf import as_jsonld, from_jsonld # from kgforge.core.conversions.json import as_json, from_json # from kgforge.core.wrappings.dict import wrap_dict # from kgforge.core.wrappings.paths import create_filters_from_dict # from urllib.parse import quote_plus, unquote, urlparse, parse_qs + + +UNIPROT_RESTAPI_URL = 'https://rest.uniprot.org/uniprotkb' +UNIPROT_ENTITY_PREFIX = "http://purl.uniprot.org" + +class UniProtStore(SPARQLStore): -class UniProtStore(Store): """A Store specialized for SPARQL queries, supporting only Reading (searching) methods.""" def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, @@ -56,302 +62,31 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, model_context, searchendpoints) - - # [C]RUD - - def register(self, data: Union[Resource, List[Resource]], schema_id: str = None - ) -> None: - # Replace None by self._register_many to switch to optimized bulk registration. - not_supported() - - def _register_one(self, resource: Resource, schema_id: str) -> None: - not_supported() - - def upload(self, path: str, content_type: str) -> Union[Resource, List[Resource]]: - not_supported() - - def _upload(path: Path, content_type: str) -> Union[Any, List[Any]]: - not_supported() - - def _upload_many(self, paths: List[Path], content_type: str) -> List[Any]: - not_supported() - - def _download_one( - self, - url: str, - path: str, - store_metadata: Optional[DictWrapper], - cross_bucket: bool, - ) -> None: - # path: FilePath. - # TODO define dowloading method - # POLICY Should notify of failures with exception DownloadingError including a message. - not_supported() - - # C[R]UD - - def retrieve( - self, id: str, version: Optional[Union[int, str]], cross_bucket: bool, **params - ) -> Resource: - not_supported() - - def _retrieve_filename(self, id: str) -> str: - not_supported() - - # CR[U]D. - - def update( - self, data: Union[Resource, List[Resource]], schema_id: Optional[str] - ) -> None: - # Replace None by self._update_many to switch to optimized bulk update. - not_supported() - - def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: - not_supported() - - def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: - # Replace None by self._tag_many to switch to optimized bulk tagging. - # POLICY If tagging modify the resource, run() should have status='_synchronized'. - not_supported() - - # CRU[D]. - - def deprecate(self, data: Union[Resource, List[Resource]]) -> None: - # Replace None by self._deprecate_many to switch to optimized bulk deprecation. - not_supported() - - # Querying. - - def search( - self, resolvers: Optional[List["Resolver"]], *filters, **params - ) -> List[Resource]: - # Positional arguments in 'filters' are instances of type Filter from wrappings/paths.py - # A dictionary can be provided for filters: - # - {'key1': 'val', 'key2': {'key3': 'val'}} will be translated to - # - [Filter(operator='__eq__', path=['key1'], value='val'), Filter(operator='__eq__', path=['key2', 'key3'], value='val')] - # Keyword arguments in 'params' could be: - # - debug: bool, - # - limit: int, - # - offset: int, - # - deprecated: bool, - # - resolving: str, with values in ('exact', 'fuzzy'), - # - lookup: str, with values in ('current', 'children'). - # POLICY Should use sparql() and contain 'search_endpoint'. - # POLICY Resource _store_metadata should be set using wrappers.dict.wrap_dict(). - # POLICY Resource _synchronized should be set to True. - pass - # if self.model_context is None: - # raise ValueError("context model missing") - - # debug = params.get("debug", False) - # limit = params.get("limit", 100) - # offset = params.get("offset", None) - # deprecated = params.get("deprecated", False) - # distinct = params.get("distinct", False) - # includes = params.get("includes", None) - # excludes = params.get("excludes", None) - # retrieve_source = params.get("retrieve_source") - # search_endpoint = params.get( - # "search_endpoint", self.service.sparql_endpoint["type"] - # ) - # if search_endpoint not in [ - # self.service.sparql_endpoint["type"], - # ]: - # raise ValueError( - # f"The provided search_endpoint value '{search_endpoint}' is not supported, only 'sparql'" - # ) - # if "filters" in params: - # raise ValueError("A 'filters' key was provided as params. Filters should be provided as iterable to be unpacked.") - - - # if filters and isinstance(filters[0], dict): - # filters = create_filters_from_dict(filters[0]) - # filters = list(filters) if not isinstance(filters, list) else filters - - # if includes or excludes: - # raise ValueError( - # "Field inclusion and exclusion are not supported when using SPARQL" - # ) - - # query_statements, query_filters = build_sparql_query_statements( - # self.model_context, filters - # ) - # store_metadata_statements = [] - # if retrieve_source: - # _vars = ["?id"] - # for i, k in enumerate(self.service.store_metadata_keys): - # _vars.append(f"?{k}") - # store_metadata_statements.insert(i+2, f"<{self.metadata_context.terms[k].id}> ?{k}") - # deprecated_filter = f"Filter (?_deprecated = {format_type[CategoryDataType.BOOLEAN](deprecated)})" - # query_filters.append(deprecated_filter) - # else: - # _vars = ["?id", "?_project", "?_rev"] - # store_metadata_statements.append(f"<{self.service.revision_property}> ?_rev") - # store_metadata_statements.append(f"<{self.service.project_property}> ?_project") - # query_statements.append( - # f"<{self.service.deprecated_property}> {format_type[CategoryDataType.BOOLEAN](deprecated)}", - # ) - # query_statements.extend(store_metadata_statements) - # statements = ";\n ".join(query_statements) - # _filters = "\n".join(".\n ".join(query_filters)) - # query = _create_select_query( - # _vars, f"?id {statements} . \n {_filters}", distinct - # ) - # # support @id and @type - # resources = self.sparql(query, debug=debug, limit=limit, offset=offset) - # params_retrieve = deepcopy(self.service.params.get("retrieve", {})) - # params_retrieve['retrieve_source'] = retrieve_source - # results = self.service.batch_request( - # resources, BatchAction.FETCH, None, QueryingError, params=params_retrieve - # ) - # resources = list() - # for result in results: - # resource = result.resource - # if retrieve_source: - # store_metadata_response = as_json(result.resource, expanded=False, store_metadata=False, - # model_context=None, - # metadata_context=None, - # context_resolver=None) # store_metadata is obtained from SPARQL (resource) and not from server (response) because of retrieve_source==True - # else: - # store_metadata_response = result.response # dict - # try: - # resource = self.service.to_resource(result.response) - # except Exception as e: - # self.service.synchronize_resource( - # resource, store_metadata_response, self.search.__name__, False, False - # ) - # raise ValueError(e) - # finally: - # self.service.synchronize_resource( - # resource, store_metadata_response, self.search.__name__, True, False - # ) - # resources.append(resource) - # return resources - - def sparql( - self, query: str, debug: bool, limit: int = None, offset: int = None, **params - ) -> List[Resource]: - rewrite = params.get("rewrite", True) - qr = ( - rewrite_sparql(query, self.model_context, self.service.metadata_context) - if self.model_context is not None and rewrite - else query - ) - qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") - qr = _replace_in_sparql(qr, "OFFSET", offset, 0, r" OFFSET \d+") - if debug: - self._debug_query(qr) - return self._sparql(qr, limit, offset, **params) - - def _sparql(self, query: str, limit: int, offset: int = None, **params) -> List[Resource]: - print('query', query) - print('self.service.sparql_endpoint', self.service.sparql_endpoint) - print('self.service.headers_sparql', self.service.headers_sparql) - try: - wrapper = SPARQLWrapper(self.service.sparql_endpoint["endpoint"]) - wrapper.setQuery(query) - wrapper.setReturnFormat(JSON) - response = wrapper.query() - except Exception as e: - raise QueryingError(e) - else: - data = response.convert() - # FIXME workaround to parse a CONSTRUCT query, this fix depends on - # https://github.com/BlueBrain/nexus/issues/1155 - _, q_comp = Query.parseString(query) - if q_comp.name == "ConstructQuery": - subject_triples = {} - for r in data["results"]["bindings"]: - subject = r["subject"]["value"] - s = f"<{r['subject']['value']}>" - p = f"<{r['predicate']['value']}>" - if r["object"]["type"] == "uri": - o = f"<{r['object']['value']}>" - else: - if "datatype" in r["object"]: - o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" - else: - o = f"\"{r['object']['value']}\"" - if subject in subject_triples: - subject_triples[subject] += f"\n{s} {p} {o} . " - else: - subject_triples[subject] = f"{s} {p} {o} . " - - def triples_to_resource(iri, triples): - graph = Graph().parse(data=triples, format="nt") - data_expanded = json.loads(graph.serialize(format="json-ld")) - data_expanded = json.loads(graph.serialize(format="json-ld")) - frame = {"@id": iri} - data_framed = jsonld.frame(data_expanded, frame) - context = self.model_context or self.context - compacted = jsonld.compact(data_framed, context.document) - resource = from_jsonld(compacted) - resource.context = ( - context.iri - if context.is_http_iri() - else context.document["@context"] - ) - return resource - - return [triples_to_resource(s, t) for s, t in subject_triples.items()] - - else: - # SELECT QUERY - results = data["results"]["bindings"] - return [ - Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and - ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') - else (int(v["value"]) if v['type'] =='literal' and - ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer') - else v["value"] - ) - for k, v in x.items()} ) - for x in results - ] - - def elastic( - self, query: str, debug: bool, limit: int, offset: int - ) -> List[Resource]: - not_supported() - - # Versioning. - - def freeze(self, data: Union[Resource, List[Resource]]) -> None: - not_supported() - - def _freeze_one(self, resource: Resource) -> None: - not_supported() - - # Utils. - - def _initialize_service( - self, - endpoint: Optional[str], - bucket: Optional[str], - token: Optional[str], - searchendpoints: Optional[Dict] = None, - **store_config, - ) -> Any: - try: - max_connection = store_config.pop("max_connection", 50) - if max_connection <= 0: - raise ValueError( - f"max_connection value should be great than 0 but {max_connection} is provided" - ) - store_context_config = store_config.pop("vocabulary", {}) - content_type = store_config.pop("Content-Type", "application/ld+json") - accept = store_config.pop("Accept", "application/ld+json") - params = store_config.pop("params", {}) - except Exception as ve: - raise ValueError(f"Store configuration error: {ve}") - else: - return Service(endpoint=endpoint, model_context=self.model_context, max_connection=max_connection, - searchendpoints=searchendpoints, content_type=content_type, accept=accept, **params) - @staticmethod - def _debug_query(query): - if isinstance(query, Dict): - print("Submitted query:", query) - else: - print(*["Submitted query:", *query.splitlines()], sep="\n ") - print() \ No newline at end of file + def resources_from_results(results): + resources = [] + for result in results: + resource = {} + for k, v in result.items(): + if v['type'] =='literal' and ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean'): + value = json.loads(str(v["value"]).lower()) + + elif v['type'] =='literal' and ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer'): + value = int(v["value"]) + # check if the value is an url, if so, get the metadata + elif v['type'] == 'uri' and UNIPROT_ENTITY_PREFIX in v['value']: + value = v["value"] + url = UNIPROT_RESTAPI_URL + f"/{value.split('/')[-1]}.json" + try: + response = requests.get(url) + response.raise_for_status() + except Exception as e: + raise QueryingError(e) + else: + metadata = response.json() + metadata['id'] = value + resource[k] = Resource(**metadata) + else: + resource[k] = v['value'] + resources.append(Resource(**resource)) + return resources \ No newline at end of file From 5200b3b0a53ccfe3d37183ed38f49982dcf25449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 12 Oct 2022 14:08:08 +0200 Subject: [PATCH 10/30] Modified the search method of the sparql store. Resources are retrieved. Missing mappings. --- .../database-sources/prod-nexus-sources.yml | 13 +- .../UniProt/jsonld_context.json | 9 +- .../17 - Database-sources.ipynb | 2035 +---------------- .../specializations/resources/db_sources.py | 1 + kgforge/specializations/stores/sparql.py | 199 +- .../specializations/stores/uniprot/service.py | 5 +- .../specializations/stores/uniprot_store.py | 3 +- 7 files changed, 155 insertions(+), 2110 deletions(-) diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml index 7942e420..e680cc9f 100644 --- a/examples/configurations/database-sources/prod-nexus-sources.yml +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -66,4 +66,15 @@ DatabaseSources: NeuroElectro: store: name: BlueBrainNexus - bucket: bbp/neuroelectro \ No newline at end of file + bucket: bbp/neuroelectro + MouseLight: + store: + name: BlueBrainNexus + bucket: dke/kgforge + protocol: 'https://www.janelia.org/project-team/mouselight/resources' + license: + id: 'https://creativecommons.org/licenses/by-nc/4.0' + label: 'CC BY-NC 4.0' + definition: + origin: directory + source: '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/' \ No newline at end of file diff --git a/examples/database_sources/UniProt/jsonld_context.json b/examples/database_sources/UniProt/jsonld_context.json index 8518b0e8..3b104d2e 100644 --- a/examples/database_sources/UniProt/jsonld_context.json +++ b/examples/database_sources/UniProt/jsonld_context.json @@ -9,16 +9,21 @@ "xsd": "http://www.w3.org/2001/XMLSchema#", "skos": "http://www.w3.org/2004/02/skos/core#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", - "core": "http://purl.uniprot.org/core/", "dc11": "http://purl.org/dc/terms/", "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "foaf": "http://xmlns.com/foaf/0.1/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, "Protein": { "@id": "up:Protein" }, "Gene": { "@id": "up:Gene" - } + }, + "type": { + "@id": "rdf:type" }, "@id": "https://bbp.epfl.ch/jsonldcontext/db/uniprot" + } } \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 21441a95..c1834125 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -58,7 +58,8 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n" + "NeuroElectro\n", + "MouseLight\n" ] } ], @@ -83,17 +84,9 @@ "source": [ "\n", "data = {\n", - " 'store':{\n", + " 'store':{\n", " 'name': 'DemoStore'\n", - " },\n", - " 'protocol': 'https://www.janelia.org/project-team/mouselight/resources', \n", - " 'license': [{'id': 'https://creativecommons.org/licenses/by-nc/4.0', \n", - " 'label': 'CC BY-NC 4.0'}\n", - " ],\n", - " 'definition': {\n", - " 'origin': 'directory',\n", - " 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/'\n", - " },\n", + " },\n", " 'model': { \n", " 'name': 'DemoModel',\n", " 'origin': 'directory',\n", @@ -109,7 +102,7 @@ "outputs": [], "source": [ "from kgforge.specializations.resources import DatabaseSource\n", - "ds = DatabaseSource(forge, name=\"MouseLight\", from_forge=False, **data)" + "ds = DatabaseSource(forge, name=\"DemoDB\", from_forge=False, **data)" ] }, { @@ -140,25 +133,12 @@ " token: null\n", " versioned_id_template: null\n", " }\n", - " definition:\n", - " {\n", - " origin: directory\n", - " source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/\n", - " }\n", - " license:\n", - " [\n", - " {\n", - " id: https://creativecommons.org/licenses/by-nc/4.0\n", - " label: CC BY-NC 4.0\n", - " }\n", - " ]\n", " model:\n", " {\n", " origin: directory\n", " source: ../../../tests/data/demo-model/\n", " }\n", - " name: MouseLight\n", - " protocol: https://www.janelia.org/project-team/mouselight/resources\n", + " name: DemoDB\n", " store:\n", " {\n", " name: DemoStore\n", @@ -183,7 +163,8 @@ "Available Database sources:\n", "UniProt\n", "NeuroElectro\n", - "MouseLight\n" + "MouseLight\n", + "DemoDB\n" ] } ], @@ -202,60 +183,9 @@ "cell_type": "code", "execution_count": 9, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " type: Database\n", - " _store:\n", - " {\n", - " context: null\n", - " bucket: null\n", - " endpoint: null\n", - " file_mapping: null\n", - " metadata_context: null\n", - " model_context: null\n", - " service:\n", - " {\n", - " archives: {}\n", - " records: {}\n", - " tags: {}\n", - " }\n", - " token: null\n", - " versioned_id_template: null\n", - " }\n", - " definition:\n", - " {\n", - " origin: directory\n", - " source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/\n", - " }\n", - " license:\n", - " [\n", - " {\n", - " id: https://creativecommons.org/licenses/by-nc/4.0\n", - " label: CC BY-NC 4.0\n", - " }\n", - " ]\n", - " model:\n", - " {\n", - " origin: directory\n", - " source: ../../../tests/data/demo-model/\n", - " }\n", - " name: MouseLight\n", - " protocol: https://www.janelia.org/project-team/mouselight/resources\n", - " store:\n", - " {\n", - " name: DemoStore\n", - " }\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ - "mouselight= sources[\"MouseLight\"]\n", - "print(mouselight)" + "mouselight = sources[\"MouseLight\"]" ] }, { @@ -276,7 +206,7 @@ "text": [ "MouseLight\n", "https://www.janelia.org/project-team/mouselight/resources\n", - "[{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}]\n" + "{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}\n" ] } ], @@ -729,1942 +659,21 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 24, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "https://bbp.neuroshapes.org/SUBCELLULAR LOCATION does not look like a valid URI, trying to serialize this will break.\n", - "https://bbp.neuroshapes.org/ALTERNATIVE PRODUCTS does not look like a valid URI, trying to serialize this will break.\n", - "https://bbp.neuroshapes.org/TISSUE SPECIFICITY does not look like a valid URI, trying to serialize this will break.\n", - "https://bbp.neuroshapes.org/DISRUPTION PHENOTYPE does not look like a valid URI, trying to serialize this will break.\n", - "https://bbp.neuroshapes.org/Topological domain does not look like a valid URI, trying to serialize this will break.\n", - "https://bbp.neuroshapes.org/Alternative sequence does not look like a valid URI, trying to serialize this will break.\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"@context\": \"https://bbp.neuroshapes.org\",\n", - " \"protein\": {\n", - " \"entryType\": \"UniProtKB reviewed (Swiss-Prot)\",\n", - " \"primaryAccession\": \"A0A131MCZ8\",\n", - " \"secondaryAccessions\": [\n", - " \"A0A131MBV5\",\n", - " \"A0A131MD56\",\n", - " \"Q21469\"\n", - " ],\n", - " \"uniProtkbId\": \"CNNM3_CAEEL\",\n", - " \"entryAudit\": {\n", - " \"firstPublicDate\": \"2016-11-30\",\n", - " \"lastAnnotationUpdateDate\": \"2022-08-03\",\n", - " \"lastSequenceUpdateDate\": \"2016-05-11\",\n", - " \"entryVersion\": 37,\n", - " \"sequenceVersion\": 1\n", - " },\n", - " \"annotationScore\": 5.0,\n", - " \"organism\": {\n", - " \"scientificName\": \"Caenorhabditis elegans\",\n", - " \"taxonId\": 6239,\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"Proteomes\",\n", - " \"@id\": \"UP000001940\"\n", - " }\n", - " ],\n", - " \"lineage\": [\n", - " \"Eukaryota\",\n", - " \"Metazoa\",\n", - " \"Ecdysozoa\",\n", - " \"Nematoda\",\n", - " \"Chromadorea\",\n", - " \"Rhabditida\",\n", - " \"Rhabditina\",\n", - " \"Rhabditomorpha\",\n", - " \"Rhabditoidea\",\n", - " \"Rhabditidae\",\n", - " \"Peloderinae\",\n", - " \"Caenorhabditis\"\n", - " ]\n", - " },\n", - " \"proteinExistence\": \"2: Evidence at transcript level\",\n", - " \"proteinDescription\": {\n", - " \"recommendedName\": {\n", - " \"fullName\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ],\n", - " \"value\": \"Metal transporter cnnm-3\"\n", - " }\n", - " },\n", - " \"alternativeNames\": [\n", - " {\n", - " \"fullName\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2a\"\n", - " }\n", - " ],\n", - " \"value\": \"CNNM family homolog 3\"\n", - " }\n", - " }\n", - " ],\n", - " \"flag\": \"Precursor\"\n", - " },\n", - " \"genes\": [\n", - " {\n", - " \"geneName\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2a\"\n", - " }\n", - " ],\n", - " \"value\": \"cnnm-3\"\n", - " },\n", - " \"orfNames\": [\n", - " {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2a\"\n", - " }\n", - " ],\n", - " \"value\": \"C33D12.2\"\n", - " }\n", - " ]\n", - " }\n", - " ],\n", - " \"comments\": [\n", - " {\n", - " \"texts\": [\n", - " {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000269\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ],\n", - " \"value\": \"Probable metal transporter. Probably acts redundantly with the other metal transport proteins cnnm-1, cnnm-2, cnnm-4 and cnnm-5 to regulate Mg(2+) homeostasis. Promotes postembryonic gonad development by regulating Mg(2+) levels, probably via AMPK signaling\"\n", - " }\n", - " ],\n", - " \"commentType\": \"FUNCTION\"\n", - " },\n", - " {\n", - " \"commentType\": \"SUBCELLULAR LOCATION\",\n", - " \"subcellularLocations\": [\n", - " {\n", - " \"location\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000269\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ],\n", - " \"value\": \"Basolateral cell membrane\",\n", - " \"@id\": \"SL-0026\"\n", - " },\n", - " \"topology\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\"\n", - " }\n", - " ],\n", - " \"value\": \"Multi-pass membrane protein\",\n", - " \"@id\": \"SL-9909\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"commentType\": \"ALTERNATIVE PRODUCTS\",\n", - " \"events\": [\n", - " \"Alternative splicing\"\n", - " ],\n", - " \"isoforms\": [\n", - " {\n", - " \"name\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2a\"\n", - " }\n", - " ],\n", - " \"value\": \"a\"\n", - " },\n", - " \"isoformIds\": [\n", - " \"A0A131MCZ8-1\"\n", - " ],\n", - " \"isoformSequenceStatus\": \"Displayed\"\n", - " },\n", - " {\n", - " \"name\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2b\"\n", - " }\n", - " ],\n", - " \"value\": \"b\"\n", - " },\n", - " \"isoformIds\": [\n", - " \"A0A131MCZ8-2\"\n", - " ],\n", - " \"sequenceIds\": [\n", - " \"VSP_058662\"\n", - " ],\n", - " \"isoformSequenceStatus\": \"Described\"\n", - " },\n", - " {\n", - " \"name\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2c\"\n", - " }\n", - " ],\n", - " \"value\": \"c\"\n", - " },\n", - " \"isoformIds\": [\n", - " \"A0A131MCZ8-3\"\n", - " ],\n", - " \"sequenceIds\": [\n", - " \"VSP_058663\",\n", - " \"VSP_058664\"\n", - " ],\n", - " \"isoformSequenceStatus\": \"Described\"\n", - " },\n", - " {\n", - " \"name\": {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"WormBase\",\n", - " \"@id\": \"C33D12.2d\"\n", - " }\n", - " ],\n", - " \"value\": \"d\"\n", - " },\n", - " \"isoformIds\": [\n", - " \"A0A131MCZ8-4\"\n", - " ],\n", - " \"sequenceIds\": [\n", - " \"VSP_058662\",\n", - " \"VSP_058663\",\n", - " \"VSP_058664\"\n", - " ],\n", - " \"isoformSequenceStatus\": \"Described\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"texts\": [\n", - " {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000269\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ],\n", - " \"value\": \"Highly expressed in the intestine and in neurons, but it is also expressed in a variety of tissues including the pharynx, hypodermis, rectum and in muscles\"\n", - " }\n", - " ],\n", - " \"commentType\": \"TISSUE SPECIFICITY\"\n", - " },\n", - " {\n", - " \"texts\": [\n", - " {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000269\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ],\n", - " \"value\": \"No visible phenotype. Double knockout with cnnm-1 results in increased levels of intestinal Mg(2+) and reduced levels in other tissues. This Mg(2+) deficiency in tissues leads to a reduced lifespan, 100% sterility, and smaller animals that exhibit a developmental delay with defective gonad development and which therefore do not produce oocytes or form vulva. In addition, the gonad development defect in the cnnm-1 and cnnm-3 double knockout is rescued when the AMPK alpha subunit aak-2 is also knocked out. Double knockout with cnnm-2 results in 22% sterility. Quintuple knockout with cnnm-1, cnnm-2, cnnm-4 and cnnm-5 results in a reduced lifespan and 100% sterility\"\n", - " }\n", - " ],\n", - " \"commentType\": \"DISRUPTION PHENOTYPE\"\n", - " },\n", - " {\n", - " \"texts\": [\n", - " {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ],\n", - " \"value\": \"Belongs to the ACDP family\"\n", - " }\n", - " ],\n", - " \"commentType\": \"SIMILARITY\"\n", - " }\n", - " ],\n", - " \"features\": [\n", - " {\n", - " \"@type\": \"Signal\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 1,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 23,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Chain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 24,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 797,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Metal transporter cnnm-3\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ],\n", - " \"featureId\": \"PRO_5007283697\"\n", - " },\n", - " {\n", - " \"@type\": \"Topological domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 24,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 200,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Extracellular\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Transmembrane\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 201,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 221,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Helical\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Topological domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 222,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 255,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Cytoplasmic\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Transmembrane\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 256,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 276,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Helical\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Topological domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 277,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 280,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Extracellular\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Transmembrane\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 281,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 301,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Helical\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Topological domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 302,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 322,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Cytoplasmic\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Transmembrane\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 323,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 343,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Helical\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Topological domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 344,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 797,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"Extracellular\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 193,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 373,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"CNNM transmembrane\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU01193\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 393,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 454,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"CBS 1\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00703\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Domain\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 461,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 527,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"CBS 2\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00703\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 31,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 31,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 42,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 42,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 676,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 676,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 692,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 692,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 724,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 724,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 731,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 731,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Glycosylation\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 761,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 761,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"N-linked (GlcNAc...) asparagine\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000255\",\n", - " \"source\": \"PROSITE-ProRule\",\n", - " \"@id\": \"PRU00498\"\n", - " }\n", - " ],\n", - " \"featureId\": \"\"\n", - " },\n", - " {\n", - " \"@type\": \"Alternative sequence\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 1,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 547,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"in isoform b and isoform d\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ],\n", - " \"featureId\": \"VSP_058662\",\n", - " \"alternativeSequence\": {}\n", - " },\n", - " {\n", - " \"@type\": \"Alternative sequence\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 753,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 766,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"in isoform c and isoform d\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ],\n", - " \"featureId\": \"VSP_058663\",\n", - " \"alternativeSequence\": {\n", - " \"originalSequence\": \"DAVSTPIRNGSVKL\",\n", - " \"alternativeSequences\": [\n", - " \"KYLIGRWKRRGTYV\"\n", - " ]\n", - " }\n", - " },\n", - " {\n", - " \"@type\": \"Alternative sequence\",\n", - " \"location\": {\n", - " \"start\": {\n", - " \"value\": 767,\n", - " \"modifier\": \"EXACT\"\n", - " },\n", - " \"end\": {\n", - " \"value\": 797,\n", - " \"modifier\": \"EXACT\"\n", - " }\n", - " },\n", - " \"description\": \"in isoform c and isoform d\",\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ],\n", - " \"featureId\": \"VSP_058664\",\n", - " \"alternativeSequence\": {}\n", - " }\n", - " ],\n", - " \"keywords\": [\n", - " {\n", - " \"@id\": \"KW-0025\",\n", - " \"category\": \"Coding sequence diversity\",\n", - " \"name\": \"Alternative splicing\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0129\",\n", - " \"category\": \"Domain\",\n", - " \"name\": \"CBS domain\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-1003\",\n", - " \"category\": \"Cellular component\",\n", - " \"name\": \"Cell membrane\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0325\",\n", - " \"category\": \"PTM\",\n", - " \"name\": \"Glycoprotein\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0406\",\n", - " \"category\": \"Biological process\",\n", - " \"name\": \"Ion transport\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0472\",\n", - " \"category\": \"Cellular component\",\n", - " \"name\": \"Membrane\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-1185\",\n", - " \"category\": \"Technical term\",\n", - " \"name\": \"Reference proteome\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0677\",\n", - " \"category\": \"Domain\",\n", - " \"name\": \"Repeat\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0732\",\n", - " \"category\": \"Domain\",\n", - " \"name\": \"Signal\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0812\",\n", - " \"category\": \"Domain\",\n", - " \"name\": \"Transmembrane\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-1133\",\n", - " \"category\": \"Domain\",\n", - " \"name\": \"Transmembrane helix\"\n", - " },\n", - " {\n", - " \"@id\": \"KW-0813\",\n", - " \"category\": \"Biological process\",\n", - " \"name\": \"Transport\"\n", - " }\n", - " ],\n", - " \"references\": [\n", - " {\n", - " \"citation\": {\n", - " \"@id\": \"9851916\",\n", - " \"citationType\": \"journal article\",\n", - " \"authoringGroup\": [\n", - " \"The C. elegans sequencing consortium\"\n", - " ],\n", - " \"citationCrossReferences\": [\n", - " {\n", - " \"database\": \"PubMed\",\n", - " \"@id\": \"9851916\"\n", - " },\n", - " {\n", - " \"database\": \"DOI\",\n", - " \"@id\": \"10.1126/science.282.5396.2012\"\n", - " }\n", - " ],\n", - " \"title\": \"Genome sequence of the nematode C. elegans: a platform for investigating biology.\",\n", - " \"publicationDate\": \"1998\",\n", - " \"journal\": \"Science\",\n", - " \"firstPage\": \"2012\",\n", - " \"lastPage\": \"2018\",\n", - " \"volume\": \"282\"\n", - " },\n", - " \"referencePositions\": [\n", - " \"NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]\"\n", - " ],\n", - " \"referenceComments\": [\n", - " {\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"Proteomes\",\n", - " \"@id\": \"UP000001940\"\n", - " }\n", - " ],\n", - " \"value\": \"Bristol N2\",\n", - " \"@type\": \"STRAIN\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000312\",\n", - " \"source\": \"Proteomes\",\n", - " \"@id\": \"UP000001940\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"citation\": {\n", - " \"@id\": \"27564576\",\n", - " \"citationType\": \"journal article\",\n", - " \"authors\": [\n", - " \"Ishii T.\",\n", - " \"Funato Y.\",\n", - " \"Hashizume O.\",\n", - " \"Yamazaki D.\",\n", - " \"Hirata Y.\",\n", - " \"Nishiwaki K.\",\n", - " \"Kono N.\",\n", - " \"Arai H.\",\n", - " \"Miki H.\"\n", - " ],\n", - " \"citationCrossReferences\": [\n", - " {\n", - " \"database\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " },\n", - " {\n", - " \"database\": \"DOI\",\n", - " \"@id\": \"10.1371/journal.pgen.1006276\"\n", - " }\n", - " ],\n", - " \"title\": \"Mg2+ extrusion from intestinal epithelia by CNNM proteins is essential for gonadogenesis via AMPK-TORC1 signaling in Caenorhabditis elegans.\",\n", - " \"publicationDate\": \"2016\",\n", - " \"journal\": \"PLoS Genet.\",\n", - " \"firstPage\": \"E1006276\",\n", - " \"lastPage\": \"E1006276\",\n", - " \"volume\": \"12\"\n", - " },\n", - " \"referencePositions\": [\n", - " \"FUNCTION\",\n", - " \"SUBCELLULAR LOCATION\",\n", - " \"TISSUE SPECIFICITY\",\n", - " \"DISRUPTION PHENOTYPE\"\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000305\"\n", - " }\n", - " ]\n", - " }\n", - " ],\n", - " \"uniProtKBCrossReferences\": [\n", - " {\n", - " \"database\": \"EMBL\",\n", - " \"@id\": \"BX284606\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CCD66487.1\"\n", - " },\n", - " {\n", - " \"key\": \"Status\",\n", - " \"value\": \"-\"\n", - " },\n", - " {\n", - " \"key\": \"MoleculeType\",\n", - " \"value\": \"Genomic_DNA\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"EMBL\",\n", - " \"@id\": \"BX284606\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CZR14596.1\"\n", - " },\n", - " {\n", - " \"key\": \"Status\",\n", - " \"value\": \"-\"\n", - " },\n", - " {\n", - " \"key\": \"MoleculeType\",\n", - " \"value\": \"Genomic_DNA\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"EMBL\",\n", - " \"@id\": \"BX284606\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CZR14597.1\"\n", - " },\n", - " {\n", - " \"key\": \"Status\",\n", - " \"value\": \"-\"\n", - " },\n", - " {\n", - " \"key\": \"MoleculeType\",\n", - " \"value\": \"Genomic_DNA\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"EMBL\",\n", - " \"@id\": \"BX284606\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CZR14608.1\"\n", - " },\n", - " {\n", - " \"key\": \"Status\",\n", - " \"value\": \"-\"\n", - " },\n", - " {\n", - " \"key\": \"MoleculeType\",\n", - " \"value\": \"Genomic_DNA\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"PIR\",\n", - " \"@id\": \"T16631\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"T16631\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"RefSeq\",\n", - " \"@id\": \"NP_001309670.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"NucleotideSequenceId\",\n", - " \"value\": \"NM_001322613.1\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-1\"\n", - " },\n", - " {\n", - " \"database\": \"RefSeq\",\n", - " \"@id\": \"NP_001309671.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"NucleotideSequenceId\",\n", - " \"value\": \"NM_001322614.1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"RefSeq\",\n", - " \"@id\": \"NP_001309682.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"NucleotideSequenceId\",\n", - " \"value\": \"NM_001322615.1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"RefSeq\",\n", - " \"@id\": \"NP_508520.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"NucleotideSequenceId\",\n", - " \"value\": \"NM_076119.3\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"AlphaFoldDB\",\n", - " \"@id\": \"A0A131MCZ8\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"SMR\",\n", - " \"@id\": \"A0A131MCZ8\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"IntAct\",\n", - " \"@id\": \"A0A131MCZ8\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Interactions\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"STRING\",\n", - " \"@id\": \"6239.C33D12.2\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"PaxDb\",\n", - " \"@id\": \"A0A131MCZ8\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2a.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2a.1\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-1\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2b.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2b.1\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2b.2\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2b.2\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2b.3\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2b.3\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2b.4\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2b.4\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2b.5\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2b.5\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2b.6\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2b.6\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2c.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2c.1\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-3\"\n", - " },\n", - " {\n", - " \"database\": \"EnsemblMetazoa\",\n", - " \"@id\": \"C33D12.2d.1\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"C33D12.2d.1\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-4\"\n", - " },\n", - " {\n", - " \"database\": \"GeneID\",\n", - " \"@id\": \"180591\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"KEGG\",\n", - " \"@id\": \"cel:CELE_C33D12.2\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"UCSC\",\n", - " \"@id\": \"M02F4.3\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"OrganismName\",\n", - " \"value\": \"c. elegans\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"CTD\",\n", - " \"@id\": \"180591\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"WormBase\",\n", - " \"@id\": \"C33D12.2a\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CE51453\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " },\n", - " {\n", - " \"key\": \"GeneName\",\n", - " \"value\": \"cnnm-3\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-1\"\n", - " },\n", - " {\n", - " \"database\": \"WormBase\",\n", - " \"@id\": \"C33D12.2b\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CE04764\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " },\n", - " {\n", - " \"key\": \"GeneName\",\n", - " \"value\": \"cnnm-3\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-2\"\n", - " },\n", - " {\n", - " \"database\": \"WormBase\",\n", - " \"@id\": \"C33D12.2c\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CE51420\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " },\n", - " {\n", - " \"key\": \"GeneName\",\n", - " \"value\": \"cnnm-3\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-3\"\n", - " },\n", - " {\n", - " \"database\": \"WormBase\",\n", - " \"@id\": \"C33D12.2d\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ProteinId\",\n", - " \"value\": \"CE51314\"\n", - " },\n", - " {\n", - " \"key\": \"GeneId\",\n", - " \"value\": \"WBGene00016343\"\n", - " },\n", - " {\n", - " \"key\": \"GeneName\",\n", - " \"value\": \"cnnm-3\"\n", - " }\n", - " ],\n", - " \"isoformId\": \"A0A131MCZ8-4\"\n", - " },\n", - " {\n", - " \"database\": \"eggNOG\",\n", - " \"@id\": \"KOG2118\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ToxonomicScope\",\n", - " \"value\": \"Eukaryota\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GeneTree\",\n", - " \"@id\": \"ENSGT00940000169533\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"OMA\",\n", - " \"@id\": \"MSTCHKI\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Fingerprint\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"OrthoDB\",\n", - " \"@id\": \"1446644at2759\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"PRO\",\n", - " \"@id\": \"PR:A0A131MCZ8\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Description\",\n", - " \"value\": \"-\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"Proteomes\",\n", - " \"@id\": \"UP000001940\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"Component\",\n", - " \"value\": \"Chromosome X\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"Bgee\",\n", - " \"@id\": \"WBGene00016343\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"ExpressionPatterns\",\n", - " \"value\": \"Expressed in larva and 3 other tissues\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0016323\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"C:basolateral plasma membrane\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IDA:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000314\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0016021\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"C:integral component of membrane\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IEA:UniProtKB-KW\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0043231\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"C:intracellular membrane-bounded organelle\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IBA:GO_Central\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000318\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"21873635\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0005886\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"C:plasma membrane\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IBA:GO_Central\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000318\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"21873635\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0022857\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"F:transmembrane transporter activity\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IBA:GO_Central\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000318\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"21873635\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0008340\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:determination of adult lifespan\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IMP:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000315\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0045087\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:innate immune response\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"HEP:WormBase\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0007007\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"16968778\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0010960\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:magnesium ion homeostasis\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IGI:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0015693\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:magnesium ion transport\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IGI:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:1905941\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:positive regulation of gonad development\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IGI:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " },\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " },\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0040018\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:positive regulation of multicellular organism growth\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IGI:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0040026\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:positive regulation of vulval development\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IGI:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"GO\",\n", - " \"@id\": \"GO:0032026\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"GoTerm\",\n", - " \"value\": \"P:response to magnesium ion\"\n", - " },\n", - " {\n", - " \"key\": \"GoEvidenceType\",\n", - " \"value\": \"IGI:UniProtKB\"\n", - " }\n", - " ],\n", - " \"evidences\": [\n", - " {\n", - " \"evidenceCode\": \"ECO:0000316\",\n", - " \"source\": \"PubMed\",\n", - " \"@id\": \"27564576\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"CDD\",\n", - " \"@id\": \"cd04590\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"CBS_pair_CorC_HlyC_assoc\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"Gene3D\",\n", - " \"@id\": \"3.10.580.10\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"-\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"InterPro\",\n", - " \"@id\": \"IPR045095\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"ACDP\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"InterPro\",\n", - " \"@id\": \"IPR000644\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"CBS_dom\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"InterPro\",\n", - " \"@id\": \"IPR046342\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"CBS_dom_sf\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"InterPro\",\n", - " \"@id\": \"IPR002550\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"CNNM\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"InterPro\",\n", - " \"@id\": \"IPR044751\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"Ion_transp-like_CBS\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"PANTHER\",\n", - " \"@id\": \"PTHR12064\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"PTHR12064\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"Pfam\",\n", - " \"@id\": \"PF01595\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"DUF21\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"SUPFAM\",\n", - " \"@id\": \"SSF54631\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"SSF54631\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"PROSITE\",\n", - " \"@id\": \"PS51371\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"CBS\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"2\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"database\": \"PROSITE\",\n", - " \"@id\": \"PS51846\",\n", - " \"properties\": [\n", - " {\n", - " \"key\": \"EntryName\",\n", - " \"value\": \"CNNM\"\n", - " },\n", - " {\n", - " \"key\": \"MatchStatus\",\n", - " \"value\": \"1\"\n", - " }\n", - " ]\n", - " }\n", - " ],\n", - " \"sequence\": {\n", - " \"value\": \"MSKTPWALGLLIFLLTFTSPLSSSPVRSTDNSTSSKGLLNVNSSVILEPSILPSSASKPESLHLSKVRVSGLRLEAHASSTENIVLGHNKKHNVVVVPNKNVRVVLFGQNFQDIGALTFTADGSCKDLAHFFEADFSSMTPIRVVVEMSFPKTTESKDSFKLCVSEKFYANPQFVIVEDPFTMVTTEIPPVDEYMPKWLSWICLLILLCFSGLFSGLNLGLMTLSPYELQLYIASGTEQEKRDAGRILPIRKKGNQLLCTLLIGNVVVNVGVSLLMDQLVGSGFAVLVAATSCIVVFGEIIPQALCVKLGLPIGARTIPITQVLLFLMYPLTWPISKVLDIFLKEELTRSLERNKLVEMLKLSEKSIIGGQSDEFKMVLGALELYDKTVAHAMTRYEDIFMLPHTLTLGAGMVTQILDMGYTRIPIYENDRKNIVALLFVKDLALLDPDDNHNVMKIASIYNHEVRRVLVDMPLRNMLEEFKRGEYHMALVERLVEQEDKDPIYELCGLITLEDIIEEIIQCEIIDETDAVCDNVHRKKRQRKRNHDMSQIVNTAHAKCAINIQMLAVTIQVMSTCHKIFSSNYILPTILEKLIRKNCKKVETTQFSCLKEVGVVQPKPAVLFTKGEFSNKFIMILSGRAVVTIGKEEMRLEAGAWHSFGTEVLDAMAEAIERSLNQSTSRSTVSLNTEITNNSIGFIPDFDTVILYECVFCEITAADLLLAYNSSQIMQNNTKMQVVRSNSRISLIEEIPKDAVSTPIRNGSVKLRTVSEGETVHLLPKNMECHFNKQEKYEEEEE\",\n", - " \"length\": 797,\n", - " \"molWeight\": 89288,\n", - " \"crc64\": \"25AD3EA350EB4E85\",\n", - " \"md5\": \"CDB9455358FB9B5496A1D047EF2A79FD\"\n", - " },\n", - " \"extraAttributes\": {\n", - " \"countByCommentType\": {\n", - " \"FUNCTION\": 1,\n", - " \"SUBCELLULAR LOCATION\": 1,\n", - " \"ALTERNATIVE PRODUCTS\": 4,\n", - " \"TISSUE SPECIFICITY\": 1,\n", - " \"DISRUPTION PHENOTYPE\": 1,\n", - " \"SIMILARITY\": 1\n", - " },\n", - " \"countByFeatureType\": {\n", - " \"Signal\": 1,\n", - " \"Chain\": 1,\n", - " \"Topological domain\": 5,\n", - " \"Transmembrane\": 4,\n", - " \"Domain\": 3,\n", - " \"Glycosylation\": 7,\n", - " \"Alternative sequence\": 3\n", - " },\n", - " \"uniParcId\": \"UPI00077FE610\"\n", - " },\n", - " \"@id\": \"http://purl.uniprot.org/uniprot/A0A131MCZ8\"\n", - " }\n", - "}\n" + " search\n", + " ValueError: context model missing\n", + "\n" ] } ], "source": [ - "print(json.dumps(forge.as_jsonld(uresources[0]), indent=4))" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "import requests" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post('https://sparql.uniprot.org/sparql', data=uquery,\n", - " headers={'Content-Type': 'text/plain', 'Accept': 'application/sparql-results+json'},\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response" + "forge.search({\"type\": 'Protein'}, db_source='UniProt')" ] }, { @@ -2676,7 +685,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -2699,7 +708,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -2716,7 +725,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -2744,7 +753,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -2781,7 +790,7 @@ "Index: []" ] }, - "execution_count": 31, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -2802,7 +811,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 29, "metadata": {}, "outputs": [ { diff --git a/kgforge/specializations/resources/db_sources.py b/kgforge/specializations/resources/db_sources.py index 435ee873..6ddb49df 100644 --- a/kgforge/specializations/resources/db_sources.py +++ b/kgforge/specializations/resources/db_sources.py @@ -82,6 +82,7 @@ def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Database self._model: Model = model(**model_config) # Store. + store_config.update(model_context=self._model.context()) store_name = store_config.pop("name") store = import_class(store_name, "stores") self._store: Store = store(**store_config) diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py index 079e605e..fb8f7fda 100644 --- a/kgforge/specializations/stores/sparql.py +++ b/kgforge/specializations/stores/sparql.py @@ -16,7 +16,7 @@ from pyld import jsonld from copy import deepcopy from pathlib import Path -from typing import Dict, List, Optional, Union, Any +from typing import Dict, List, Tuple, Optional, Union, Any from uuid import uuid4 import requests from rdflib import Graph @@ -30,9 +30,15 @@ from kgforge.core.commons.context import Context from kgforge.core.wrappings.dict import DictWrapper from kgforge.specializations.stores.uniprot.service import Service +from kgforge.core.wrappings.paths import Filter, create_filters_from_dict from kgforge.core.commons.exceptions import DownloadingError, QueryingError from kgforge.core.commons.execution import not_supported +from kgforge.core.commons.parser import _parse_type from kgforge.core.conversions.rdf import as_jsonld, from_jsonld +from kgforge.specializations.stores.bluebrain_nexus import ( + CategoryDataType, _create_select_query, + _box_value_as_full_iri, sparql_operator_map, + type_map, format_type) class SPARQLStore(Store): @@ -113,7 +119,7 @@ def deprecate(self, data: Union[Resource, List[Resource]]) -> None: # Querying. def search( - self, resolvers: Optional[List["Resolver"]], *filters, **params + self, resolvers: Optional[List["Resolver"]]=None, *filters, **params ) -> List[Resource]: # Positional arguments in 'filters' are instances of type Filter from wrappings/paths.py # A dictionary can be provided for filters: @@ -129,102 +135,60 @@ def search( # POLICY Should use sparql() and contain 'search_endpoint'. # POLICY Resource _store_metadata should be set using wrappers.dict.wrap_dict(). # POLICY Resource _synchronized should be set to True. - pass + # pass # if self.model_context is None: - # raise ValueError("context model missing") - - # debug = params.get("debug", False) - # limit = params.get("limit", 100) - # offset = params.get("offset", None) - # deprecated = params.get("deprecated", False) - # distinct = params.get("distinct", False) - # includes = params.get("includes", None) - # excludes = params.get("excludes", None) - # retrieve_source = params.get("retrieve_source") - # search_endpoint = params.get( - # "search_endpoint", self.service.sparql_endpoint["type"] - # ) - # if search_endpoint not in [ - # self.service.sparql_endpoint["type"], - # ]: - # raise ValueError( - # f"The provided search_endpoint value '{search_endpoint}' is not supported, only 'sparql'" - # ) - # if "filters" in params: - # raise ValueError("A 'filters' key was provided as params. Filters should be provided as iterable to be unpacked.") - - - # if filters and isinstance(filters[0], dict): - # filters = create_filters_from_dict(filters[0]) - # filters = list(filters) if not isinstance(filters, list) else filters - - # if includes or excludes: - # raise ValueError( - # "Field inclusion and exclusion are not supported when using SPARQL" - # ) - - # query_statements, query_filters = build_sparql_query_statements( - # self.model_context, filters - # ) - # store_metadata_statements = [] - # if retrieve_source: - # _vars = ["?id"] - # for i, k in enumerate(self.service.store_metadata_keys): - # _vars.append(f"?{k}") - # store_metadata_statements.insert(i+2, f"<{self.metadata_context.terms[k].id}> ?{k}") - # deprecated_filter = f"Filter (?_deprecated = {format_type[CategoryDataType.BOOLEAN](deprecated)})" - # query_filters.append(deprecated_filter) - # else: - # _vars = ["?id", "?_project", "?_rev"] - # store_metadata_statements.append(f"<{self.service.revision_property}> ?_rev") - # store_metadata_statements.append(f"<{self.service.project_property}> ?_project") - # query_statements.append( - # f"<{self.service.deprecated_property}> {format_type[CategoryDataType.BOOLEAN](deprecated)}", - # ) - # query_statements.extend(store_metadata_statements) - # statements = ";\n ".join(query_statements) - # _filters = "\n".join(".\n ".join(query_filters)) - # query = _create_select_query( - # _vars, f"?id {statements} . \n {_filters}", distinct - # ) - # # support @id and @type - # resources = self.sparql(query, debug=debug, limit=limit, offset=offset) - # params_retrieve = deepcopy(self.service.params.get("retrieve", {})) - # params_retrieve['retrieve_source'] = retrieve_source - # results = self.service.batch_request( - # resources, BatchAction.FETCH, None, QueryingError, params=params_retrieve - # ) - # resources = list() - # for result in results: - # resource = result.resource - # if retrieve_source: - # store_metadata_response = as_json(result.resource, expanded=False, store_metadata=False, - # model_context=None, - # metadata_context=None, - # context_resolver=None) # store_metadata is obtained from SPARQL (resource) and not from server (response) because of retrieve_source==True - # else: - # store_metadata_response = result.response # dict - # try: - # resource = self.service.to_resource(result.response) - # except Exception as e: - # self.service.synchronize_resource( - # resource, store_metadata_response, self.search.__name__, False, False - # ) - # raise ValueError(e) - # finally: - # self.service.synchronize_resource( - # resource, store_metadata_response, self.search.__name__, True, False - # ) - # resources.append(resource) - # return resources + # raise ValueError("context model missing") + + debug = params.get("debug", False) + limit = params.get("limit", 100) + offset = params.get("offset", None) + deprecated = params.get("deprecated", False) + distinct = params.get("distinct", False) + includes = params.get("includes", None) + excludes = params.get("excludes", None) + search_endpoint = params.get( + "search_endpoint", self.service.sparql_endpoint["type"] + ) + if search_endpoint not in [ + self.service.sparql_endpoint["type"], + ]: + raise ValueError( + f"The provided search_endpoint value '{search_endpoint}' is not supported, only 'sparql'" + ) + if "filters" in params: + raise ValueError("A 'filters' key was provided as params. Filters should be provided as iterable to be unpacked.") + + + if filters and isinstance(filters[0], dict): + filters = create_filters_from_dict(filters[0]) + filters = list(filters) if not isinstance(filters, list) else filters + + if includes or excludes: + raise ValueError( + "Field inclusion and exclusion are not supported when using SPARQL" + ) + + query_statements, query_filters = build_sparql_query_statements( + self.context, filters + ) + statements = ";\n ".join(query_statements) + _filters = ".\n".join(query_filters) + '\n' + _vars = ["?id"] + query = _create_select_query( + _vars, f"?id {statements} . \n {_filters}", distinct, False + ) + # support @id and @type + resources = self.sparql(query, debug=debug, limit=limit, offset=offset) + return resources def sparql( self, query: str, debug: bool, limit: int = None, offset: int = None, **params ) -> List[Resource]: rewrite = params.get("rewrite", True) + print('query in sparql', query) qr = ( - rewrite_sparql(query, self.model_context, self.service.metadata_context) - if self.model_context is not None and rewrite + rewrite_sparql(query, self.context, self.service.metadata_context) + if self.context is not None and rewrite else query ) qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") @@ -345,4 +309,55 @@ def _debug_query(query): print("Submitted query:", query) else: print(*["Submitted query:", *query.splitlines()], sep="\n ") - print() \ No newline at end of file + print() + + +def build_sparql_query_statements(context: Context, *conditions) -> Tuple[List, List]: + statements = list() + filters = list() + for index, f in enumerate(*conditions): + last_path = f.path[-1] + try: + last_term = context.terms[last_path] + except KeyError: + last_term = None + if last_path in ["id", "@id"]: + property_path = "/".join(f.path[:-1]) + elif last_path == "@type": + minus_last_path = f.path[:-1] + minus_last_path.append("type") + property_path = "/".join(minus_last_path) + else: + property_path = "/".join(f.path) + try: + if ( + last_path in ["type", "@type"] + or last_path in ["id", "@id"] + or (last_term is not None and last_term.type == "@id") + ): + if f.operator == "__eq__": + statements.append(f"{property_path} {_box_value_as_full_iri(f.value)}") + elif f.operator == "__ne__": + statements.append(f"{property_path} ?v{index}") + filters.append(f"FILTER(?v{index} != {f.value})") + else: + raise NotImplementedError( + f"supported operators are '==' and '!=' when filtering by type or id." + ) + else: + parsed_type, parsed_value = _parse_type(f.value, parse_str=False) + value_type = type_map[parsed_type] + value = format_type[value_type](parsed_value if parsed_value else f.value) + if value_type is CategoryDataType.LITERAL: + if f.operator not in ["__eq__", "__ne__"]: + raise NotImplementedError(f"supported operators are '==' and '!=' when filtering with a str.") + statements.append(f"{property_path} ?v{index}") + filters.append(f"FILTER(?v{index} = {_box_value_as_full_iri(value)})") + else: + statements.append(f"{property_path} ?v{index}") + filters.append( + f"FILTER(?v{index} {sparql_operator_map[f.operator]} {_box_value_as_full_iri(value)})" + ) + except NotImplementedError as nie: + raise ValueError(f"Operator '{sparql_operator_map[f.operator]}' is not supported with the value '{f.value}': {str(nie)}") + return statements, filters diff --git a/kgforge/specializations/stores/uniprot/service.py b/kgforge/specializations/stores/uniprot/service.py index d42bfc27..4000b772 100644 --- a/kgforge/specializations/stores/uniprot/service.py +++ b/kgforge/specializations/stores/uniprot/service.py @@ -96,7 +96,10 @@ def __init__( uniprot_context = json.load(jfile) self.context = Context(uniprot_context) - self.metadata_context = Context(uniprot_context) + self.metadata_context = Context( + recursive_resolve(model_context, self.resolve_context, already_loaded=[]), + uniprot_context + ) self.sparql_endpoint = dict() self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] diff --git a/kgforge/specializations/stores/uniprot_store.py b/kgforge/specializations/stores/uniprot_store.py index 36ccfe31..6374d7b6 100644 --- a/kgforge/specializations/stores/uniprot_store.py +++ b/kgforge/specializations/stores/uniprot_store.py @@ -64,6 +64,7 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, @staticmethod def resources_from_results(results): + print(f'amount of results = {len(results)}') resources = [] for result in results: resource = {} @@ -89,4 +90,4 @@ def resources_from_results(results): else: resource[k] = v['value'] resources.append(Resource(**resource)) - return resources \ No newline at end of file + return resources \ No newline at end of file From a01af1ebf3ca85238b6b6cccab78ce778b58e6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 12 Oct 2022 14:12:36 +0200 Subject: [PATCH 11/30] Updated notebook with search example. --- .../17 - Database-sources.ipynb | 262 +++++++++++++----- 1 file changed, 191 insertions(+), 71 deletions(-) diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index c1834125..d8773571 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -109,46 +109,9 @@ "cell_type": "code", "execution_count": 7, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " type: Database\n", - " _store:\n", - " {\n", - " context: null\n", - " bucket: null\n", - " endpoint: null\n", - " file_mapping: null\n", - " metadata_context: null\n", - " model_context: null\n", - " service:\n", - " {\n", - " archives: {}\n", - " records: {}\n", - " tags: {}\n", - " }\n", - " token: null\n", - " versioned_id_template: null\n", - " }\n", - " model:\n", - " {\n", - " origin: directory\n", - " source: ../../../tests/data/demo-model/\n", - " }\n", - " name: DemoDB\n", - " store:\n", - " {\n", - " name: DemoStore\n", - " }\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ - "print(ds)" + "# print(ds)" ] }, { @@ -524,7 +487,29 @@ "cell_type": "code", "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "query in rewrite_sparql SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id type ScholarlyArticle;\n", + " ?_constrainedBy;\n", + " ?_createdAt;\n", + " ?_createdBy;\n", + " ?_deprecated;\n", + " ?_incoming;\n", + " ?_outgoing;\n", + " ?_project;\n", + " ?_rev;\n", + " ?_schemaProject;\n", + " ?_self;\n", + " ?_updatedAt;\n", + " ?_updatedBy . \n", + " Filter (?_deprecated = 'false'^^xsd:boolean)\n", + "Filter (?_project = )}}\n" + ] + } + ], "source": [ "# Type, source or target brain region, \n", "filters = {\"type\":\"ScholarlyArticle\"}\n", @@ -564,49 +549,68 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/35463\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Rationally, an increased intrinsic excitability of dorsal horn neurons could be a factor contributing to alter the gain of the nociceptive system during central sensitization, however direct evidence is scarce. Here we have examined this hypothesis using current and voltage-clamp recordings from dorsal horn neurons in the spinal cord in vitro preparation obtained from mice pups of either sex. Cords were extracted from carrageenan-pretreated and control animals to allow for comparison. Dorsal horn neurons from treated animals showed significantly larger and faster synaptic responses. Synaptic changes started developing shortly after inflammation (1 h) and developed further after a longer-term inflammation (20 h). However, these neurons showed biphasic changes in membrane excitability with an increase shortly after inflammation and a decrease in the longer term. Concomitant changes were observed in transient (I(A)) and sustained potassium currents (I(DR)). Prolonged superfusion of naive spinal cords with NMDA led to a decreased neuronal excitability and to increased potassium currents. Results suggest that excitability plays a role more complex than expected during the process of central sensitization of dorsal horn neurons and that modulation of potassium currents may contribute to shape the changing states of excitability. The decreased excitability observed after long-term inflammation is interpreted as a homeostatic correction to an abnormal state of synaptic activity.\n", + " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Rivera-Arconada\n", - " givenName: Ivan\n", + " familyName: Henderson\n", + " givenName: Z\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Morris\n", + " givenName: N P\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Grimwood\n", + " givenName: P\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Fiddler\n", + " givenName: G\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Yang\n", + " givenName: H W\n", " }\n", " {\n", " type: Person\n", - " familyName: Lopez-Garcia\n", - " givenName: Jose A\n", + " familyName: Appenteng\n", + " givenName: K\n", " }\n", " ]\n", - " datePublished: 2010-4-14\n", + " datePublished: 2001-2-12\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 20392959\n", + " value: 11169477\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1523/JNEUROSCI.4359-09.2010\n", + " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0270-6474\n", - " name: The Journal of neuroscience : the official journal of the Society for Neuroscience\n", - " publisher: Society for Neuroscience\n", + " issn: 0021-9967\n", + " name: The Journal of comparative neurology\n", " }\n", - " name: article_35463\n", - " sameAs: http://www.jneurosci.org/content/30/15/5376.long\n", - " title: Changes in membrane excitability and potassium currents in sensitized dorsal horn neurons of mice pups.\n", - " url: http://www.jneurosci.org/content/30/15/5376.long\n", + " name: article_91941\n", + " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", + " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", "}\n" ] } @@ -625,8 +629,8 @@ "PREFIX up: \n", "SELECT ?protein\n", "WHERE {\n", - " ?protein a up:Protein .\n", - " ?protein up:reviewed true .\n", + " ?protein a up:Protein ;\n", + " up:reviewed true.\n", "}\n", "\"\"\"" ] @@ -640,16 +644,33 @@ "name": "stdout", "output_type": "stream", "text": [ + "query in sparql \n", + "PREFIX up: \n", + "SELECT ?protein\n", + "WHERE {\n", + " ?protein a up:Protein ;\n", + " up:reviewed true.\n", + "}\n", + "\n", + "query in rewrite_sparql \n", + "PREFIX up: \n", + "SELECT ?protein\n", + "WHERE {\n", + " ?protein a up:Protein ;\n", + " up:reviewed true.\n", + "}\n", + "\n", "Submitted query:\n", " \n", " PREFIX up: \n", " SELECT ?protein\n", " WHERE {\n", - " ?protein a up:Protein .\n", - " ?protein up:reviewed true .\n", + " ?protein a up:Protein ;\n", + " up:reviewed true.\n", " }\n", " LIMIT 10\n", - "\n" + "\n", + "amount of results = 10\n" ] } ], @@ -661,19 +682,103 @@ "cell_type": "code", "execution_count": 24, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "10" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(uresources)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "# uresources" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "from kgforge.core.wrappings.paths import Filter, FilterOperator" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " search\n", - " ValueError: context model missing\n", - "\n" + "query in sparql SELECT ?id WHERE {?id type Protein;\n", + " up:reviewed ?v1 . \n", + " FILTER(?v1 = 'true'^^xsd:boolean)\n", + "}\n", + "query in rewrite_sparql SELECT ?id WHERE {?id type Protein;\n", + " up:reviewed ?v1 . \n", + " FILTER(?v1 = 'true'^^xsd:boolean)\n", + "}\n", + "amount of results = 10\n" ] } ], "source": [ - "forge.search({\"type\": 'Protein'}, db_source='UniProt')" + "proteins = forge.search({'type': 'Protein', 'up:reviewed': True}, db_source='UniProt', limit=10)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "uniprot = sources['UniProt']" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'up': 'http://purl.uniprot.org/core/',\n", + " 'owl': 'http://www.w3.org/2002/07/owl#',\n", + " 'owl2xml': 'http://www.w3.org/2006/12/owl2-xml#',\n", + " 'swrlb': 'http://www.w3.org/2003/11/swrlb#',\n", + " 'protege': 'http://protege.stanford.edu/plugins/owl/protege#',\n", + " 'swrl': 'http://www.w3.org/2003/11/swrl#',\n", + " 'xsd': 'http://www.w3.org/2001/XMLSchema#',\n", + " 'skos': 'http://www.w3.org/2004/02/skos/core#',\n", + " 'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',\n", + " 'dc11': 'http://purl.org/dc/terms/',\n", + " 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',\n", + " 'foaf': 'http://xmlns.com/foaf/0.1/'}" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "uniprot._store.context.prefixes" ] }, { @@ -685,7 +790,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -708,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -725,13 +830,28 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "query in rewrite_sparql SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id type NeuronMorphology;\n", + " ?_constrainedBy;\n", + " ?_createdAt;\n", + " ?_createdBy;\n", + " ?_deprecated;\n", + " ?_incoming;\n", + " ?_outgoing;\n", + " ?_project;\n", + " ?_rev;\n", + " ?_schemaProject;\n", + " ?_self;\n", + " ?_updatedAt;\n", + " ?_updatedBy . \n", + " Filter (?_deprecated = 'false'^^xsd:boolean)\n", + "Filter (?_project = )}}\n", "0 dataset(s) of type NeuronMorphology found\n" ] } @@ -753,7 +873,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -790,7 +910,7 @@ "Index: []" ] }, - "execution_count": 28, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -811,7 +931,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 34, "metadata": {}, "outputs": [ { From 8d7bea774106e634bd1a45099f4e2b012a31af87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 12 Oct 2022 17:47:54 +0200 Subject: [PATCH 12/30] Added a draft of UniProt mappings. --- .../mappings/DictionaryMapping/Gene.hjson | 90 +++ .../NeuronElectrophysiologicalFeature.hjson | 38 -- .../mappings/DictionaryMapping/Protein.hjson | 20 + .../17 - Database-sources.ipynb | 621 +++++++++++++++--- 4 files changed, 641 insertions(+), 128 deletions(-) create mode 100644 examples/database_sources/UniProt/mappings/DictionaryMapping/Gene.hjson delete mode 100644 examples/database_sources/UniProt/mappings/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson create mode 100644 examples/database_sources/UniProt/mappings/DictionaryMapping/Protein.hjson diff --git a/examples/database_sources/UniProt/mappings/DictionaryMapping/Gene.hjson b/examples/database_sources/UniProt/mappings/DictionaryMapping/Gene.hjson new file mode 100644 index 00000000..11caa996 --- /dev/null +++ b/examples/database_sources/UniProt/mappings/DictionaryMapping/Gene.hjson @@ -0,0 +1,90 @@ +{ + "id": f"https://bbp.epfl.ch/neurosciencegraph/data/genes/{x.uid}" + "type": [ + "Entity" + "Gene" + ] + "description": x.description + "enables": [ + { + "id": "GO:0017046" + "type": "Class" + "label": "peptide hormone binding" + } + { + "id": "GO:0008528" + "type": "Class" + "label": "G protein-coupled peptide receptor activity" + } + ] + "identifier": [ + { + "propertyID": "ArrayExpress" + "value": "ENSMUSG00000025127" + } + { + "propertyID": "EntrezGene" + "value": "14527" + } + { + "propertyID": "MGI" + "value": "MGI:99572" + } + { + "propertyID": "Uniprot_gn" + "value": "Q61606" + } + { + "propertyID": "WikiGene" + "value": "14527" + } + { + "propertyID": "Ensembl" + "value": "ENSMUSG00000025127" + } + ] + "involvedIn": [ + { + "id": "GO:0007189" + "type": "Class" + "label": "adenylate cyclase-activating G protein-coupled receptor signaling pathway" + } + { + "id": "GO:0007188" + "type": "Class" + "label": "adenylate cyclase-modulating G protein-coupled receptor signaling pathway" + } + { + "id": "GO:0071377" + "type": "Class" + "label": "cellular response to glucagon stimulus" + } + ] + "label": "Gcgr" + "locatedIn": [ + { + "id": "GO:0005887" + "type": "Class" + "label": "integral component of plasma membrane" + } + { + "id": "GO:0016020" + "type": "Class" + "label": "membrane" + } + { + "id": "GO:0016021" + "type": "Class" + "label": "integral component of membrane" + } + ] + "name": "Gcgr" + "prefLabel": "Gcgr" + "subject": { + "@type": "Subject" + "species": { + "id": "NCBITaxon:10090" + "label": "Mus musculus" + } + } +} \ No newline at end of file diff --git a/examples/database_sources/UniProt/mappings/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson b/examples/database_sources/UniProt/mappings/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson deleted file mode 100644 index 67b19146..00000000 --- a/examples/database_sources/UniProt/mappings/DictionaryMapping/NeuronElectrophysiologicalFeature.hjson +++ /dev/null @@ -1,38 +0,0 @@ -{ - type: 'Class', - id: forge.format("identifier_neuroelectro", "e", x.id) - label: x.name if 'name' in x else '' - prefLabel: x.name if 'name' in x else '' - definition: x.definition, if 'definition' in x and x.definition else '' - notation: x.short_name if 'short_name' in x and x.short_name else '' - subClassOf: 'NeuronElectrophysiologicalFeature' - isDefinedBy: 'https://neuroelectro.org', - maxValue: x.max_range if 'max_range' in x and x.max_range else '', - minValue: x.min_range, if 'min_range' in x and x.min_range else '', - "unitCode": { - "id":, - "type": "http://qudt.org/schema/qudt/Unit", - "sameAs": - }, - "plot_transform": x.plot_transform if 'plot_transform' in x and x.plot_transform else '', - "norm_criteria": x.norm_criteria, if 'norm_criteria' in x and x.norm_criteria else '', - sameAs,: forge.format("identifier_neurolex",x.nlex_id) if 'nlex_id' in x and x.nlex_id else '' -} - - -{ - "definition": "Input resistance measured at steady-state voltage response to current injection", - "id": 2, - "max_range": 20000.0, - "min_range": 5.0, - "name": "input resistance", - "nlex_id": null, - "norm_criteria": "Values corrected for differences in units, but are otherwise unchanged. Refer to individual articles for specific definitions and calculation methodologies.", - "plot_transform": "log10", - "short_name": "rin", - "units": { - "id": 2, - "name": "Ω", - "prefix": "M" - } -} \ No newline at end of file diff --git a/examples/database_sources/UniProt/mappings/DictionaryMapping/Protein.hjson b/examples/database_sources/UniProt/mappings/DictionaryMapping/Protein.hjson new file mode 100644 index 00000000..1b637039 --- /dev/null +++ b/examples/database_sources/UniProt/mappings/DictionaryMapping/Protein.hjson @@ -0,0 +1,20 @@ +{ + "id": "https://bbp.epfl.ch/neurosciencegraph/data/proteins/4f5b79f8-1abe-48e0-95d1-e017d621b058" + "type": [ + "Entity" + "Protein" + ] + "encodedBy": { #genes + "id": "https://bbp.epfl.ch/neurosciencegraph/data/genes/98d04429-b003-4fc4-9e66-817546fd2f5f" + "type": "Gene" + } + "identifier": x.identifier #primaryAccession + "name": x.name #proteinDescription/recommendedName/fullName + "label": x.label #proteinDescription/recommendedName/fullName/value + "altLabel": x.altlabel #proteinDescription/alternativeNames/0/fullName/value + "structureAvailable": x.hasstructure ??? + "subject": x.subject #resolve organism/scientificName or taxonID + #optional + "sequence": x.sequence #sequence/value + "references": ... +} \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index d8773571..3f833314 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -219,7 +219,9 @@ "output_type": "stream", "text": [ "Managed mappings for the data source per entity type and mapping type:\n", - " - NeuronElectrophysiologicalFeature:\n", + " - Gene:\n", + " * DictionaryMapping\n", + " - Protein:\n", " * DictionaryMapping\n" ] } @@ -487,29 +489,7 @@ "cell_type": "code", "execution_count": 19, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "query in rewrite_sparql SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id type ScholarlyArticle;\n", - " ?_constrainedBy;\n", - " ?_createdAt;\n", - " ?_createdBy;\n", - " ?_deprecated;\n", - " ?_incoming;\n", - " ?_outgoing;\n", - " ?_project;\n", - " ?_rev;\n", - " ?_schemaProject;\n", - " ?_self;\n", - " ?_updatedAt;\n", - " ?_updatedBy . \n", - " Filter (?_deprecated = 'false'^^xsd:boolean)\n", - "Filter (?_project = )}}\n" - ] - } - ], + "outputs": [], "source": [ "# Type, source or target brain region, \n", "filters = {\"type\":\"ScholarlyArticle\"}\n", @@ -549,68 +529,63 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/34164\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", + " abstract: Striatal spiny projection (SP) neurons control movement initiation by integrating cortical inputs and inhibiting basal ganglia outputs. Central to this control lies a \"microcircuit\" that consists of a feedback pathway formed by axon collaterals between GABAergic SP neurons and a feedforward pathway from fast spiking (FS) GABAergic interneurons to SP neurons. Here, somatically evoked postsynaptic potentials (PSPs) and currents (PSCs) were compared for both pathways with dual whole cell patch recording in voltage- and current-clamp mode using cortex-striatum-substantia nigra organotypic cultures. On average, feedforward inputs were 1 ms earlier, more reliable, and about twice as large in amplitude compared with most feedback inputs. On the other hand, both pathways exhibited widely varying, partially overlapping amplitude distributions. This variability was already established for single FS neurons targeting many SP neurons. In response to precisely timed action potential bursts, feedforward and feedback inputs consistently showed short-term depression < or =50-70% in voltage-clamp, although feedback inputs also displayed strong augmentation in current-clamp in line with previous reports. The augmentation of feedback inputs was absent in gramicidin D perforated-patch recording, which also showed the natural reversal potential for both inputs to be near firing threshold. Preceding depolarizing feedback inputs during the down state did not consistently change subsequent postsynaptic action potentials. We conclude that feedback and feedforward inputs have their dominant effect during the up-state. The reversal potential close to the up-state potential, which supports shunting operation with millisecond precision and the strong synaptic depression, should enable both pathways to carry time-critical information.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Henderson\n", - " givenName: Z\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Morris\n", - " givenName: N P\n", + " familyName: Gustafson\n", + " givenName: Nicholas\n", " }\n", " {\n", " type: Person\n", - " familyName: Grimwood\n", - " givenName: P\n", + " familyName: Gireesh-Dharmaraj\n", + " givenName: Elakkat\n", " }\n", " {\n", " type: Person\n", - " familyName: Fiddler\n", - " givenName: G\n", + " familyName: Czubayko\n", + " givenName: Uwe\n", " }\n", " {\n", " type: Person\n", - " familyName: Yang\n", - " givenName: H W\n", + " familyName: Blackwell\n", + " givenName: Kim T\n", " }\n", " {\n", " type: Person\n", - " familyName: Appenteng\n", - " givenName: K\n", + " familyName: Plenz\n", + " givenName: Dietmar\n", " }\n", " ]\n", - " datePublished: 2001-2-12\n", + " datePublished: 2006-2\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 11169477\n", + " value: 16236782\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", + " value: 10.1152/jn.00802.2005\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0021-9967\n", - " name: The Journal of comparative neurology\n", + " issn: 0022-3077\n", + " name: Journal of neurophysiology\n", " }\n", - " name: article_91941\n", - " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", - " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", - " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " name: article_34164\n", + " sameAs: http://jn.physiology.org/content/95/2/737.long\n", + " title: A comparative voltage and current-clamp analysis of feedback and feedforward synaptic transmission in the striatal microcircuit in vitro.\n", + " url: http://jn.physiology.org/content/95/2/737.long\n", "}\n" ] } @@ -652,14 +627,6 @@ " up:reviewed true.\n", "}\n", "\n", - "query in rewrite_sparql \n", - "PREFIX up: \n", - "SELECT ?protein\n", - "WHERE {\n", - " ?protein a up:Protein ;\n", - " up:reviewed true.\n", - "}\n", - "\n", "Submitted query:\n", " \n", " PREFIX up: \n", @@ -729,10 +696,6 @@ " up:reviewed ?v1 . \n", " FILTER(?v1 = 'true'^^xsd:boolean)\n", "}\n", - "query in rewrite_sparql SELECT ?id WHERE {?id type Protein;\n", - " up:reviewed ?v1 . \n", - " FILTER(?v1 = 'true'^^xsd:boolean)\n", - "}\n", "amount of results = 10\n" ] } @@ -781,6 +744,46 @@ "uniprot._store.context.prefixes" ] }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "query in sparql SELECT ?id WHERE {?id type Gene . \n", + " \n", + "}\n", + "amount of results = 10\n" + ] + } + ], + "source": [ + "genes = forge.search({'type': 'Gene'}, db_source='UniProt', limit=10)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id=Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/H5SQ95#gene-MD58301D33AF640374C84A4DA4CAF383BE6', _inner_sync=False, annotationScore=2.0, comments=[{'texts': [{'evidences': [{'evidenceCode': 'ECO:0000305', 'source': 'PubMed', 'id': '28087277'}], 'value': 'May function as a protein modifier covalently attached to lysine residues of substrate proteins. This may serve to target the modified proteins for degradation by proteasomes'}], 'commentType': 'FUNCTION'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'HAMAP-Rule', 'id': 'MF_02133'}], 'value': 'Belongs to the ubiquitin-like protein UBact family'}], 'commentType': 'SIMILARITY'}], entryAudit={'firstPublicDate': '2017-10-25', 'lastAnnotationUpdateDate': '2022-05-25', 'lastSequenceUpdateDate': '2012-04-18', 'entryVersion': 15, 'sequenceVersion': 1}, entryType='UniProtKB reviewed (Swiss-Prot)', extraAttributes={'countByCommentType': {'FUNCTION': 1, 'SIMILARITY': 1}, 'countByFeatureType': {'Chain': 1, 'Region': 1, 'Compositional bias': 1, 'Cross-link': 1}, 'uniParcId': 'UPI00024DBA4F'}, features=[{'type': 'Chain', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Prokaryotic ubiquitin-like protein UBact', 'featureId': 'PRO_0000441772'}, {'type': 'Region', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Disordered', 'evidences': [{'evidenceCode': 'ECO:0000256', 'source': 'SAM', 'id': 'MobiDB-lite'}]}, {'type': 'Compositional bias', 'location': {'start': {'value': 26, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Basic and acidic residues', 'evidences': [{'evidenceCode': 'ECO:0000256', 'source': 'SAM', 'id': 'MobiDB-lite'}]}, {'type': 'Cross-link', 'location': {'start': {'value': 56, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Isoglutamyl lysine isopeptide (Glu-Lys) (interchain with K-? in acceptor proteins)', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}], genes=[{'geneName': {'evidences': [{'evidenceCode': 'ECO:0000303', 'source': 'PubMed', 'id': '28087277'}], 'value': 'ubact'}, 'orfNames': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'EMBL', 'id': 'BAL58331.1'}], 'value': 'HGMM_OP1C026'}]}], keywords=[{'id': 'KW-1017', 'category': 'PTM', 'name': 'Isopeptide bond'}, {'id': 'KW-0833', 'category': 'Biological process', 'name': 'Ubl conjugation pathway'}], organism={'scientificName': 'Acetothermus autotrophicum', 'taxonId': 1446466, 'lineage': ['Bacteria', 'Candidatus Bipolaricaulota', 'Candidatus Acetothermum']}, primaryAccession='H5SQ95', proteinDescription={'recommendedName': {'fullName': {'evidences': [{'evidenceCode': 'ECO:0000303', 'source': 'PubMed', 'id': '28087277'}], 'value': 'Prokaryotic ubiquitin-like protein UBact'}}}, proteinExistence='3: Inferred from homology', references=[{'citation': {'id': '22303444', 'citationType': 'journal article', 'authors': ['Takami H.', 'Noguchi H.', 'Takaki Y.', 'Uchiyama I.', 'Toyoda A.', 'Nishi S.', 'Chee G.J.', 'Arai W.', 'Nunoura T.', 'Itoh T.', 'Hattori M.', 'Takai K.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '22303444'}, {'database': 'DOI', 'id': '10.1371/journal.pone.0030559'}], 'title': 'A deeply branching thermophilic bacterium with an ancient acetyl-CoA pathway dominates a subsurface ecosystem.', 'publicationDate': '2012', 'journal': 'PLoS ONE', 'firstPage': 'E30559', 'lastPage': 'E30559', 'volume': '7'}, 'referencePositions': ['NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]']}, {'citation': {'id': '28087277', 'citationType': 'journal article', 'authors': ['Lehmann G.', 'Udasin R.G.', 'Livneh I.', 'Ciechanover A.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '28087277'}, {'database': 'DOI', 'id': '10.1016/j.bbrc.2017.01.037'}], 'title': 'Identification of UBact, a ubiquitin-like protein, along with other homologous components of a conjugation system and the proteasome in different gram-negative bacteria.', 'publicationDate': '2017', 'journal': 'Biochem. Biophys. Res. Commun.', 'firstPage': '946', 'lastPage': '950', 'volume': '483'}, 'referencePositions': ['PREDICTED FUNCTION']}], sequence={'value': 'MPERIVKPMPQDPVTKPGDEGPRTPNVPKPDTERLLERMRRVDPRQAQRYRQRSGE', 'length': 56, 'molWeight': 6594, 'crc64': '323CA0429FBA02D0', 'md5': 'EF137CE5E7D3D2D873E21B0ECFB25FF6'}, uniProtKBCrossReferences=[{'database': 'EMBL', 'id': 'AP011800', 'properties': [{'key': 'ProteinId', 'value': 'BAL58331.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'AlphaFoldDB', 'id': 'H5SQ95', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'SMR', 'id': 'H5SQ95', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'GO', 'id': 'GO:0031386', 'properties': [{'key': 'GoTerm', 'value': 'F:protein tag'}, {'key': 'GoEvidenceType', 'value': 'IEA:UniProtKB-UniRule'}]}, {'database': 'HAMAP', 'id': 'MF_02133', 'properties': [{'key': 'EntryName', 'value': 'UBact'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'InterPro', 'id': 'IPR037543', 'properties': [{'key': 'EntryName', 'value': 'UBact'}]}], uniProtkbId='UBACT_ACEAU'), _inner_sync=False)" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "genes[0]" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -790,7 +793,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -813,7 +816,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -830,36 +833,21 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "query in rewrite_sparql SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id type NeuronMorphology;\n", - " ?_constrainedBy;\n", - " ?_createdAt;\n", - " ?_createdBy;\n", - " ?_deprecated;\n", - " ?_incoming;\n", - " ?_outgoing;\n", - " ?_project;\n", - " ?_rev;\n", - " ?_schemaProject;\n", - " ?_self;\n", - " ?_updatedAt;\n", - " ?_updatedBy . \n", - " Filter (?_deprecated = 'false'^^xsd:boolean)\n", - "Filter (?_project = )}}\n", - "0 dataset(s) of type NeuronMorphology found\n" + "10 dataset(s) of type NeuronMorphology found\n" ] } ], "source": [ "limit = 10 # You can limit the number of results, pass `None` to fetch all the results\n", "\n", - "data = forge.search(filters, limit=limit)\n", + "data = forge.search(filters, db_source='MouseLight', limit=limit)\n", "\n", "print(f\"{str(len(data))} dataset(s) of type {_type} found\")" ] @@ -873,7 +861,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -897,20 +885,400 @@ " \n", " \n", " \n", + " id\n", + " brainLocation.brainRegion.id\n", + " brainLocation.brainRegion.label\n", + " contribution.type\n", + " contribution.agent.id\n", + " contribution.agent.type\n", + " distribution.contentUrl\n", + " distribution.encodingFormat\n", + " distribution.name\n", + " name\n", + " ...\n", + " subject.strain\n", + " brainLocation.layer\n", + " contribution.agent.label\n", + " subject.age.period\n", + " subject.age.unitCode\n", + " subject.age.value\n", + " subject.identifier\n", + " subject.name\n", + " subject.sex.label\n", + " subject.strain.label\n", " \n", " \n", " \n", + " \n", + " 0\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " Simple lobule\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.443970.d\n", + " Organization\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " application/swc\n", + " AA1015.swc\n", + " AA1015\n", + " ...\n", + " Sim1-Cre\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " 1\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " Ansiform lobule\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.443970.d\n", + " Organization\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " application/swc\n", + " AA1024.swc\n", + " AA1024\n", + " ...\n", + " Sim1-Cre\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " 2\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " MTG\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " H17.03.010.11.13.01\n", + " ...\n", + " NaN\n", + " 3\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " yrs\n", + " 38\n", + " 601901227.0\n", + " H17.03.010\n", + " Female\n", + " \n", + " \n", + " \n", + " 3\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " FroL\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " H16.06.007.01.07.02\n", + " ...\n", + " NaN\n", + " 3\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " yrs\n", + " 26\n", + " 518229880.0\n", + " H16.06.007\n", + " Male\n", + " \n", + " \n", + " \n", + " 4\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " MTG\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " H16.06.008.01.26.04\n", + " ...\n", + " NaN\n", + " 5\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " yrs\n", + " 24\n", + " 527747035.0\n", + " H16.06.008\n", + " Female\n", + " \n", + " \n", + " \n", + " 5\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure/74\n", + " VISl6a\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " Rorb-IRES2-Cre-D;Ai14-234246.02.02.01\n", + " ...\n", + " NaN\n", + " 6a\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " \n", + " \n", + " 503871576.0\n", + " Rorb-IRES2-Cre-D;Ai14-234246\n", + " \n", + " Rorb-IRES2-Cre\n", + " \n", + " \n", + " 6\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " VISp2/3\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " Chrna2-Cre_OE25;Ai14(IVSCC)-294465.05.01.01\n", + " ...\n", + " NaN\n", + " 2/3\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " \n", + " \n", + " 565087402.0\n", + " Chrna2-Cre_OE25;Ai14(IVSCC)-294465\n", + " \n", + " Chrna2-Cre_OE25\n", + " \n", + " \n", + " 7\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " VISp5\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " Cux2-CreERT2;Ai14-205530.03.02.01\n", + " ...\n", + " NaN\n", + " 5\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " \n", + " \n", + " 485250100.0\n", + " Cux2-CreERT2;Ai14-205530\n", + " \n", + " Cux2-CreERT2\n", + " \n", + " \n", + " 8\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure/33\n", + " VISp6a\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " Ctgf-2A-dgCre;Ai14(IVSCC)-229233.03.02.01\n", + " ...\n", + " NaN\n", + " 6a\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " \n", + " \n", + " 501715368.0\n", + " Ctgf-2A-dgCre;Ai14(IVSCC)-229233\n", + " \n", + " Ctgf-T2A-dgCre\n", + " \n", + " \n", + " 9\n", + " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " MTG\n", + " Contribution\n", + " https://www.grid.ac/institutes/grid.417881.3\n", + " Organization\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " application/swc\n", + " reconstruction.swc\n", + " H17.06.005.12.15.01\n", + " ...\n", + " NaN\n", + " 4\n", + " Allen Institute for Brain Science\n", + " Post-natal\n", + " yrs\n", + " 38\n", + " 571364629.0\n", + " H17.06.005\n", + " Male\n", + " \n", + " \n", " \n", "\n", + "

10 rows × 23 columns

\n", "" ], "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" + " id \\\n", + "0 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "1 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "2 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "3 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "4 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "5 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "6 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "7 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "8 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "9 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", + "\n", + " brainLocation.brainRegion.id \\\n", + "0 http://api.brain-map.org/api/v2/data/Structure... \n", + "1 http://api.brain-map.org/api/v2/data/Structure... \n", + "2 http://api.brain-map.org/api/v2/data/Structure... \n", + "3 http://api.brain-map.org/api/v2/data/Structure... \n", + "4 http://api.brain-map.org/api/v2/data/Structure... \n", + "5 http://api.brain-map.org/api/v2/data/Structure/74 \n", + "6 http://api.brain-map.org/api/v2/data/Structure... \n", + "7 http://api.brain-map.org/api/v2/data/Structure... \n", + "8 http://api.brain-map.org/api/v2/data/Structure/33 \n", + "9 http://api.brain-map.org/api/v2/data/Structure... \n", + "\n", + " brainLocation.brainRegion.label contribution.type \\\n", + "0 Simple lobule Contribution \n", + "1 Ansiform lobule Contribution \n", + "2 MTG Contribution \n", + "3 FroL Contribution \n", + "4 MTG Contribution \n", + "5 VISl6a Contribution \n", + "6 VISp2/3 Contribution \n", + "7 VISp5 Contribution \n", + "8 VISp6a Contribution \n", + "9 MTG Contribution \n", + "\n", + " contribution.agent.id contribution.agent.type \\\n", + "0 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "1 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "2 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "3 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "4 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "5 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "6 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "7 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "8 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "9 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "\n", + " distribution.contentUrl \\\n", + "0 https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "1 https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "2 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "3 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "4 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "5 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "6 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "7 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "8 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "9 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "\n", + " distribution.encodingFormat distribution.name \\\n", + "0 application/swc AA1015.swc \n", + "1 application/swc AA1024.swc \n", + "2 application/swc reconstruction.swc \n", + "3 application/swc reconstruction.swc \n", + "4 application/swc reconstruction.swc \n", + "5 application/swc reconstruction.swc \n", + "6 application/swc reconstruction.swc \n", + "7 application/swc reconstruction.swc \n", + "8 application/swc reconstruction.swc \n", + "9 application/swc reconstruction.swc \n", + "\n", + " name ... subject.strain \\\n", + "0 AA1015 ... Sim1-Cre \n", + "1 AA1024 ... Sim1-Cre \n", + "2 H17.03.010.11.13.01 ... NaN \n", + "3 H16.06.007.01.07.02 ... NaN \n", + "4 H16.06.008.01.26.04 ... NaN \n", + "5 Rorb-IRES2-Cre-D;Ai14-234246.02.02.01 ... NaN \n", + "6 Chrna2-Cre_OE25;Ai14(IVSCC)-294465.05.01.01 ... NaN \n", + "7 Cux2-CreERT2;Ai14-205530.03.02.01 ... NaN \n", + "8 Ctgf-2A-dgCre;Ai14(IVSCC)-229233.03.02.01 ... NaN \n", + "9 H17.06.005.12.15.01 ... NaN \n", + "\n", + " brainLocation.layer contribution.agent.label subject.age.period \\\n", + "0 NaN NaN NaN \n", + "1 NaN NaN NaN \n", + "2 3 Allen Institute for Brain Science Post-natal \n", + "3 3 Allen Institute for Brain Science Post-natal \n", + "4 5 Allen Institute for Brain Science Post-natal \n", + "5 6a Allen Institute for Brain Science Post-natal \n", + "6 2/3 Allen Institute for Brain Science Post-natal \n", + "7 5 Allen Institute for Brain Science Post-natal \n", + "8 6a Allen Institute for Brain Science Post-natal \n", + "9 4 Allen Institute for Brain Science Post-natal \n", + "\n", + " subject.age.unitCode subject.age.value subject.identifier \\\n", + "0 NaN NaN NaN \n", + "1 NaN NaN NaN \n", + "2 yrs 38 601901227.0 \n", + "3 yrs 26 518229880.0 \n", + "4 yrs 24 527747035.0 \n", + "5 503871576.0 \n", + "6 565087402.0 \n", + "7 485250100.0 \n", + "8 501715368.0 \n", + "9 yrs 38 571364629.0 \n", + "\n", + " subject.name subject.sex.label subject.strain.label \n", + "0 NaN NaN NaN \n", + "1 NaN NaN NaN \n", + "2 H17.03.010 Female \n", + "3 H16.06.007 Male \n", + "4 H16.06.008 Female \n", + "5 Rorb-IRES2-Cre-D;Ai14-234246 Rorb-IRES2-Cre \n", + "6 Chrna2-Cre_OE25;Ai14(IVSCC)-294465 Chrna2-Cre_OE25 \n", + "7 Cux2-CreERT2;Ai14-205530 Cux2-CreERT2 \n", + "8 Ctgf-2A-dgCre;Ai14(IVSCC)-229233 Ctgf-T2A-dgCre \n", + "9 H17.06.005 Male \n", + "\n", + "[10 rows x 23 columns]" ] }, - "execution_count": 33, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -931,15 +1299,15 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " download\n", - " DownloadingError: path to follow 'distribution.contentUrl' was not found in any provided resource.\n", + " send\n", + " ConnectionError: HTTPSConnectionPool(host='staging.nexus.ocp.bbp.epfl.ch', port=443): Max retries exceeded with url: /v1/files/dke/kgforge/f0c59cf7-3b1b-4cb9-b81b-62fefd5bbd0f (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))\n", "\n" ] } @@ -948,6 +1316,79 @@ "dirpath = \"./downloaded/\"\n", "forge.download(data, \"distribution.contentUrl\", dirpath)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Try query" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted query:\n", + " \n", + " # PREFIXES\n", + " SELECT ?id WHERE {\n", + " ?id a nsg:DetailedCircuit\n", + " } LIMIT 100\n", + "\n", + " _sparql\n", + " QueryingError: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/views/neurosciencegraph/datamodels/https%3A%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FdefaultSparqlIndex/sparql\n", + "\n" + ] + } + ], + "source": [ + "mquery = \"\"\"\n", + "# PREFIXES\n", + "SELECT ?id WHERE {\n", + " ?id a nsg:DetailedCircuit\n", + "} LIMIT 100\n", + "\"\"\"\n", + "\n", + "forge.sparql(mquery, debug=True, rewrite=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted query:\n", + " \n", + " # PREFIXES\n", + " SELECT ?id WHERE {\n", + " ?id a nsg:DetailedCircuit\n", + " } LIMIT 100\n", + "\n", + " _sparql\n", + " QueryingError: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/views/neurosciencegraph/datamodels/https%3A%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FdefaultSparqlIndex/sparql\n", + "\n" + ] + } + ], + "source": [ + "forge.sparql(mquery, debug=True, rewrite=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 686aea43ebf8e2618fdb80d68052fd3724e30db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 17 Oct 2022 18:21:46 +0200 Subject: [PATCH 13/30] Make Databases be new archetype. Updated most methods, missing search, sparql, and tests. --- .../prod-nexus-sources_progress.yml | 82 ++ .../17 - Database-sources.ipynb | 1080 ++++++++--------- kgforge/core/archetypes/__init__.py | 1 + kgforge/core/archetypes/store.py | 39 + kgforge/core/forge.py | 71 +- kgforge/specializations/databases/__init__.py | 15 + .../databases/store_database.py | 91 ++ kgforge/specializations/resources/__init__.py | 1 - .../specializations/resources/db_sources.py | 157 --- .../specializations/stores/bluebrain_nexus.py | 41 +- .../stores/{uniprot => databases}/__init__.py | 0 .../stores/{uniprot => databases}/service.py | 116 -- kgforge/specializations/stores/sparql.py | 52 +- .../specializations/stores/uniprot_store.py | 3 - 14 files changed, 799 insertions(+), 950 deletions(-) create mode 100644 examples/configurations/database-sources/prod-nexus-sources_progress.yml create mode 100644 kgforge/specializations/databases/__init__.py create mode 100644 kgforge/specializations/databases/store_database.py delete mode 100644 kgforge/specializations/resources/db_sources.py rename kgforge/specializations/stores/{uniprot => databases}/__init__.py (100%) rename kgforge/specializations/stores/{uniprot => databases}/service.py (54%) diff --git a/examples/configurations/database-sources/prod-nexus-sources_progress.yml b/examples/configurations/database-sources/prod-nexus-sources_progress.yml new file mode 100644 index 00000000..9e87a3ec --- /dev/null +++ b/examples/configurations/database-sources/prod-nexus-sources_progress.yml @@ -0,0 +1,82 @@ +Model: + name: RdfModel + origin: store + source: BlueBrainNexus + context: + iri: "https://bbp.neuroshapes.org" + bucket: "neurosciencegraph/datamodels" +Store: + name: BlueBrainNexus + endpoint: https://bbp.epfl.ch/nexus/v1 + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + elastic: + endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" + mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" + default_str_keyword_field: "keyword" + vocabulary: + metadata: + iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" + local_iri: "https://bluebrainnexus.io/contexts/metadata.json" + namespace: "https://bluebrain.github.io/nexus/vocabulary/" + deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" + project_property: "https://bluebrain.github.io/nexus/vocabulary/project" + max_connection: 50 + token: eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjYwMjkyNjEsImlhdCI6MTY2NjAxODIzNSwiYXV0aF90aW1lIjoxNjY2MDAwNDYxLCJqdGkiOiIxNjQwNTBkOS04OTAzLTQzMjItOWQyZi0xOGQ0ODg3MTYzYWEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwOWM0NDA5YjExZjRkNjI4YjM2NmM2N2MxOGU0MDk5Iiwic2Vzc2lvbl9zdGF0ZSI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.XHZ55pSp2zRQcgcvsXHYWS2adj12K411PtQaZesiABtKH9tgsjz0dgY_4ijneBHdHd1zgmqNRFqnHHd78EMD7XoHtmbpZOI5h1t12wK2YbI35mOwLfS1RcRnT6kKMuiFe5m4AzW1XWwHn9AcO_xeZTbixL2vqPLfdPg1jzk5uW-rgs3Sg-77Kr3LEBVTzVSHSokhtTqjHNY2EQlBljmpJpGCno-6keLX5LrNERFek303oJrAIgvfGSCcpKoHYYWCPiReD4-cjd8l-zMCVrfxCeuRhihVSpNGgPX67F4O7u8EnU_EMeA39eg9HSp5KahhrGwecCzqPMVg4R37cysz_Q + versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" + file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson + +Resolvers: + ontology: + - resolver: OntologyResolver + origin: store + source: BlueBrainNexus + targets: + - identifier: terms + bucket: neurosciencegraph/datamodels + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson + agent: + - resolver: AgentResolver + origin: store + source: BlueBrainNexus + targets: + - identifier: agents + bucket: bbp/agents + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson + +Formatters: + identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} + identifier_neuroelectro: http://neuroelectro.org/api/1/{}/{} + identifier_neurolex: http://neurolex.org/wiki/{} + +Databases: + UniProt: + origin: store + source: SPARQLStore + endpoint: "http://purl.uniprot.org/core/" + searchendpoints: + sparql: + endpoint: "https://sparql.uniprot.org/sparql" + model: + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt + context: + iri: "https://bbp.epfl.ch/jsonldcontext/db/uniprot" + bucket: jsonld_context.json + + NeuroElectro: + origin: store + source: BlueBrainNexus + bucket: bbp/neuroelectro + model: + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro + context: + bucket: jsonld_context.json \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 3f833314..622ef042 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -33,11 +33,29 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I am in BBNexus store\n", + "I am in BBNexus store\n", + "I am in BBNexus store\n", + "I am in BBNexus store\n", + "Configuration {'origin': 'store', 'source': 'SPARQLStore', 'endpoint': 'http://purl.uniprot.org/core/', 'searchendpoints': {'sparql': {'endpoint': 'https://sparql.uniprot.org/sparql'}}, 'model': {'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', 'context': {'iri': 'https://bbp.epfl.ch/jsonldcontext/db/uniprot', 'bucket': 'jsonld_context.json'}}}\n", + "store config {'endpoint': 'http://purl.uniprot.org/core/', 'searchendpoints': {'sparql': {'endpoint': 'https://sparql.uniprot.org/sparql'}}, 'model_context': }\n", + "inside SPARQL Store\n", + "initializing service\n", + "Configuration {'origin': 'store', 'source': 'BlueBrainNexus', 'bucket': 'bbp/neuroelectro', 'model': {'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', 'context': {'bucket': 'jsonld_context.json'}}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjYwMjkyNjEsImlhdCI6MTY2NjAxODIzNSwiYXV0aF90aW1lIjoxNjY2MDAwNDYxLCJqdGkiOiIxNjQwNTBkOS04OTAzLTQzMjItOWQyZi0xOGQ0ODg3MTYzYWEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwOWM0NDA5YjExZjRkNjI4YjM2NmM2N2MxOGU0MDk5Iiwic2Vzc2lvbl9zdGF0ZSI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.XHZ55pSp2zRQcgcvsXHYWS2adj12K411PtQaZesiABtKH9tgsjz0dgY_4ijneBHdHd1zgmqNRFqnHHd78EMD7XoHtmbpZOI5h1t12wK2YbI35mOwLfS1RcRnT6kKMuiFe5m4AzW1XWwHn9AcO_xeZTbixL2vqPLfdPg1jzk5uW-rgs3Sg-77Kr3LEBVTzVSHSokhtTqjHNY2EQlBljmpJpGCno-6keLX5LrNERFek303oJrAIgvfGSCcpKoHYYWCPiReD4-cjd8l-zMCVrfxCeuRhihVSpNGgPX67F4O7u8EnU_EMeA39eg9HSp5KahhrGwecCzqPMVg4R37cysz_Q', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'model_context': , 'name': 'BlueBrainNexus'}\n", + "store config {'bucket': 'bbp/neuroelectro', 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjYwMjkyNjEsImlhdCI6MTY2NjAxODIzNSwiYXV0aF90aW1lIjoxNjY2MDAwNDYxLCJqdGkiOiIxNjQwNTBkOS04OTAzLTQzMjItOWQyZi0xOGQ0ODg3MTYzYWEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwOWM0NDA5YjExZjRkNjI4YjM2NmM2N2MxOGU0MDk5Iiwic2Vzc2lvbl9zdGF0ZSI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.XHZ55pSp2zRQcgcvsXHYWS2adj12K411PtQaZesiABtKH9tgsjz0dgY_4ijneBHdHd1zgmqNRFqnHHd78EMD7XoHtmbpZOI5h1t12wK2YbI35mOwLfS1RcRnT6kKMuiFe5m4AzW1XWwHn9AcO_xeZTbixL2vqPLfdPg1jzk5uW-rgs3Sg-77Kr3LEBVTzVSHSokhtTqjHNY2EQlBljmpJpGCno-6keLX5LrNERFek303oJrAIgvfGSCcpKoHYYWCPiReD4-cjd8l-zMCVrfxCeuRhihVSpNGgPX67F4O7u8EnU_EMeA39eg9HSp5KahhrGwecCzqPMVg4R37cysz_Q', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'model_context': None}\n", + "I am in BBNexus store\n" + ] + } + ], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"neurosciencegraph/datamodels\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", endpoint=endpoint, bucket=BUCKET)" + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources_progress.yml\", endpoint=endpoint, bucket=BUCKET)" ] }, { @@ -58,8 +76,7 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n", - "MouseLight\n" + "NeuroElectro\n" ] } ], @@ -73,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "sources = forge.db_sources(pretty=False)" + "sources = forge.db_sources()" ] }, { @@ -84,10 +101,9 @@ "source": [ "\n", "data = {\n", - " 'store':{\n", - " 'name': 'DemoStore'\n", - " },\n", - " 'model': { \n", + " 'origin': 'store',\n", + " 'source': 'DemoStore',\n", + " 'model': { \n", " 'name': 'DemoModel',\n", " 'origin': 'directory',\n", " 'source': \"../../../tests/data/demo-model/\" \n", @@ -99,10 +115,18 @@ "cell_type": "code", "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "store config {'model_context': None}\n" + ] + } + ], "source": [ - "from kgforge.specializations.resources import DatabaseSource\n", - "ds = DatabaseSource(forge, name=\"DemoDB\", from_forge=False, **data)" + "from kgforge.specializations.databases import StoreDatabase\n", + "ds = StoreDatabase(forge, name=\"DemoDB\", **data)" ] }, { @@ -125,9 +149,7 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n", - "MouseLight\n", - "DemoDB\n" + "NeuroElectro\n" ] } ], @@ -148,35 +170,7 @@ "metadata": {}, "outputs": [], "source": [ - "mouselight = sources[\"MouseLight\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Name, description, url, license, protocol => more can be added through configuration" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MouseLight\n", - "https://www.janelia.org/project-team/mouselight/resources\n", - "{'id': 'https://creativecommons.org/licenses/by-nc/4.0', 'label': 'CC BY-NC 4.0'}\n" - ] - } - ], - "source": [ - "print(mouselight.name)\n", - "print(mouselight.protocol)\n", - "print(mouselight.license)" + "neuroelectro = sources['NeuroElectro']" ] }, { @@ -191,86 +185,62 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'NeuronMorphology': ['DictionaryMapping']}" + "{'ElectrophysiologicalFeatureAnnotation': ['DictionaryMapping'],\n", + " 'ParameterAnnotation': ['DictionaryMapping'],\n", + " 'ParameterBody': ['DictionaryMapping'],\n", + " 'ScholarlyArticle': ['DictionaryMapping'],\n", + " 'SeriesBody': ['DictionaryMapping']}" ] }, - "execution_count": 11, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "forge.mappings(\"MouseLight\", pretty=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Managed mappings for the data source per entity type and mapping type:\n", - " - Gene:\n", - " * DictionaryMapping\n", - " - Protein:\n", - " * DictionaryMapping\n" - ] - } - ], - "source": [ - "forge.mappings('UniProt', pretty=True)" + "forge.mappings(\"NeuroElectro\")" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Managed mappings for the data source per entity type and mapping type:\n", - " - ElectrophysiologicalFeatureAnnotation:\n", - " * DictionaryMapping\n", - " - ParameterAnnotation:\n", - " * DictionaryMapping\n", - " - ParameterBody:\n", - " * DictionaryMapping\n", - " - ScholarlyArticle:\n", - " * DictionaryMapping\n", - " - SeriesBody:\n", - " * DictionaryMapping\n" - ] + "data": { + "text/plain": [ + "{'Gene': ['DictionaryMapping'], 'Protein': ['DictionaryMapping']}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "forge.mappings('NeuroElectro', pretty=True)" + "forge.mappings('UniProt')" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "from kgforge.specializations.mappings import DictionaryMapping\n", - "mapping = forge.mapping(\"NeuronMorphology\", \"MouseLight\", type=DictionaryMapping)\n", - "direct_mapping = mouselight.mapping(\"NeuronMorphology\", type=DictionaryMapping)" + "mapping = forge.mapping(\"ScholarlyArticle\", \"NeuroElectro\")\n", + "direct_mapping = neuroelectro.mapping(\"ScholarlyArticle\", type=DictionaryMapping)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -278,78 +248,27 @@ "output_type": "stream", "text": [ "{\n", - " id: forge.format(\"identifier\", \"neuronmorphologies/mouselight\", x.neurons[0][\"idString\"])\n", + " id: forge.format(\"identifier\", \"scholarlyarticles\", x.id)\n", " type:\n", " [\n", - " Dataset\n", - " NeuronMorphology\n", + " Entity\n", + " ScholarlyArticle\n", " ]\n", - " brainLocation:\n", - " {\n", - " type: BrainLocation\n", - " brainRegion:\n", - " {\n", - " id: f\"http://api.brain-map.org/api/v2/data/Structure/{x.neurons[0]['soma']['allenId']}\"\n", - " label: x.neurons[0][\"allenLabel\"]\n", - " }\n", - " coordinatesInBrainAtlas:\n", - " {\n", - " valueX: x.neurons[0][\"soma\"][\"x\"]\n", - " valueY: x.neurons[0][\"soma\"][\"y\"]\n", - " valueZ: x.neurons[0][\"soma\"][\"z\"]\n", - " }\n", - " }\n", - " contribution:\n", - " {\n", - " type: Contribution\n", - " agent:\n", - " {\n", - " id: https://www.grid.ac/institutes/grid.443970.d\n", - " type: Organization\n", - " label: Janelia Research Campus\n", - " }\n", - " }\n", - " dateCreated: x.neurons[0][\"sample\"][\"date\"]\n", - " description: x.neurons[0][\"annotationSpace\"][\"description\"]\n", - " distribution: forge.attach(f\"./mouselight/{x.neurons[0]['idString']}.swc\", content_type=\"application/swc\")\n", - " fluorophore: x.neurons[0][\"label\"][\"fluorophore\"]\n", - " generation:\n", - " {\n", - " type: Generation\n", - " activity:\n", - " {\n", - " type: nsg:NeuronMorphologyReconstruction\n", - " hadProtocol: {}\n", - " }\n", - " }\n", - " identifier: x.neurons[0][\"idString\"]\n", - " license:\n", - " {\n", - " id: https://mouselight.janelia.org\n", - " type: License\n", - " }\n", - " name: x.neurons[0][\"idString\"]\n", - " objectOfStudy:\n", - " {\n", - " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", - " type: ObjectOfStudy\n", - " label: Single Cell\n", - " }\n", - " subject:\n", + " abstract: x.abstract\n", + " author: x.authors_shaped\n", + " datePublished: x.date_issued\n", + " identifier: x.identifiers\n", + " isPartOf:\n", " {\n", - " type: Subject\n", - " species:\n", - " {\n", - " id: http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " label: Mus musculus\n", - " }\n", - " strain:\n", - " {\n", - " label: x.neurons[0][\"sample\"][\"strain\"]\n", - " }\n", + " type: Periodical\n", + " issn: x.issn\n", + " name: x.journal\n", + " publisher: x.publisher\n", " }\n", - " version: x.neurons[0][\"annotationSpace\"][\"version\"]\n", - " virus: x.neurons[0][\"label\"][\"virus\"]\n", + " name: f\"article_{x.id}\"\n", + " sameAs: x.full_text_link\n", + " title: x.title\n", + " url: x.full_text_link\n", "}\n" ] } @@ -360,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -368,78 +287,27 @@ "output_type": "stream", "text": [ "{\n", - " id: forge.format(\"identifier\", \"neuronmorphologies/mouselight\", x.neurons[0][\"idString\"])\n", + " id: forge.format(\"identifier\", \"scholarlyarticles\", x.id)\n", " type:\n", " [\n", - " Dataset\n", - " NeuronMorphology\n", + " Entity\n", + " ScholarlyArticle\n", " ]\n", - " brainLocation:\n", - " {\n", - " type: BrainLocation\n", - " brainRegion:\n", - " {\n", - " id: f\"http://api.brain-map.org/api/v2/data/Structure/{x.neurons[0]['soma']['allenId']}\"\n", - " label: x.neurons[0][\"allenLabel\"]\n", - " }\n", - " coordinatesInBrainAtlas:\n", - " {\n", - " valueX: x.neurons[0][\"soma\"][\"x\"]\n", - " valueY: x.neurons[0][\"soma\"][\"y\"]\n", - " valueZ: x.neurons[0][\"soma\"][\"z\"]\n", - " }\n", - " }\n", - " contribution:\n", - " {\n", - " type: Contribution\n", - " agent:\n", - " {\n", - " id: https://www.grid.ac/institutes/grid.443970.d\n", - " type: Organization\n", - " label: Janelia Research Campus\n", - " }\n", - " }\n", - " dateCreated: x.neurons[0][\"sample\"][\"date\"]\n", - " description: x.neurons[0][\"annotationSpace\"][\"description\"]\n", - " distribution: forge.attach(f\"./mouselight/{x.neurons[0]['idString']}.swc\", content_type=\"application/swc\")\n", - " fluorophore: x.neurons[0][\"label\"][\"fluorophore\"]\n", - " generation:\n", - " {\n", - " type: Generation\n", - " activity:\n", - " {\n", - " type: nsg:NeuronMorphologyReconstruction\n", - " hadProtocol: {}\n", - " }\n", - " }\n", - " identifier: x.neurons[0][\"idString\"]\n", - " license:\n", - " {\n", - " id: https://mouselight.janelia.org\n", - " type: License\n", - " }\n", - " name: x.neurons[0][\"idString\"]\n", - " objectOfStudy:\n", - " {\n", - " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", - " type: ObjectOfStudy\n", - " label: Single Cell\n", - " }\n", - " subject:\n", + " abstract: x.abstract\n", + " author: x.authors_shaped\n", + " datePublished: x.date_issued\n", + " identifier: x.identifiers\n", + " isPartOf:\n", " {\n", - " type: Subject\n", - " species:\n", - " {\n", - " id: http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " label: Mus musculus\n", - " }\n", - " strain:\n", - " {\n", - " label: x.neurons[0][\"sample\"][\"strain\"]\n", - " }\n", + " type: Periodical\n", + " issn: x.issn\n", + " name: x.journal\n", + " publisher: x.publisher\n", " }\n", - " version: x.neurons[0][\"annotationSpace\"][\"version\"]\n", - " virus: x.neurons[0][\"label\"][\"virus\"]\n", + " name: f\"article_{x.id}\"\n", + " sameAs: x.full_text_link\n", + " title: x.title\n", + " url: x.full_text_link\n", "}\n" ] } @@ -450,29 +318,22 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Available Database sources:\n", - "MouseLight\n" - ] + "data": { + "text/plain": [ + "{'UniProt': StoreDatabase(context=, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, name='UniProt', service=SPARQLStore(context=None, bucket=None, endpoint='http://purl.uniprot.org/core/', file_mapping=None, metadata_context=None, model_context=, service=, token=None, versioned_id_template=None), source='SPARQLStore')}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "forge.db_sources(with_datatype='NeuronMorphology', pretty=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "ne = sources['NeuroElectro']" + "forge.db_sources(mappings='Gene')" ] }, { @@ -487,9 +348,19 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " search\n", + " ValueError: context model missing\n", + "\n" + ] + } + ], "source": [ "# Type, source or target brain region, \n", "filters = {\"type\":\"ScholarlyArticle\"}\n", @@ -500,18 +371,19 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "2" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" + "ename": "TypeError", + "evalue": "object of type 'NoneType' has no len()", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_81136/3721108832.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()" + ] } ], "source": [ @@ -520,7 +392,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -529,51 +401,56 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/34164\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/14353\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Striatal spiny projection (SP) neurons control movement initiation by integrating cortical inputs and inhibiting basal ganglia outputs. Central to this control lies a \"microcircuit\" that consists of a feedback pathway formed by axon collaterals between GABAergic SP neurons and a feedforward pathway from fast spiking (FS) GABAergic interneurons to SP neurons. Here, somatically evoked postsynaptic potentials (PSPs) and currents (PSCs) were compared for both pathways with dual whole cell patch recording in voltage- and current-clamp mode using cortex-striatum-substantia nigra organotypic cultures. On average, feedforward inputs were 1 ms earlier, more reliable, and about twice as large in amplitude compared with most feedback inputs. On the other hand, both pathways exhibited widely varying, partially overlapping amplitude distributions. This variability was already established for single FS neurons targeting many SP neurons. In response to precisely timed action potential bursts, feedforward and feedback inputs consistently showed short-term depression < or =50-70% in voltage-clamp, although feedback inputs also displayed strong augmentation in current-clamp in line with previous reports. The augmentation of feedback inputs was absent in gramicidin D perforated-patch recording, which also showed the natural reversal potential for both inputs to be near firing threshold. Preceding depolarizing feedback inputs during the down state did not consistently change subsequent postsynaptic action potentials. We conclude that feedback and feedforward inputs have their dominant effect during the up-state. The reversal potential close to the up-state potential, which supports shunting operation with millisecond precision and the strong synaptic depression, should enable both pathways to carry time-critical information.\n", + " abstract: Cerebellar Purkinje cells (PCs) from spinocerebellar ataxia type 1 (SCA1) transgenic mice develop dendritic and somatic atrophy with age. Inositol 1,4,5-trisphosphate receptor type 1 and the sarco/endoplasmic reticulum Ca(2+) ATPase pump, which regulate [Ca(2+)](i), are expressed at lower levels in these cells compared with the levels in cells from wild-type (WT) mice. To examine PCs in SCA1 mice, we used whole-cell patch clamp recording combined with fluorometric [Ca(2+)](i) and [Na(+)](i) measurements in cerebellar slices. PCs in SCA1 mice had Na(+) spikes, Ca(2+) spikes, climbing fiber (CF) electrical responses, parallel fiber (PF) electrical responses, and metabotropic glutamate receptor (mGluR)-mediated, PF-evoked Ca(2+) release from intracellular stores that were qualitatively similar to those recorded from WT mice. Under our experimental conditions, it was easier to evoke the mGluR-mediated secondary [Ca(2+)](i) increase in SCA1 PCs. The membrane resistance of SCA1 PCs was 3.3 times higher than that of WT cells, which correlated with the 1.7 times smaller cell body size. Most SCA1 PCs (but not WT) had a delayed onset (about 50--200 ms) to Na(+) spike firing induced by current injection. This delay was increased by hyperpolarizing prepulses and was eliminated by 4-aminopyridine, which suggests that this delay was due to enhancement of the A-like K(+) conductance in the SCA1 PCs. In response to CF stimulation, most PCs in mutant and WT mice had rapid, widespread [Ca(2+)](i) changes that recovered in <200 ms. Some SCA1 PCs showed a slow, localized, secondary Ca(2+) transient following the initial CF Ca(2+) transient, which may reflect release of Ca(2+) from intracellular stores. Thus, with these exceptions, the basic physiological properties of mutant PCs are similar to those of WT neurons, even with dramatic alteration of their morphology and downregulation of Ca(2+) handling molecules.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Gustafson\n", - " givenName: Nicholas\n", + " familyName: Inoue\n", + " givenName: T\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Lin\n", + " givenName: X\n", " }\n", " {\n", " type: Person\n", - " familyName: Gireesh-Dharmaraj\n", - " givenName: Elakkat\n", + " familyName: Kohlmeier\n", + " givenName: K A\n", " }\n", " {\n", " type: Person\n", - " familyName: Czubayko\n", - " givenName: Uwe\n", + " familyName: Orr\n", + " givenName: H T\n", " }\n", " {\n", " type: Person\n", - " familyName: Blackwell\n", - " givenName: Kim T\n", + " familyName: Zoghbi\n", + " givenName: H Y\n", " }\n", " {\n", " type: Person\n", - " familyName: Plenz\n", - " givenName: Dietmar\n", + " familyName: Ross\n", + " givenName: W N\n", " }\n", " ]\n", - " datePublished: 2006-2\n", + " datePublished: 2001-4\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 16236782\n", + " value: 11287496\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1152/jn.00802.2005\n", + " value: 10.1152/jn.2001.85.4.1750\n", " }\n", " ]\n", " isPartOf:\n", @@ -582,10 +459,10 @@ " issn: 0022-3077\n", " name: Journal of neurophysiology\n", " }\n", - " name: article_34164\n", - " sameAs: http://jn.physiology.org/content/95/2/737.long\n", - " title: A comparative voltage and current-clamp analysis of feedback and feedforward synaptic transmission in the striatal microcircuit in vitro.\n", - " url: http://jn.physiology.org/content/95/2/737.long\n", + " name: article_14353\n", + " sameAs: http://jn.physiology.org/content/85/4/1750.long\n", + " title: Calcium dynamics and electrophysiological properties of cerebellar Purkinje cells in SCA1 transgenic mice.\n", + " url: http://jn.physiology.org/content/85/4/1750.long\n", "}\n" ] } @@ -596,7 +473,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -612,7 +489,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -647,7 +524,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -667,7 +544,27 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein=Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0A131MCZ8', _inner_sync=False, annotationScore=5.0, comments=[{'texts': [{'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'Probable metal transporter. Probably acts redundantly with the other metal transport proteins cnnm-1, cnnm-2, cnnm-4 and cnnm-5 to regulate Mg(2+) homeostasis. Promotes postembryonic gonad development by regulating Mg(2+) levels, probably via AMPK signaling'}], 'commentType': 'FUNCTION'}, {'commentType': 'SUBCELLULAR LOCATION', 'subcellularLocations': [{'location': {'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'Basolateral cell membrane', 'id': 'SL-0026'}, 'topology': {'evidences': [{'evidenceCode': 'ECO:0000255'}], 'value': 'Multi-pass membrane protein', 'id': 'SL-9909'}}]}, {'commentType': 'ALTERNATIVE PRODUCTS', 'events': ['Alternative splicing'], 'isoforms': [{'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'a'}, 'isoformIds': ['A0A131MCZ8-1'], 'isoformSequenceStatus': 'Displayed'}, {'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2b'}], 'value': 'b'}, 'isoformIds': ['A0A131MCZ8-2'], 'sequenceIds': ['VSP_058662'], 'isoformSequenceStatus': 'Described'}, {'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2c'}], 'value': 'c'}, 'isoformIds': ['A0A131MCZ8-3'], 'sequenceIds': ['VSP_058663', 'VSP_058664'], 'isoformSequenceStatus': 'Described'}, {'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2d'}], 'value': 'd'}, 'isoformIds': ['A0A131MCZ8-4'], 'sequenceIds': ['VSP_058662', 'VSP_058663', 'VSP_058664'], 'isoformSequenceStatus': 'Described'}]}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'Highly expressed in the intestine and in neurons, but it is also expressed in a variety of tissues including the pharynx, hypodermis, rectum and in muscles'}], 'commentType': 'TISSUE SPECIFICITY'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'No visible phenotype. Double knockout with cnnm-1 results in increased levels of intestinal Mg(2+) and reduced levels in other tissues. This Mg(2+) deficiency in tissues leads to a reduced lifespan, 100% sterility, and smaller animals that exhibit a developmental delay with defective gonad development and which therefore do not produce oocytes or form vulva. In addition, the gonad development defect in the cnnm-1 and cnnm-3 double knockout is rescued when the AMPK alpha subunit aak-2 is also knocked out. Double knockout with cnnm-2 results in 22% sterility. Quintuple knockout with cnnm-1, cnnm-2, cnnm-4 and cnnm-5 results in a reduced lifespan and 100% sterility'}], 'commentType': 'DISRUPTION PHENOTYPE'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000305'}], 'value': 'Belongs to the ACDP family'}], 'commentType': 'SIMILARITY'}], entryAudit={'firstPublicDate': '2016-11-30', 'lastAnnotationUpdateDate': '2022-08-03', 'lastSequenceUpdateDate': '2016-05-11', 'entryVersion': 37, 'sequenceVersion': 1}, entryType='UniProtKB reviewed (Swiss-Prot)', extraAttributes={'countByCommentType': {'FUNCTION': 1, 'SUBCELLULAR LOCATION': 1, 'ALTERNATIVE PRODUCTS': 4, 'TISSUE SPECIFICITY': 1, 'DISRUPTION PHENOTYPE': 1, 'SIMILARITY': 1}, 'countByFeatureType': {'Signal': 1, 'Chain': 1, 'Topological domain': 5, 'Transmembrane': 4, 'Domain': 3, 'Glycosylation': 7, 'Alternative sequence': 3}, 'uniParcId': 'UPI00077FE610'}, features=[{'type': 'Signal', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 23, 'modifier': 'EXACT'}}, 'description': '', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Chain', 'location': {'start': {'value': 24, 'modifier': 'EXACT'}, 'end': {'value': 797, 'modifier': 'EXACT'}}, 'description': 'Metal transporter cnnm-3', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'PRO_5007283697'}, {'type': 'Topological domain', 'location': {'start': {'value': 24, 'modifier': 'EXACT'}, 'end': {'value': 200, 'modifier': 'EXACT'}}, 'description': 'Extracellular', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 201, 'modifier': 'EXACT'}, 'end': {'value': 221, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 222, 'modifier': 'EXACT'}, 'end': {'value': 255, 'modifier': 'EXACT'}}, 'description': 'Cytoplasmic', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 256, 'modifier': 'EXACT'}, 'end': {'value': 276, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 277, 'modifier': 'EXACT'}, 'end': {'value': 280, 'modifier': 'EXACT'}}, 'description': 'Extracellular', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 281, 'modifier': 'EXACT'}, 'end': {'value': 301, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 302, 'modifier': 'EXACT'}, 'end': {'value': 322, 'modifier': 'EXACT'}}, 'description': 'Cytoplasmic', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 323, 'modifier': 'EXACT'}, 'end': {'value': 343, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 344, 'modifier': 'EXACT'}, 'end': {'value': 797, 'modifier': 'EXACT'}}, 'description': 'Extracellular', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Domain', 'location': {'start': {'value': 193, 'modifier': 'EXACT'}, 'end': {'value': 373, 'modifier': 'EXACT'}}, 'description': 'CNNM transmembrane', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU01193'}]}, {'type': 'Domain', 'location': {'start': {'value': 393, 'modifier': 'EXACT'}, 'end': {'value': 454, 'modifier': 'EXACT'}}, 'description': 'CBS 1', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00703'}]}, {'type': 'Domain', 'location': {'start': {'value': 461, 'modifier': 'EXACT'}, 'end': {'value': 527, 'modifier': 'EXACT'}}, 'description': 'CBS 2', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00703'}]}, {'type': 'Glycosylation', 'location': {'start': {'value': 31, 'modifier': 'EXACT'}, 'end': {'value': 31, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 42, 'modifier': 'EXACT'}, 'end': {'value': 42, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 676, 'modifier': 'EXACT'}, 'end': {'value': 676, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 692, 'modifier': 'EXACT'}, 'end': {'value': 692, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 724, 'modifier': 'EXACT'}, 'end': {'value': 724, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 731, 'modifier': 'EXACT'}, 'end': {'value': 731, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 761, 'modifier': 'EXACT'}, 'end': {'value': 761, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Alternative sequence', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 547, 'modifier': 'EXACT'}}, 'description': 'in isoform b and isoform d', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'VSP_058662', 'alternativeSequence': {}}, {'type': 'Alternative sequence', 'location': {'start': {'value': 753, 'modifier': 'EXACT'}, 'end': {'value': 766, 'modifier': 'EXACT'}}, 'description': 'in isoform c and isoform d', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'VSP_058663', 'alternativeSequence': {'originalSequence': 'DAVSTPIRNGSVKL', 'alternativeSequences': ['KYLIGRWKRRGTYV']}}, {'type': 'Alternative sequence', 'location': {'start': {'value': 767, 'modifier': 'EXACT'}, 'end': {'value': 797, 'modifier': 'EXACT'}}, 'description': 'in isoform c and isoform d', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'VSP_058664', 'alternativeSequence': {}}], genes=[{'geneName': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'cnnm-3'}, 'orfNames': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'C33D12.2'}]}], keywords=[{'id': 'KW-0025', 'category': 'Coding sequence diversity', 'name': 'Alternative splicing'}, {'id': 'KW-0129', 'category': 'Domain', 'name': 'CBS domain'}, {'id': 'KW-1003', 'category': 'Cellular component', 'name': 'Cell membrane'}, {'id': 'KW-0325', 'category': 'PTM', 'name': 'Glycoprotein'}, {'id': 'KW-0406', 'category': 'Biological process', 'name': 'Ion transport'}, {'id': 'KW-0472', 'category': 'Cellular component', 'name': 'Membrane'}, {'id': 'KW-1185', 'category': 'Technical term', 'name': 'Reference proteome'}, {'id': 'KW-0677', 'category': 'Domain', 'name': 'Repeat'}, {'id': 'KW-0732', 'category': 'Domain', 'name': 'Signal'}, {'id': 'KW-0812', 'category': 'Domain', 'name': 'Transmembrane'}, {'id': 'KW-1133', 'category': 'Domain', 'name': 'Transmembrane helix'}, {'id': 'KW-0813', 'category': 'Biological process', 'name': 'Transport'}], organism={'scientificName': 'Caenorhabditis elegans', 'taxonId': 6239, 'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'Proteomes', 'id': 'UP000001940'}], 'lineage': ['Eukaryota', 'Metazoa', 'Ecdysozoa', 'Nematoda', 'Chromadorea', 'Rhabditida', 'Rhabditina', 'Rhabditomorpha', 'Rhabditoidea', 'Rhabditidae', 'Peloderinae', 'Caenorhabditis']}, primaryAccession='A0A131MCZ8', proteinDescription={'recommendedName': {'fullName': {'evidences': [{'evidenceCode': 'ECO:0000305'}], 'value': 'Metal transporter cnnm-3'}}, 'alternativeNames': [{'fullName': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'CNNM family homolog 3'}}], 'flag': 'Precursor'}, proteinExistence='2: Evidence at transcript level', references=[{'citation': {'id': '9851916', 'citationType': 'journal article', 'authoringGroup': ['The C. elegans sequencing consortium'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '9851916'}, {'database': 'DOI', 'id': '10.1126/science.282.5396.2012'}], 'title': 'Genome sequence of the nematode C. elegans: a platform for investigating biology.', 'publicationDate': '1998', 'journal': 'Science', 'firstPage': '2012', 'lastPage': '2018', 'volume': '282'}, 'referencePositions': ['NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]'], 'referenceComments': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'Proteomes', 'id': 'UP000001940'}], 'value': 'Bristol N2', 'type': 'STRAIN'}], 'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'Proteomes', 'id': 'UP000001940'}]}, {'citation': {'id': '27564576', 'citationType': 'journal article', 'authors': ['Ishii T.', 'Funato Y.', 'Hashizume O.', 'Yamazaki D.', 'Hirata Y.', 'Nishiwaki K.', 'Kono N.', 'Arai H.', 'Miki H.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '27564576'}, {'database': 'DOI', 'id': '10.1371/journal.pgen.1006276'}], 'title': 'Mg2+ extrusion from intestinal epithelia by CNNM proteins is essential for gonadogenesis via AMPK-TORC1 signaling in Caenorhabditis elegans.', 'publicationDate': '2016', 'journal': 'PLoS Genet.', 'firstPage': 'E1006276', 'lastPage': 'E1006276', 'volume': '12'}, 'referencePositions': ['FUNCTION', 'SUBCELLULAR LOCATION', 'TISSUE SPECIFICITY', 'DISRUPTION PHENOTYPE'], 'evidences': [{'evidenceCode': 'ECO:0000305'}]}], secondaryAccessions=['A0A131MBV5', 'A0A131MD56', 'Q21469'], sequence={'value': 'MSKTPWALGLLIFLLTFTSPLSSSPVRSTDNSTSSKGLLNVNSSVILEPSILPSSASKPESLHLSKVRVSGLRLEAHASSTENIVLGHNKKHNVVVVPNKNVRVVLFGQNFQDIGALTFTADGSCKDLAHFFEADFSSMTPIRVVVEMSFPKTTESKDSFKLCVSEKFYANPQFVIVEDPFTMVTTEIPPVDEYMPKWLSWICLLILLCFSGLFSGLNLGLMTLSPYELQLYIASGTEQEKRDAGRILPIRKKGNQLLCTLLIGNVVVNVGVSLLMDQLVGSGFAVLVAATSCIVVFGEIIPQALCVKLGLPIGARTIPITQVLLFLMYPLTWPISKVLDIFLKEELTRSLERNKLVEMLKLSEKSIIGGQSDEFKMVLGALELYDKTVAHAMTRYEDIFMLPHTLTLGAGMVTQILDMGYTRIPIYENDRKNIVALLFVKDLALLDPDDNHNVMKIASIYNHEVRRVLVDMPLRNMLEEFKRGEYHMALVERLVEQEDKDPIYELCGLITLEDIIEEIIQCEIIDETDAVCDNVHRKKRQRKRNHDMSQIVNTAHAKCAINIQMLAVTIQVMSTCHKIFSSNYILPTILEKLIRKNCKKVETTQFSCLKEVGVVQPKPAVLFTKGEFSNKFIMILSGRAVVTIGKEEMRLEAGAWHSFGTEVLDAMAEAIERSLNQSTSRSTVSLNTEITNNSIGFIPDFDTVILYECVFCEITAADLLLAYNSSQIMQNNTKMQVVRSNSRISLIEEIPKDAVSTPIRNGSVKLRTVSEGETVHLLPKNMECHFNKQEKYEEEEE', 'length': 797, 'molWeight': 89288, 'crc64': '25AD3EA350EB4E85', 'md5': 'CDB9455358FB9B5496A1D047EF2A79FD'}, uniProtKBCrossReferences=[{'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CCD66487.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CZR14596.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CZR14597.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CZR14608.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'PIR', 'id': 'T16631', 'properties': [{'key': 'EntryName', 'value': 'T16631'}]}, {'database': 'RefSeq', 'id': 'NP_001309670.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_001322613.1'}], 'isoformId': 'A0A131MCZ8-1'}, {'database': 'RefSeq', 'id': 'NP_001309671.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_001322614.1'}]}, {'database': 'RefSeq', 'id': 'NP_001309682.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_001322615.1'}]}, {'database': 'RefSeq', 'id': 'NP_508520.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_076119.3'}]}, {'database': 'AlphaFoldDB', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'SMR', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'IntAct', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Interactions', 'value': '1'}]}, {'database': 'STRING', 'id': '6239.C33D12.2', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'PaxDb', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2a.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2a.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-1'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.2', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.2'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.3', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.3'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.4', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.4'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.5', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.5'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.6', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.6'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2c.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2c.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-3'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2d.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2d.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-4'}, {'database': 'GeneID', 'id': '180591', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'KEGG', 'id': 'cel:CELE_C33D12.2', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'UCSC', 'id': 'M02F4.3', 'properties': [{'key': 'OrganismName', 'value': 'c. elegans'}]}, {'database': 'CTD', 'id': '180591', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'WormBase', 'id': 'C33D12.2a', 'properties': [{'key': 'ProteinId', 'value': 'CE51453'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-1'}, {'database': 'WormBase', 'id': 'C33D12.2b', 'properties': [{'key': 'ProteinId', 'value': 'CE04764'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'WormBase', 'id': 'C33D12.2c', 'properties': [{'key': 'ProteinId', 'value': 'CE51420'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-3'}, {'database': 'WormBase', 'id': 'C33D12.2d', 'properties': [{'key': 'ProteinId', 'value': 'CE51314'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-4'}, {'database': 'eggNOG', 'id': 'KOG2118', 'properties': [{'key': 'ToxonomicScope', 'value': 'Eukaryota'}]}, {'database': 'GeneTree', 'id': 'ENSGT00940000169533', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'OMA', 'id': 'MSTCHKI', 'properties': [{'key': 'Fingerprint', 'value': '-'}]}, {'database': 'OrthoDB', 'id': '1446644at2759', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'PRO', 'id': 'PR:A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'Proteomes', 'id': 'UP000001940', 'properties': [{'key': 'Component', 'value': 'Chromosome X'}]}, {'database': 'Bgee', 'id': 'WBGene00016343', 'properties': [{'key': 'ExpressionPatterns', 'value': 'Expressed in larva and 3 other tissues'}]}, {'database': 'GO', 'id': 'GO:0016323', 'properties': [{'key': 'GoTerm', 'value': 'C:basolateral plasma membrane'}, {'key': 'GoEvidenceType', 'value': 'IDA:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000314', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0016021', 'properties': [{'key': 'GoTerm', 'value': 'C:integral component of membrane'}, {'key': 'GoEvidenceType', 'value': 'IEA:UniProtKB-KW'}]}, {'database': 'GO', 'id': 'GO:0043231', 'properties': [{'key': 'GoTerm', 'value': 'C:intracellular membrane-bounded organelle'}, {'key': 'GoEvidenceType', 'value': 'IBA:GO_Central'}], 'evidences': [{'evidenceCode': 'ECO:0000318', 'source': 'PubMed', 'id': '21873635'}]}, {'database': 'GO', 'id': 'GO:0005886', 'properties': [{'key': 'GoTerm', 'value': 'C:plasma membrane'}, {'key': 'GoEvidenceType', 'value': 'IBA:GO_Central'}], 'evidences': [{'evidenceCode': 'ECO:0000318', 'source': 'PubMed', 'id': '21873635'}]}, {'database': 'GO', 'id': 'GO:0022857', 'properties': [{'key': 'GoTerm', 'value': 'F:transmembrane transporter activity'}, {'key': 'GoEvidenceType', 'value': 'IBA:GO_Central'}], 'evidences': [{'evidenceCode': 'ECO:0000318', 'source': 'PubMed', 'id': '21873635'}]}, {'database': 'GO', 'id': 'GO:0008340', 'properties': [{'key': 'GoTerm', 'value': 'P:determination of adult lifespan'}, {'key': 'GoEvidenceType', 'value': 'IMP:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000315', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0045087', 'properties': [{'key': 'GoTerm', 'value': 'P:innate immune response'}, {'key': 'GoEvidenceType', 'value': 'HEP:WormBase'}], 'evidences': [{'evidenceCode': 'ECO:0007007', 'source': 'PubMed', 'id': '16968778'}]}, {'database': 'GO', 'id': 'GO:0010960', 'properties': [{'key': 'GoTerm', 'value': 'P:magnesium ion homeostasis'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0015693', 'properties': [{'key': 'GoTerm', 'value': 'P:magnesium ion transport'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:1905941', 'properties': [{'key': 'GoTerm', 'value': 'P:positive regulation of gonad development'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}, {'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}, {'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0040018', 'properties': [{'key': 'GoTerm', 'value': 'P:positive regulation of multicellular organism growth'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0040026', 'properties': [{'key': 'GoTerm', 'value': 'P:positive regulation of vulval development'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0032026', 'properties': [{'key': 'GoTerm', 'value': 'P:response to magnesium ion'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'CDD', 'id': 'cd04590', 'properties': [{'key': 'EntryName', 'value': 'CBS_pair_CorC_HlyC_assoc'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'Gene3D', 'id': '3.10.580.10', 'properties': [{'key': 'EntryName', 'value': '-'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'InterPro', 'id': 'IPR045095', 'properties': [{'key': 'EntryName', 'value': 'ACDP'}]}, {'database': 'InterPro', 'id': 'IPR000644', 'properties': [{'key': 'EntryName', 'value': 'CBS_dom'}]}, {'database': 'InterPro', 'id': 'IPR046342', 'properties': [{'key': 'EntryName', 'value': 'CBS_dom_sf'}]}, {'database': 'InterPro', 'id': 'IPR002550', 'properties': [{'key': 'EntryName', 'value': 'CNNM'}]}, {'database': 'InterPro', 'id': 'IPR044751', 'properties': [{'key': 'EntryName', 'value': 'Ion_transp-like_CBS'}]}, {'database': 'PANTHER', 'id': 'PTHR12064', 'properties': [{'key': 'EntryName', 'value': 'PTHR12064'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'Pfam', 'id': 'PF01595', 'properties': [{'key': 'EntryName', 'value': 'DUF21'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'SUPFAM', 'id': 'SSF54631', 'properties': [{'key': 'EntryName', 'value': 'SSF54631'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'PROSITE', 'id': 'PS51371', 'properties': [{'key': 'EntryName', 'value': 'CBS'}, {'key': 'MatchStatus', 'value': '2'}]}, {'database': 'PROSITE', 'id': 'PS51846', 'properties': [{'key': 'EntryName', 'value': 'CNNM'}, {'key': 'MatchStatus', 'value': '1'}]}], uniProtkbId='CNNM3_CAEEL'))" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "uresources[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -676,7 +573,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -685,7 +582,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -696,17 +593,37 @@ " up:reviewed ?v1 . \n", " FILTER(?v1 = 'true'^^xsd:boolean)\n", "}\n", + "Submitted query:\n", + " PREFIX up: \n", + " PREFIX owl: \n", + " PREFIX owl2xml: \n", + " PREFIX swrlb: \n", + " PREFIX protege: \n", + " PREFIX swrl: \n", + " PREFIX xsd: \n", + " PREFIX skos: \n", + " PREFIX rdfs: \n", + " PREFIX dc11: \n", + " PREFIX rdf: \n", + " PREFIX foaf: \n", + " SELECT ?id WHERE {?id rdf:type up:Protein;\n", + " up:reviewed ?v1 . \n", + " FILTER(?v1 = 'true'^^xsd:boolean)\n", + " } LIMIT 10\n", + "\n", "amount of results = 10\n" ] } ], "source": [ - "proteins = forge.search({'type': 'Protein', 'up:reviewed': True}, db_source='UniProt', limit=10)" + "\n", + "\n", + "proteins = forge.search({'type': 'Protein', 'up:reviewed': True}, db_source='UniProt', limit=10, debug=True)" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -715,7 +632,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -746,7 +663,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -766,7 +683,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -775,7 +692,7 @@ "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id=Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/H5SQ95#gene-MD58301D33AF640374C84A4DA4CAF383BE6', _inner_sync=False, annotationScore=2.0, comments=[{'texts': [{'evidences': [{'evidenceCode': 'ECO:0000305', 'source': 'PubMed', 'id': '28087277'}], 'value': 'May function as a protein modifier covalently attached to lysine residues of substrate proteins. This may serve to target the modified proteins for degradation by proteasomes'}], 'commentType': 'FUNCTION'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'HAMAP-Rule', 'id': 'MF_02133'}], 'value': 'Belongs to the ubiquitin-like protein UBact family'}], 'commentType': 'SIMILARITY'}], entryAudit={'firstPublicDate': '2017-10-25', 'lastAnnotationUpdateDate': '2022-05-25', 'lastSequenceUpdateDate': '2012-04-18', 'entryVersion': 15, 'sequenceVersion': 1}, entryType='UniProtKB reviewed (Swiss-Prot)', extraAttributes={'countByCommentType': {'FUNCTION': 1, 'SIMILARITY': 1}, 'countByFeatureType': {'Chain': 1, 'Region': 1, 'Compositional bias': 1, 'Cross-link': 1}, 'uniParcId': 'UPI00024DBA4F'}, features=[{'type': 'Chain', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Prokaryotic ubiquitin-like protein UBact', 'featureId': 'PRO_0000441772'}, {'type': 'Region', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Disordered', 'evidences': [{'evidenceCode': 'ECO:0000256', 'source': 'SAM', 'id': 'MobiDB-lite'}]}, {'type': 'Compositional bias', 'location': {'start': {'value': 26, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Basic and acidic residues', 'evidences': [{'evidenceCode': 'ECO:0000256', 'source': 'SAM', 'id': 'MobiDB-lite'}]}, {'type': 'Cross-link', 'location': {'start': {'value': 56, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Isoglutamyl lysine isopeptide (Glu-Lys) (interchain with K-? in acceptor proteins)', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}], genes=[{'geneName': {'evidences': [{'evidenceCode': 'ECO:0000303', 'source': 'PubMed', 'id': '28087277'}], 'value': 'ubact'}, 'orfNames': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'EMBL', 'id': 'BAL58331.1'}], 'value': 'HGMM_OP1C026'}]}], keywords=[{'id': 'KW-1017', 'category': 'PTM', 'name': 'Isopeptide bond'}, {'id': 'KW-0833', 'category': 'Biological process', 'name': 'Ubl conjugation pathway'}], organism={'scientificName': 'Acetothermus autotrophicum', 'taxonId': 1446466, 'lineage': ['Bacteria', 'Candidatus Bipolaricaulota', 'Candidatus Acetothermum']}, primaryAccession='H5SQ95', proteinDescription={'recommendedName': {'fullName': {'evidences': [{'evidenceCode': 'ECO:0000303', 'source': 'PubMed', 'id': '28087277'}], 'value': 'Prokaryotic ubiquitin-like protein UBact'}}}, proteinExistence='3: Inferred from homology', references=[{'citation': {'id': '22303444', 'citationType': 'journal article', 'authors': ['Takami H.', 'Noguchi H.', 'Takaki Y.', 'Uchiyama I.', 'Toyoda A.', 'Nishi S.', 'Chee G.J.', 'Arai W.', 'Nunoura T.', 'Itoh T.', 'Hattori M.', 'Takai K.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '22303444'}, {'database': 'DOI', 'id': '10.1371/journal.pone.0030559'}], 'title': 'A deeply branching thermophilic bacterium with an ancient acetyl-CoA pathway dominates a subsurface ecosystem.', 'publicationDate': '2012', 'journal': 'PLoS ONE', 'firstPage': 'E30559', 'lastPage': 'E30559', 'volume': '7'}, 'referencePositions': ['NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]']}, {'citation': {'id': '28087277', 'citationType': 'journal article', 'authors': ['Lehmann G.', 'Udasin R.G.', 'Livneh I.', 'Ciechanover A.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '28087277'}, {'database': 'DOI', 'id': '10.1016/j.bbrc.2017.01.037'}], 'title': 'Identification of UBact, a ubiquitin-like protein, along with other homologous components of a conjugation system and the proteasome in different gram-negative bacteria.', 'publicationDate': '2017', 'journal': 'Biochem. Biophys. Res. Commun.', 'firstPage': '946', 'lastPage': '950', 'volume': '483'}, 'referencePositions': ['PREDICTED FUNCTION']}], sequence={'value': 'MPERIVKPMPQDPVTKPGDEGPRTPNVPKPDTERLLERMRRVDPRQAQRYRQRSGE', 'length': 56, 'molWeight': 6594, 'crc64': '323CA0429FBA02D0', 'md5': 'EF137CE5E7D3D2D873E21B0ECFB25FF6'}, uniProtKBCrossReferences=[{'database': 'EMBL', 'id': 'AP011800', 'properties': [{'key': 'ProteinId', 'value': 'BAL58331.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'AlphaFoldDB', 'id': 'H5SQ95', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'SMR', 'id': 'H5SQ95', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'GO', 'id': 'GO:0031386', 'properties': [{'key': 'GoTerm', 'value': 'F:protein tag'}, {'key': 'GoEvidenceType', 'value': 'IEA:UniProtKB-UniRule'}]}, {'database': 'HAMAP', 'id': 'MF_02133', 'properties': [{'key': 'EntryName', 'value': 'UBact'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'InterPro', 'id': 'IPR037543', 'properties': [{'key': 'EntryName', 'value': 'UBact'}]}], uniProtkbId='UBACT_ACEAU'), _inner_sync=False)" ] }, - "execution_count": 39, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -793,7 +710,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -816,7 +733,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -833,7 +750,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -861,7 +778,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -891,20 +808,14 @@ " contribution.type\n", " contribution.agent.id\n", " contribution.agent.type\n", + " contribution.agent.label\n", " distribution.contentUrl\n", " distribution.encodingFormat\n", " distribution.name\n", " name\n", - " ...\n", - " subject.strain\n", - " brainLocation.layer\n", - " contribution.agent.label\n", - " subject.age.period\n", - " subject.age.unitCode\n", - " subject.age.value\n", - " subject.identifier\n", - " subject.name\n", - " subject.sex.label\n", + " subject.type\n", + " subject.species.id\n", + " subject.species.label\n", " subject.strain.label\n", " \n", " \n", @@ -913,245 +824,184 @@ " 0\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Simple lobule\n", + " Primary motor area Layer 5\n", " Contribution\n", " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", + " Janelia Research Campus\n", " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " AA1015.swc\n", - " AA1015\n", - " ...\n", + " AA1050.swc\n", + " AA1050\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", " Sim1-Cre\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", " \n", " \n", " 1\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Ansiform lobule\n", + " Primary somatosensory area mouth layer 5\n", " Contribution\n", " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", + " Janelia Research Campus\n", " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " AA1024.swc\n", - " AA1024\n", - " ...\n", + " AA1049.swc\n", + " AA1049\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", " Sim1-Cre\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", " \n", " \n", " 2\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " MTG\n", + " Retrosplenial area ventral part layer 5\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " H17.03.010.11.13.01\n", - " ...\n", - " NaN\n", - " 3\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " yrs\n", - " 38\n", - " 601901227.0\n", - " H17.03.010\n", - " Female\n", - " \n", + " AA1045.swc\n", + " AA1045\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 3\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " FroL\n", + " Parafascicular nucleus\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " H16.06.007.01.07.02\n", - " ...\n", - " NaN\n", - " 3\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " yrs\n", - " 26\n", - " 518229880.0\n", - " H16.06.007\n", - " Male\n", - " \n", + " AA1046.swc\n", + " AA1046\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 4\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " MTG\n", + " Medial mammillary nucleus\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " H16.06.008.01.26.04\n", - " ...\n", - " NaN\n", - " 5\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " yrs\n", - " 24\n", - " 527747035.0\n", - " H16.06.008\n", - " Female\n", - " \n", + " AA1048.swc\n", + " AA1048\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 5\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", - " http://api.brain-map.org/api/v2/data/Structure/74\n", - " VISl6a\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " Primary somatosensory area mouth layer 5\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " Rorb-IRES2-Cre-D;Ai14-234246.02.02.01\n", - " ...\n", - " NaN\n", - " 6a\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " \n", - " \n", - " 503871576.0\n", - " Rorb-IRES2-Cre-D;Ai14-234246\n", - " \n", - " Rorb-IRES2-Cre\n", + " AA1051.swc\n", + " AA1051\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 6\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISp2/3\n", + " Entorhinal area lateral part\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " Chrna2-Cre_OE25;Ai14(IVSCC)-294465.05.01.01\n", - " ...\n", - " NaN\n", - " 2/3\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " \n", - " \n", - " 565087402.0\n", - " Chrna2-Cre_OE25;Ai14(IVSCC)-294465\n", - " \n", - " Chrna2-Cre_OE25\n", + " AA1047.swc\n", + " AA1047\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 7\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " VISp5\n", + " Primary motor area Layer 6a\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " Cux2-CreERT2;Ai14-205530.03.02.01\n", - " ...\n", - " NaN\n", - " 5\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " \n", - " \n", - " 485250100.0\n", - " Cux2-CreERT2;Ai14-205530\n", - " \n", - " Cux2-CreERT2\n", + " AA1043.swc\n", + " AA1043\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 8\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", - " http://api.brain-map.org/api/v2/data/Structure/33\n", - " VISp6a\n", + " http://api.brain-map.org/api/v2/data/Structure...\n", + " Medial mammillary nucleus\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " Ctgf-2A-dgCre;Ai14(IVSCC)-229233.03.02.01\n", - " ...\n", - " NaN\n", - " 6a\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " \n", - " \n", - " 501715368.0\n", - " Ctgf-2A-dgCre;Ai14(IVSCC)-229233\n", - " \n", - " Ctgf-T2A-dgCre\n", + " AA1041.swc\n", + " AA1041\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", " 9\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " MTG\n", + " Medial mammillary nucleus\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.417881.3\n", + " https://www.grid.ac/institutes/grid.443970.d\n", " Organization\n", - " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", + " Janelia Research Campus\n", + " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", " application/swc\n", - " reconstruction.swc\n", - " H17.06.005.12.15.01\n", - " ...\n", - " NaN\n", - " 4\n", - " Allen Institute for Brain Science\n", - " Post-natal\n", - " yrs\n", - " 38\n", - " 571364629.0\n", - " H17.06.005\n", - " Male\n", - " \n", + " AA1039.swc\n", + " AA1039\n", + " Subject\n", + " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", + " Mus musculus\n", + " Sim1-Cre\n", " \n", " \n", "\n", - "

10 rows × 23 columns

\n", "" ], "text/plain": [ @@ -1173,112 +1023,86 @@ "2 http://api.brain-map.org/api/v2/data/Structure... \n", "3 http://api.brain-map.org/api/v2/data/Structure... \n", "4 http://api.brain-map.org/api/v2/data/Structure... \n", - "5 http://api.brain-map.org/api/v2/data/Structure/74 \n", + "5 http://api.brain-map.org/api/v2/data/Structure... \n", "6 http://api.brain-map.org/api/v2/data/Structure... \n", "7 http://api.brain-map.org/api/v2/data/Structure... \n", - "8 http://api.brain-map.org/api/v2/data/Structure/33 \n", + "8 http://api.brain-map.org/api/v2/data/Structure... \n", "9 http://api.brain-map.org/api/v2/data/Structure... \n", "\n", - " brainLocation.brainRegion.label contribution.type \\\n", - "0 Simple lobule Contribution \n", - "1 Ansiform lobule Contribution \n", - "2 MTG Contribution \n", - "3 FroL Contribution \n", - "4 MTG Contribution \n", - "5 VISl6a Contribution \n", - "6 VISp2/3 Contribution \n", - "7 VISp5 Contribution \n", - "8 VISp6a Contribution \n", - "9 MTG Contribution \n", + " brainLocation.brainRegion.label contribution.type \\\n", + "0 Primary motor area Layer 5 Contribution \n", + "1 Primary somatosensory area mouth layer 5 Contribution \n", + "2 Retrosplenial area ventral part layer 5 Contribution \n", + "3 Parafascicular nucleus Contribution \n", + "4 Medial mammillary nucleus Contribution \n", + "5 Primary somatosensory area mouth layer 5 Contribution \n", + "6 Entorhinal area lateral part Contribution \n", + "7 Primary motor area Layer 6a Contribution \n", + "8 Medial mammillary nucleus Contribution \n", + "9 Medial mammillary nucleus Contribution \n", "\n", " contribution.agent.id contribution.agent.type \\\n", "0 https://www.grid.ac/institutes/grid.443970.d Organization \n", "1 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "2 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "3 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "4 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "5 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "6 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "7 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "8 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "9 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "2 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "3 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "4 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "5 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "6 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "7 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "8 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "9 https://www.grid.ac/institutes/grid.443970.d Organization \n", "\n", - " distribution.contentUrl \\\n", - "0 https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "1 https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "2 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "3 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "4 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "5 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "6 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "7 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "8 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "9 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + " contribution.agent.label distribution.contentUrl \\\n", + "0 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "1 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "2 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "3 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "4 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "5 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "6 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "7 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "8 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + "9 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", "\n", - " distribution.encodingFormat distribution.name \\\n", - "0 application/swc AA1015.swc \n", - "1 application/swc AA1024.swc \n", - "2 application/swc reconstruction.swc \n", - "3 application/swc reconstruction.swc \n", - "4 application/swc reconstruction.swc \n", - "5 application/swc reconstruction.swc \n", - "6 application/swc reconstruction.swc \n", - "7 application/swc reconstruction.swc \n", - "8 application/swc reconstruction.swc \n", - "9 application/swc reconstruction.swc \n", + " distribution.encodingFormat distribution.name name subject.type \\\n", + "0 application/swc AA1050.swc AA1050 Subject \n", + "1 application/swc AA1049.swc AA1049 Subject \n", + "2 application/swc AA1045.swc AA1045 Subject \n", + "3 application/swc AA1046.swc AA1046 Subject \n", + "4 application/swc AA1048.swc AA1048 Subject \n", + "5 application/swc AA1051.swc AA1051 Subject \n", + "6 application/swc AA1047.swc AA1047 Subject \n", + "7 application/swc AA1043.swc AA1043 Subject \n", + "8 application/swc AA1041.swc AA1041 Subject \n", + "9 application/swc AA1039.swc AA1039 Subject \n", "\n", - " name ... subject.strain \\\n", - "0 AA1015 ... Sim1-Cre \n", - "1 AA1024 ... Sim1-Cre \n", - "2 H17.03.010.11.13.01 ... NaN \n", - "3 H16.06.007.01.07.02 ... NaN \n", - "4 H16.06.008.01.26.04 ... NaN \n", - "5 Rorb-IRES2-Cre-D;Ai14-234246.02.02.01 ... NaN \n", - "6 Chrna2-Cre_OE25;Ai14(IVSCC)-294465.05.01.01 ... NaN \n", - "7 Cux2-CreERT2;Ai14-205530.03.02.01 ... NaN \n", - "8 Ctgf-2A-dgCre;Ai14(IVSCC)-229233.03.02.01 ... NaN \n", - "9 H17.06.005.12.15.01 ... NaN \n", + " subject.species.id subject.species.label \\\n", + "0 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "1 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "2 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "3 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "4 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "5 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "6 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "7 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "8 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + "9 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", "\n", - " brainLocation.layer contribution.agent.label subject.age.period \\\n", - "0 NaN NaN NaN \n", - "1 NaN NaN NaN \n", - "2 3 Allen Institute for Brain Science Post-natal \n", - "3 3 Allen Institute for Brain Science Post-natal \n", - "4 5 Allen Institute for Brain Science Post-natal \n", - "5 6a Allen Institute for Brain Science Post-natal \n", - "6 2/3 Allen Institute for Brain Science Post-natal \n", - "7 5 Allen Institute for Brain Science Post-natal \n", - "8 6a Allen Institute for Brain Science Post-natal \n", - "9 4 Allen Institute for Brain Science Post-natal \n", - "\n", - " subject.age.unitCode subject.age.value subject.identifier \\\n", - "0 NaN NaN NaN \n", - "1 NaN NaN NaN \n", - "2 yrs 38 601901227.0 \n", - "3 yrs 26 518229880.0 \n", - "4 yrs 24 527747035.0 \n", - "5 503871576.0 \n", - "6 565087402.0 \n", - "7 485250100.0 \n", - "8 501715368.0 \n", - "9 yrs 38 571364629.0 \n", - "\n", - " subject.name subject.sex.label subject.strain.label \n", - "0 NaN NaN NaN \n", - "1 NaN NaN NaN \n", - "2 H17.03.010 Female \n", - "3 H16.06.007 Male \n", - "4 H16.06.008 Female \n", - "5 Rorb-IRES2-Cre-D;Ai14-234246 Rorb-IRES2-Cre \n", - "6 Chrna2-Cre_OE25;Ai14(IVSCC)-294465 Chrna2-Cre_OE25 \n", - "7 Cux2-CreERT2;Ai14-205530 Cux2-CreERT2 \n", - "8 Ctgf-2A-dgCre;Ai14(IVSCC)-229233 Ctgf-T2A-dgCre \n", - "9 H17.06.005 Male \n", - "\n", - "[10 rows x 23 columns]" + " subject.strain.label \n", + "0 Sim1-Cre \n", + "1 Sim1-Cre \n", + "2 Sim1-Cre \n", + "3 Sim1-Cre \n", + "4 Sim1-Cre \n", + "5 Sim1-Cre \n", + "6 Sim1-Cre \n", + "7 Sim1-Cre \n", + "8 Sim1-Cre \n", + "9 Sim1-Cre " ] }, - "execution_count": 34, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -1299,15 +1123,15 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " send\n", - " ConnectionError: HTTPSConnectionPool(host='staging.nexus.ocp.bbp.epfl.ch', port=443): Max retries exceeded with url: /v1/files/dke/kgforge/f0c59cf7-3b1b-4cb9-b81b-62fefd5bbd0f (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))\n", + " _download\n", + " DownloadingError: Downloading from neurosciencegraph/datamodels:404, message='Not Found', url=URL('https://staging.nise.bbp.epfl.ch/nexus/v1/files/neurosciencegraph/datamodels/665642bf-4d60-45be-853b-ace965f0057f')\n", "\n" ] } @@ -1326,7 +1150,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1359,7 +1183,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1383,6 +1207,156 @@ "forge.sparql(mquery, debug=True, rewrite=False)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "example_region = forge.retrieve(\"http://api.brain-map.org/api/v2/data/Structure/614454282\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"\"\"\n", + "SELECT ?id ?label ?preflabel\n", + "WHERE {\n", + " ?id subClassOf \"nsg:BrainRegion\" .\n", + "}\n", + " \"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted query:\n", + " PREFIX bmc: \n", + " PREFIX bmo: \n", + " PREFIX commonshapes: \n", + " PREFIX datashapes: \n", + " PREFIX dc: \n", + " PREFIX dcat: \n", + " PREFIX dcterms: \n", + " PREFIX mba: \n", + " PREFIX nsg: \n", + " PREFIX nxv: \n", + " PREFIX oa: \n", + " PREFIX obo: \n", + " PREFIX owl: \n", + " PREFIX prov: \n", + " PREFIX rdf: \n", + " PREFIX rdfs: \n", + " PREFIX schema: \n", + " PREFIX sh: \n", + " PREFIX shsh: \n", + " PREFIX skos: \n", + " PREFIX vann: \n", + " PREFIX void: \n", + " PREFIX xml: \n", + " PREFIX xsd: \n", + " PREFIX : \n", + " \n", + " SELECT ?id ?label ?preflabel\n", + " WHERE {\n", + " ?id rdfs:subClassOf \"nsg:BrainRegion\" .\n", + " }\n", + " LIMIT 100\n", + "\n" + ] + } + ], + "source": [ + "bregions = forge.sparql(query, limit=100, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 67, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bregions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'context': 'https://neuroshapes.org',\n", + " 'id': 'http://api.brain-map.org/api/v2/data/Structure/614454282',\n", + " 'type': 'Class',\n", + " 'atlas_id': 966,\n", + " 'color_hex_triplet': '1F9D5A',\n", + " 'graph_order': 20,\n", + " 'hemisphere_id': 3,\n", + " 'identifier': '614454282',\n", + " 'isDefinedBy': 'http://bbp.epfl.ch/neurosciencegraph/ontologies/core/brainregion',\n", + " 'isPartOf': 'mba:943',\n", + " 'notation': 'MOp2',\n", + " 'rdfs:label': 'Primary motor area, Layer 2',\n", + " 'skos:prefLabel': 'Primary motor area, Layer 2',\n", + " 'st_level': 11,\n", + " 'subClassOf': ['nsg:BrainRegion'],\n", + " '_last_action': Action(error=None, message=None, operation='retrieve', succeeded=True),\n", + " '_validated': False,\n", + " '_inner_sync': True,\n", + " '_synchronized': True,\n", + " '_store_metadata': {'id': 'mba:614454282',\n", + " '_constrainedBy': 'https://neuroshapes.org/dash/ontologyentity',\n", + " '_createdAt': '2022-05-27T07:24:49.109Z',\n", + " '_createdBy': 'https://staging.nise.bbp.epfl.ch/nexus/v1/realms/serviceaccounts/users/service-account-brain-modeling-ontology-ci-cd',\n", + " '_deprecated': False,\n", + " '_incoming': 'https://staging.nise.bbp.epfl.ch/nexus/v1/resources/neurosciencegraph/datamodels/datashapes:ontologyentity/http:%2F%2Fapi.brain-map.org%2Fapi%2Fv2%2Fdata%2FStructure%2F614454282/incoming',\n", + " '_outgoing': 'https://staging.nise.bbp.epfl.ch/nexus/v1/resources/neurosciencegraph/datamodels/datashapes:ontologyentity/http:%2F%2Fapi.brain-map.org%2Fapi%2Fv2%2Fdata%2FStructure%2F614454282/outgoing',\n", + " '_project': 'https://staging.nise.bbp.epfl.ch/nexus/v1/projects/neurosciencegraph/datamodels',\n", + " '_rev': 8,\n", + " '_schemaProject': 'https://staging.nise.bbp.epfl.ch/nexus/v1/projects/neurosciencegraph/datamodels',\n", + " '_self': 'https://staging.nise.bbp.epfl.ch/nexus/v1/resources/neurosciencegraph/datamodels/datashapes:ontologyentity/http:%2F%2Fapi.brain-map.org%2Fapi%2Fv2%2Fdata%2FStructure%2F614454282',\n", + " '_updatedAt': '2022-06-17T22:48:31.217Z',\n", + " '_updatedBy': 'https://staging.nise.bbp.epfl.ch/nexus/v1/realms/serviceaccounts/users/service-account-nexus-sa'}}" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "example_region.__dict__" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/kgforge/core/archetypes/__init__.py b/kgforge/core/archetypes/__init__.py index abda5bcd..7ba09666 100644 --- a/kgforge/core/archetypes/__init__.py +++ b/kgforge/core/archetypes/__init__.py @@ -19,3 +19,4 @@ from .store import Store from .model import Model from .resolver import Resolver +from .database import Database diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index 0af43410..bdd5a3ff 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -15,7 +15,9 @@ import re import time +from pyld import jsonld from abc import ABC, abstractmethod +from rdflib import Graph from pathlib import Path from typing import Any, Callable, Dict, List, Match, Optional, Tuple, Union @@ -34,6 +36,7 @@ QueryingError, ) from kgforge.core.commons.execution import not_supported, run +from kgforge.core.conversions.rdf import from_jsonld from kgforge.core.reshaping import collect_values @@ -595,3 +598,39 @@ def replace(match: Match) -> str: if context.has_vocab(): pfx = "\n".join([pfx, f"PREFIX : <{context.vocab}>"]) return f"{pfx}\n{qr}" + + +def build_construct_query(data, context): + subject_triples = {} + for r in data["results"]["bindings"]: + subject = r["subject"]["value"] + s = f"<{r['subject']['value']}>" + p = f"<{r['predicate']['value']}>" + if r["object"]["type"] == "uri": + o = f"<{r['object']['value']}>" + else: + if "datatype" in r["object"]: + o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" + else: + o = f"\"{r['object']['value']}\"" + if subject in subject_triples: + subject_triples[subject] += f"\n{s} {p} {o} . " + else: + subject_triples[subject] = f"{s} {p} {o} . " + + def triples_to_resource(iri, triples): + graph = Graph().parse(data=triples, format="nt") + data_expanded = json.loads(graph.serialize(format="json-ld")) + data_expanded = json.loads(graph.serialize(format="json-ld")) + frame = {"@id": iri} + data_framed = jsonld.frame(data_expanded, frame) + compacted = jsonld.compact(data_framed, context.document) + resource = from_jsonld(compacted) + resource.context = ( + context.iri + if context.is_http_iri() + else context.document["@context"] + ) + return resource + + return [triples_to_resource(s, t) for s, t in subject_triples.items()] \ No newline at end of file diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index fa21e38e..819fe989 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -25,12 +25,12 @@ from urllib.parse import quote_plus, urlparse from kgforge.core import Resource -from kgforge.core.archetypes import Mapping, Model, Resolver, Store from kgforge.core.commons.context import Context +from kgforge.core.archetypes import Mapping, Model, Resolver, Store, Database from kgforge.core.commons.actions import LazyAction from kgforge.core.commons.dictionaries import with_defaults from kgforge.core.commons.exceptions import ResolvingError -from kgforge.core.commons.execution import catch +from kgforge.core.commons.execution import catch, not_supported from kgforge.core.commons.imports import import_class from kgforge.core.commons.strategies import ResolvingStrategy from kgforge.core.commons.formatter import Formatter @@ -47,7 +47,7 @@ from kgforge.core.wrappings.paths import PathsWrapper, wrap_paths from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping -from kgforge.specializations.resources import DatabaseSource +from kgforge.specializations.databases import StoreDatabase class KnowledgeGraphForge: @@ -249,20 +249,6 @@ def __init__(self, configuration: Union[str, Dict], **kwargs) -> None: # Formatters. self._formatters: Optional[Dict[str, str]] = config.pop("Formatters", None) -<<<<<<< HEAD -======= - # Database sources. - db_sources_config: Optional[Dict[str, Dict[str, str]]] = config.pop("DatabaseSources", None) - - self._db_sources: Union[DatabaseSource, List[DatabaseSource]] = ( - self.create_db_sources(db_sources_config, store_config, model_config) - if db_sources_config - else None - ) - - # Modeling User Interface. - ->>>>>>> a48cc2b (Improved DatabaseSource class. Made possible to load mappings from local directory.) @catch def prefixes(self, pretty: bool = True) -> Optional[Dict[str, str]]: """ @@ -531,23 +517,23 @@ def sources(self, pretty: bool = True) -> Optional[List[str]]: return self._model.sources(pretty) @catch - def db_sources(self, with_datatype: Optional[List[str]] = None, + def db_sources(self, mappings: Optional[List[str]] = None, pretty: bool = False) -> Optional[List[str]]: """ Print(pretty=True) or return (pretty=False) configured data sources. :param pretty: a boolean :return: Optional[List[str]] """ - if with_datatype is not None: + if mappings is not None: sources = {} - if isinstance(with_datatype, list): - for type in with_datatype: + if isinstance(mappings, list): + for type in mappings: for db in self._db_sources: if type in self._db_sources[db].datatypes(): sources[db] = self._db_sources[db] else: for db in self._db_sources: - if with_datatype in self._db_sources[db].datatypes(): + if mappings in self._db_sources[db].datatypes(): sources[db] = self._db_sources[db] if not sources: print("No Database sources were found for the given datatype(s)") @@ -558,7 +544,7 @@ def db_sources(self, with_datatype: Optional[List[str]] = None, else: return sources - def add_db_source(self, db_source: DatabaseSource) -> None: + def add_db_source(self, db_source: Database) -> None: """ Add a DatabaseSource to the KG. """ @@ -577,7 +563,7 @@ def mappings( :return: Optional[Dict[str, List[str]]] """ if source in self._db_sources: - return self._db_sources[source].mappings(pretty) + return self._db_sources[source].mappings() else: return self._model.mappings(source, pretty) @@ -686,7 +672,7 @@ def search(self, *filters, **params) -> List[Resource]: ) db_source = params.pop('db_source', None) if db_source: - return self._db_sources[db_source]._store.search(resolvers, *filters, **params) + return self._db_sources[db_source].search(resolvers, *filters, **params) else: return self._store.search(resolvers, *filters, **params) @@ -711,7 +697,7 @@ def sparql( """ db_source = params.pop('db_source', None) if db_source: - return self._db_sources[db_source]._store.sparql(query, debug, limit, offset, **params) + return self._db_sources[db_source].sparql(query, debug, limit, offset, **params) else: return self._store.sparql(query, debug, limit, offset, **params) @@ -734,7 +720,7 @@ def elastic( :return: List[Resource] """ if db_source: - return self._db_sources[db_source]._store.elastic(query, debug, limit, offset) + return self._db_sources[db_source].elastic(query, debug, limit, offset) else: return self._store.elastic(query, debug, limit, offset) @@ -1007,22 +993,29 @@ def get_model_context(self): def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], store_config : Optional[Dict[str, Dict[str, str]]], model_config : Optional[Dict[str, Dict[str, str]]] - ) -> Union[DatabaseSource, List[DatabaseSource]]: + ) -> Union[Database, List[Database]]: names = all_config.keys() dbs = {} for name in names: config = all_config[name] - # Provide store and model configuration to the database sources - if "model" not in config: - config.update(model=deepcopy(model_config)) - - # Complete configuration of the db store in case is the same - if config['store']['name'] == store_config['name']: - store_copy = deepcopy(store_config) - with_defaults(config['store'], store_copy, - "name", "name", - store_copy.keys()) - dbs[name] = DatabaseSource(self, name=name, from_forge=True, **config) + origin = config.get('origin') + if origin == 'store': + source = config.get('source') + # Provide store and model configuration to the database sources + if "model" not in config: + config.update(model=deepcopy(model_config)) + # Complete configuration of the db store in case is the same + if source == store_config['name']: + store_copy = deepcopy(store_config) + with_defaults(config, store_copy, + "source", "name", + store_copy.keys()) + config.update(origin=origin) + print('Configuration', config) + config['name'] = name + dbs[name] = StoreDatabase(self, **config) + else: + raise NotImplementedError(f'Database from {origin} is not yet implemented.') return dbs diff --git a/kgforge/specializations/databases/__init__.py b/kgforge/specializations/databases/__init__.py new file mode 100644 index 00000000..081f712c --- /dev/null +++ b/kgforge/specializations/databases/__init__.py @@ -0,0 +1,15 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +from .store_database import StoreDatabase \ No newline at end of file diff --git a/kgforge/specializations/databases/store_database.py b/kgforge/specializations/databases/store_database.py new file mode 100644 index 00000000..58227887 --- /dev/null +++ b/kgforge/specializations/databases/store_database.py @@ -0,0 +1,91 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . +import os +import json +from pathlib import Path, PurePath +from re import I +import copy +from typing import Callable, Optional, Union, Dict, List, Any + +from kgforge.core.archetypes import Mapping, Store, Database +from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.core.commons.execution import not_supported +from kgforge.core.commons.dictionaries import with_defaults +from kgforge.core.commons.imports import import_class + + +class StoreDatabase(Database): + """A high-level class to retrieve and create Database-related Resources.""" + + + _REQUIRED = ("name", "origin", "source", "model") + + def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Database", + **config) -> None: + """ + The properties defining the StoreDatabase are: + + name: - REQUIRED + type: Database - REQUIRED + origin: <'store'> - REQUIRED + source: - REQUIRED + bucket: + endpoint: + token: + source: + context: <'directory', 'url', or 'store'> + bucket: + iri: + """ + self._check_properties(**config) + self.name = config.pop('name') + self.type: str = type + self._forge: Optional["KnowledgeGraphForge"] = forge + source = config.pop('source') + super().__init__(source, **config) + + def _check_properties(self, **info): + properties = info.keys() + for r in self._REQUIRED: + if r not in properties: + raise ValueError(f'Missing {r} from the properties to define the DatabasSource') + + def datatypes(self): + # TODO: add other datatypes used, for instance, inside the mappings + return self.mappings().keys() + + def search(self, *filters, **params) -> Any: + self.service.search(*filters, **params) + + def sparql(self, query: str, debug: bool, limit: int = None, offset: int = None, **params) -> Any: + self.service.sparql(query, debug, limit, offset, **params) + + @staticmethod + def _service_from_directory(dirpath: Path, **source_config) -> Any: + not_supported() + + @staticmethod + def _service_from_web_service(endpoint: str, **source_config) -> Any: + not_supported() + + @staticmethod + def _service_from_store(store: Callable, **store_config) -> Store: + # Store. + print('store config', store_config) + return store(**store_config) + + def health(self) -> Callable: + not_supported() diff --git a/kgforge/specializations/resources/__init__.py b/kgforge/specializations/resources/__init__.py index 613c0151..46bdaea7 100644 --- a/kgforge/specializations/resources/__init__.py +++ b/kgforge/specializations/resources/__init__.py @@ -13,5 +13,4 @@ # along with Blue Brain Nexus Forge. If not, see . from .datasets import Dataset -from .db_sources import DatabaseSource from .entity_linking_candidate import EntityLinkingCandidate \ No newline at end of file diff --git a/kgforge/specializations/resources/db_sources.py b/kgforge/specializations/resources/db_sources.py deleted file mode 100644 index 6ddb49df..00000000 --- a/kgforge/specializations/resources/db_sources.py +++ /dev/null @@ -1,157 +0,0 @@ -# -# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Blue Brain Nexus Forge is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser -# General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Blue Brain Nexus Forge. If not, see . -import os -import json -from pathlib import Path, PurePath -from re import I -import copy -from typing import Callable, Optional, Union, Dict, List - -from kgforge.core import Resource -from kgforge.core.archetypes import Mapping, Store, Model -from kgforge.core.commons.execution import not_supported -from kgforge.core.commons.dictionaries import with_defaults -from kgforge.core.commons.imports import import_class - -# Get local paths -pathparts = list(Path(__file__).resolve().parts) -KGFORGE_PATH = os.path.join(*pathparts[:-4]) -DBS_PATH = Path(KGFORGE_PATH, "examples", "database_sources") - -class DatabaseSource(Resource): - """A high-level class for Database-related Resources.""" - - - _DBDIR = Path - _REQUIRED = ("name", "store", "model") - _RESERVED = {"_forge", "_from_forge", "_check_properties", "_save_config", "_dirpath", - "health", "_mappings", "mappings", "mapping", "dump_config", "_model", "._store"} | Resource._RESERVED - - def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Database", - from_forge: bool = True, **properties) -> None: - """ - The properties defining the Databasesource in YAML are: - - name: - REQUIRED - type: Database - REQUIRED - origin: <'directory', 'url', or 'store'> - REQUIRED - source: - REQUIRED - bucket: - endpoint: - token: - bucket: - endpoint: - token: - iri: - """ - self._check_properties(**properties) - super().__init__(**properties) - self.type: str = type - self._forge: Optional["KnowledgeGraphForge"] = forge - - - store_config = properties.pop('store') - - # Model - model_config = properties.pop("model") - # Assume that the model is a store - # TODO: get configuration from forge._store when using BlueBrainNexus - if model_config["origin"] == "store": - with_defaults( - model_config, - store_config, - "source", - "name", - ["endpoint", "token", "bucket", "vocabulary"], - ) - model_name = model_config.pop("name") - model = import_class(model_name, "models") - self._model: Model = model(**model_config) - - # Store. - store_config.update(model_context=self._model.context()) - store_name = store_config.pop("name") - store = import_class(store_name, "stores") - self._store: Store = store(**store_config) - store_config.update(name=store_name) - - self._from_forge = from_forge - self._dirpath = os.path.join(DBS_PATH, self.name) - if self._from_forge is False: - # Save in directory and add it to forge instance - self._save_config() - else: - if not Path(self._dirpath).is_dir(): - raise ValueError(f"Database directory for {self.name} was not found.\ - To create a new database use from_forge=False") - - def _check_properties(self, **info): - properties = info.keys() - for r in self._REQUIRED: - if r not in properties: - raise ValueError(f'Missing {r} from the properties to define the DatabaseResource') - - def _save_config(self) -> None: - """Save database information inside the kgforge database folder.""" - # Make mappings directory - Path(self._dirpath).mkdir(parents=True, exist_ok=True) - Path(self._dirpath, 'mappings').mkdir(parents=True, exist_ok=True) - # Add the source to forge - self._forge.add_db_source(self) - - def datatypes(self): - # TODO: add other datatypes used, for instance, inside the mappings - return self.mappings(pretty=False).keys() - - def _model(self) -> None: - not_supported() - - def _mappings(self) -> Dict[str, List[str]]: - dirpath = Path(self._dirpath, "mappings") - mappings = {} - if dirpath.is_dir(): - for x in dirpath.glob("*/*.hjson"): - mappings.setdefault(x.stem, []).append(x.parent.name) - else: - raise ValueError("unrecognized source") - return mappings - - def mappings(self, pretty: bool) -> Optional[Dict[str, List[str]]]: - mappings = {k: sorted(v) for k, v in - sorted(self._mappings().items(), key=lambda kv: kv[0])} - if pretty: - print("Managed mappings for the data source per entity type and mapping type:") - for k, v in mappings.items(): - print(*[f" - {k}:", *v], sep="\n * ") - else: - return mappings - - def mapping(self, entity: str, type: Callable) -> Mapping: - filename = f"{entity}.hjson" - filepath = Path(self._dirpath, "mappings", type.__name__, filename) - if filepath.is_file(): - return type.load(filepath) - else: - raise ValueError("unrecognized entity type or source") - - def dump_config(self) -> None: - filename = "config.json" - filepath = Path(self._dirpath, filename) - with open(filepath, 'w') as mfile: - return mfile.write(json.dumps(self._forge.as_json(self), indent=4)) - - def health(self): - not_supported() \ No newline at end of file diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index 0bd6f375..322696ac 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -21,7 +21,6 @@ import re from asyncio import Semaphore, Task from enum import Enum -from json import JSONDecodeError from kgforge.core.commons.dictionaries import update_dict from kgforge.core.commons.es_query_builder import ESQueryBuilder @@ -65,7 +64,8 @@ from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping from kgforge.specializations.stores.nexus.service import BatchAction, Service, _error_message - +from kgforge.core.archetypes.store import build_construct_query +from kgforge.core.commons.es_query_builder import ESQueryBuilder class CategoryDataType(Enum): DATETIME = "datetime" @@ -875,41 +875,8 @@ def _sparql(self, query: str) -> List[Resource]: # https://github.com/BlueBrain/nexus/issues/1155 _, q_comp = Query.parseString(query) if q_comp.name == "ConstructQuery": - subject_triples = {} - for r in data["results"]["bindings"]: - subject = r["subject"]["value"] - s = f"<{r['subject']['value']}>" - p = f"<{r['predicate']['value']}>" - if r["object"]["type"] == "uri": - o = f"<{r['object']['value']}>" - else: - if "datatype" in r["object"]: - o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" - else: - o = f"\"{r['object']['value']}\"" - if subject in subject_triples: - subject_triples[subject] += f"\n{s} {p} {o} . " - else: - subject_triples[subject] = f"{s} {p} {o} . " - - def triples_to_resource(iri, triples): - graph = Graph().parse(data=triples, format="nt") - data_expanded = json.loads(graph.serialize(format="json-ld")) - data_expanded = json.loads(graph.serialize(format="json-ld")) - frame = {"@id": iri} - data_framed = jsonld.frame(data_expanded, frame) - context = self.model_context or self.context - compacted = jsonld.compact(data_framed, context.document) - resource = from_jsonld(compacted) - resource.context = ( - context.iri - if context.is_http_iri() - else context.document["@context"] - ) - return resource - - return [triples_to_resource(s, t) for s, t in subject_triples.items()] - + context = self.model_context or context + return build_construct_query(data, context) else: # SELECT QUERY results = data["results"]["bindings"] diff --git a/kgforge/specializations/stores/uniprot/__init__.py b/kgforge/specializations/stores/databases/__init__.py similarity index 100% rename from kgforge/specializations/stores/uniprot/__init__.py rename to kgforge/specializations/stores/databases/__init__.py diff --git a/kgforge/specializations/stores/uniprot/service.py b/kgforge/specializations/stores/databases/service.py similarity index 54% rename from kgforge/specializations/stores/uniprot/service.py rename to kgforge/specializations/stores/databases/service.py index 4000b772..4a0d3eff 100644 --- a/kgforge/specializations/stores/uniprot/service.py +++ b/kgforge/specializations/stores/databases/service.py @@ -48,14 +48,10 @@ recursive_resolve, ) from kgforge.core.wrappings.dict import wrap_dict -from kgforge.specializations.resources.db_sources import DBS_PATH class Service: - UNIPROT_CONTEXT_PATH = Path(DBS_PATH, 'UniProt', 'jsonld_context.json') - UNIPROT_NAMESPACE = 'http://purl.uniprot.org/core/' - def __init__( self, endpoint: str, @@ -72,7 +68,6 @@ def __init__( self.context_cache: Dict = dict() self.max_connection = max_connection self.params = copy.deepcopy(params) - self.namespace = self.UNIPROT_NAMESPACE self.headers = {"Content-Type": content_type, "Accept": accept} @@ -91,16 +86,6 @@ def __init__( else "application/sparql-results+json", } - # load context from file - with open(self.UNIPROT_CONTEXT_PATH) as jfile: - uniprot_context = json.load(jfile) - self.context = Context(uniprot_context) - - self.metadata_context = Context( - recursive_resolve(model_context, self.resolve_context, already_loaded=[]), - uniprot_context - ) - self.sparql_endpoint = dict() self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] self.sparql_endpoint["type"] = "sparql" @@ -125,107 +110,6 @@ def resolve_context(self, iri: str) -> Dict: self.context_cache.update({context_to_resolve: document}) return document - async def request(method, session, url, resource, payload, params, exception): - async with session.request( - method, - url, - headers=self.headers, - data=json.dumps(payload), - params=params, - ) as response: - content = await response.json() - if response.status < 400: - return (resource, content) - else: - msg = " ".join(re.findall("[A-Z][^A-Z]*", content["@type"])).lower() - error = exception(msg) - return (resource, error) - - def sync_metadata(self, resource: Resource, result: Dict) -> None: - metadata = ( - {"id": resource.id} - if hasattr(resource, "id") - else ( - {"id": resource.__getattribute__("@id")} - if hasattr(resource, "@id") - else dict() - ) - ) - keys = sorted(self.metadata_context.terms.keys()) - keys.extend(["_index", "_score", "id", "@id"]) - only_meta = {k: v for k, v in result.items() if k in keys} - metadata.update(_remove_ld_keys(only_meta, self.metadata_context, False)) - if not hasattr(resource, "id") and not hasattr(resource, "@id"): - resource.id= result.get("id", result.get("@id",None)) - resource._store_metadata = wrap_dict(metadata) - - def synchronize_resource( - self, - resource: Resource, - response: Union[Exception, Dict], - action_name: str, - succeeded: bool, - synchronized: bool, - ) -> None: - if succeeded: - action = Action(action_name, succeeded, None) - self.sync_metadata(resource, response) - else: - action = Action(action_name, succeeded, response) - resource._last_action = action - resource._synchronized = synchronized - - def default_callback(self, fun_name: str) -> Callable: - def callback(task: Task): - result = task.result() - if isinstance(result.response, Exception): - self.synchronize_resource( - result.resource, result.response, fun_name, False, False - ) - else: - self.synchronize_resource( - result.resource, result.response, fun_name, True, True - ) - - return callback - - def verify( - self, - resources: List[Resource], - function_name, - exception: Callable, - id_required: bool, - required_synchronized: bool, - execute_actions: bool, - ) -> List[Resource]: - valid = list() - for resource in resources: - if id_required and not hasattr(resource, "id"): - error = exception("resource should have an id") - self.synchronize_resource(resource, error, function_name, False, False) - continue - if required_synchronized is not None: - synchronized = resource._synchronized - if synchronized is not required_synchronized: - be_or_not_be = "be" if required_synchronized is True else "not be" - error = exception(f"resource should {be_or_not_be} synchronized") - self.synchronize_resource( - resource, error, function_name, False, False - ) - continue - if execute_actions: - lazy_actions = collect_lazy_actions(resource) - if lazy_actions is not None: - try: - execute_lazy_actions(resource, lazy_actions) - except Exception as e: - self.synchronize_resource( - resource, exception(e), function_name, False, False - ) - continue - valid.append(resource) - return valid - def to_resource( self, payload: Dict, sync_metadata: bool = True, **kwargs ) -> Resource: diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py index fb8f7fda..6376fea9 100644 --- a/kgforge/specializations/stores/sparql.py +++ b/kgforge/specializations/stores/sparql.py @@ -13,28 +13,24 @@ # along with Blue Brain Nexus Forge. If not, see . import json -from pyld import jsonld from copy import deepcopy from pathlib import Path from typing import Dict, List, Tuple, Optional, Union, Any from uuid import uuid4 -import requests from rdflib import Graph from rdflib.plugins.sparql.parser import Query -from requests import HTTPError from SPARQLWrapper import SPARQLWrapper, JSON from kgforge.core import Resource from kgforge.core.archetypes import Resolver, Store -from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql +from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql, build_construct_query from kgforge.core.commons.context import Context from kgforge.core.wrappings.dict import DictWrapper -from kgforge.specializations.stores.uniprot.service import Service -from kgforge.core.wrappings.paths import Filter, create_filters_from_dict -from kgforge.core.commons.exceptions import DownloadingError, QueryingError +from kgforge.specializations.stores.databases import Service +from kgforge.core.wrappings.paths import create_filters_from_dict +from kgforge.core.commons.exceptions import QueryingError from kgforge.core.commons.execution import not_supported from kgforge.core.commons.parser import _parse_type -from kgforge.core.conversions.rdf import as_jsonld, from_jsonld from kgforge.specializations.stores.bluebrain_nexus import ( CategoryDataType, _create_select_query, _box_value_as_full_iri, sparql_operator_map, @@ -49,6 +45,7 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, file_resource_mapping: Optional[str] = None, model_context: Optional[Context] = None, searchendpoints: Optional[Dict] = None,) -> None: + print('inside SPARQL Store') super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, model_context, searchendpoints) @@ -211,41 +208,8 @@ def _sparql(self, query: str, limit: int, offset: int = None, **params) -> List[ # https://github.com/BlueBrain/nexus/issues/1155 _, q_comp = Query.parseString(query) if q_comp.name == "ConstructQuery": - subject_triples = {} - for r in data["results"]["bindings"]: - subject = r["subject"]["value"] - s = f"<{r['subject']['value']}>" - p = f"<{r['predicate']['value']}>" - if r["object"]["type"] == "uri": - o = f"<{r['object']['value']}>" - else: - if "datatype" in r["object"]: - o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" - else: - o = f"\"{r['object']['value']}\"" - if subject in subject_triples: - subject_triples[subject] += f"\n{s} {p} {o} . " - else: - subject_triples[subject] = f"{s} {p} {o} . " - - def triples_to_resource(iri, triples): - graph = Graph().parse(data=triples, format="nt") - data_expanded = json.loads(graph.serialize(format="json-ld")) - data_expanded = json.loads(graph.serialize(format="json-ld")) - frame = {"@id": iri} - data_framed = jsonld.frame(data_expanded, frame) - context = self.model_context or self.context - compacted = jsonld.compact(data_framed, context.document) - resource = from_jsonld(compacted) - resource.context = ( - context.iri - if context.is_http_iri() - else context.document["@context"] - ) - return resource - - return [triples_to_resource(s, t) for s, t in subject_triples.items()] - + context = self.model_context or self.context + return build_construct_query(data, context) else: # SELECT QUERY results = data["results"]["bindings"] @@ -287,13 +251,13 @@ def _initialize_service( searchendpoints: Optional[Dict] = None, **store_config, ) -> Any: + print('initializing service') try: max_connection = store_config.pop("max_connection", 50) if max_connection <= 0: raise ValueError( f"max_connection value should be great than 0 but {max_connection} is provided" ) - store_context_config = store_config.pop("vocabulary", {}) content_type = store_config.pop("Content-Type", "application/ld+json") accept = store_config.pop("Accept", "application/ld+json") params = store_config.pop("params", {}) diff --git a/kgforge/specializations/stores/uniprot_store.py b/kgforge/specializations/stores/uniprot_store.py index 6374d7b6..431a39f9 100644 --- a/kgforge/specializations/stores/uniprot_store.py +++ b/kgforge/specializations/stores/uniprot_store.py @@ -35,8 +35,6 @@ # ) from kgforge.specializations.stores import SPARQLStore from kgforge.core.commons.context import Context -from kgforge.core.wrappings.dict import DictWrapper -from kgforge.specializations.stores.uniprot.service import Service from kgforge.core.commons.exceptions import QueryingError from kgforge.core.commons.execution import not_supported from kgforge.core.conversions.rdf import as_jsonld, from_jsonld @@ -64,7 +62,6 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, @staticmethod def resources_from_results(results): - print(f'amount of results = {len(results)}') resources = [] for result in results: resource = {} From 573ae502f593bc1ee22caa54b40c15684aeb523e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 18 Oct 2022 09:04:29 +0200 Subject: [PATCH 14/30] Adding the missing file. --- kgforge/core/archetypes/database.py | 147 ++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 kgforge/core/archetypes/database.py diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py new file mode 100644 index 00000000..c2b43012 --- /dev/null +++ b/kgforge/core/archetypes/database.py @@ -0,0 +1,147 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +from abc import ABC, abstractmethod +import json +from pathlib import Path +from typing import Any, Optional, Callable, Dict, List + +from kgforge.core import Resource +from kgforge.core.commons.context import Context +from kgforge.core.archetypes import Mapping, Model +from kgforge.core.commons.attributes import repr_class +from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.core.commons.execution import not_supported +from kgforge.core.commons.imports import import_class +from kgforge.core.commons.dictionaries import with_defaults + + +class Database(ABC): + + # POLICY Methods of archetypes, except __init__, should not have optional arguments. + + # POLICY Implementations should be declared in kgforge/specializations/databases/__init__.py. + # POLICY Implementations should not add methods but private functions in the file. + # POLICY Implementations should pass tests/specializations/databases/test_databases.py. + + def __init__(self, source: str, **config) -> None: + # POLICY Resolver data access should be lazy, unless it takes less than a second. + # POLICY There could be data caching but it should be aware of changes made in the source. + # Model + model_config = config.pop("model") + if model_config.get('origin') == 'directory': + dirpath = model_config.get('source') + self._dirpath = dirpath + bucket = model_config.get('bucket', 'jsonld_context.json') + iri = model_config.get('iri', None) + context_path = Path(dirpath, bucket) + try: + # load context from file + with open(context_path, 'r') as jfile: + context_file = json.load(jfile) + self.context = Context(context_file, iri) + except Exception: + self.context = None + config['model_context'] = self.context + elif model_config["origin"] == "store": + with_defaults( + model_config, + config, + "source", + "name", + ["endpoint", "token", "bucket", "vocabulary"], + ) + model_name = model_config.pop("name") + model = import_class(model_name, "models") + self._model: Model = model(**model_config) + config['model_context'] = self._model.context() + else: + raise NotImplementedError('DB Model not yet implemented.') + self.source: str = source + self.service: Any = self._initialize_service(self.source, **config) + + def __repr__(self) -> str: + return repr_class(self) + + def datatypes(self): + # TODO: add other datatypes used, for instance, inside the mappings + return self.mappings(pretty=False).keys() + + def _mappings(self) -> Dict[str, List[str]]: + try: + dirpath = Path(self._dirpath, "mappings") + mappings = {} + if dirpath.is_dir(): + for x in dirpath.glob("*/*.hjson"): + mappings.setdefault(x.stem, []).append(x.parent.name) + else: + raise ValueError("Mapping directory not found.") + return mappings + except AttributeError: + raise ConfigurationError('No directory path was found from the configuration.') + + def mappings(self) -> Optional[Dict[str, List[str]]]: + mappings = {k: sorted(v) for k, v in + sorted(self._mappings().items(), key=lambda kv: kv[0])} + return mappings + + def mapping(self, entity: str, type: Callable) -> Mapping: + filename = f"{entity}.hjson" + try: + filepath = Path(self._dirpath, "mappings", type.__name__, filename) + if filepath.is_file(): + return type.load(filepath) + else: + raise ValueError("unrecognized entity type or source file") + except AttributeError: + raise ConfigurationError('No directory path was found from the configuration.') + + @property + @abstractmethod + def health(self) -> Callable: + """Health of the database.""" + pass + + # Utils. + + def _initialize_service(self, source: str, **source_config) -> Any: + # Resolver data could be accessed from a directory, a web service, or a Store. + # Initialize the access to the resolver data according to the source type. + # POLICY Should not use 'self'. This is not a function only for the specialization to work. + origin = source_config.pop("origin") + if origin == "directory": + dirpath = Path(source) + return self._service_from_directory(dirpath, **source_config) + elif origin == "web_service": + return self._service_from_web_service(source, **source_config) + elif origin == "store": + store = import_class(source, "stores") + return self._service_from_store(store, **source_config) + else: + raise ConfigurationError(f"unrecognized DataBase origin '{origin}'") + + @staticmethod + @abstractmethod + def _service_from_directory(dirpath: Path, **source_config) -> Any: + pass + + @staticmethod + @abstractmethod + def _service_from_web_service(endpoint: str, **source_config) -> Any: + pass + + @staticmethod + @abstractmethod + def _service_from_store(store: Callable, **store_config) -> Any: + pass From 2520b7f8cfe0a4a1654866115e9bd3f6df6322a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 19 Oct 2022 14:25:51 +0200 Subject: [PATCH 15/30] Sorted mapping functions. Missing tests. --- .../DictionaryMapping/NeuronMorphology.hjson | 0 .../MouseLight/metadata.json | 0 .../NeuroElectro/jsonld_context.json | 0 ...lectrophysiologicalFeatureAnnotation.hjson | 0 .../ParameterAnnotation.hjson | 0 .../DictionaryMapping/ParameterBody.hjson | 0 .../DictionaryMapping/ScholarlyArticle.hjson | 0 .../DictionaryMapping/SeriesBody.hjson | 0 .../NeuroElectro/metadata.json | 0 .../UniProt/jsonld_context.json | 0 .../mappings/DictionaryMapping/Gene.hjson | 0 .../mappings/DictionaryMapping/Protein.hjson | 0 .../UniProt/metadata.json | 0 .../DictionaryMapping/NeuronMorphology.hjson | 0 .../DictionaryMapping/PatchedCell.hjson | 0 .../mappings/DictionaryMapping/Subject.hjson | 0 .../DictionaryMapping/Association.hjson | 0 .../DictionaryMapping/Contribution.hjson | 0 .../17 - Database-sources.ipynb | 549 +++++------------- kgforge/core/archetypes/database.py | 55 +- kgforge/core/archetypes/store.py | 2 +- kgforge/core/forge.py | 41 +- .../databases/store_database.py | 70 ++- .../specializations/stores/bluebrain_nexus.py | 4 +- .../stores/databases/service.py | 66 +-- kgforge/specializations/stores/sparql.py | 17 +- 26 files changed, 283 insertions(+), 521 deletions(-) rename examples/{database_sources => database-sources}/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson (100%) rename examples/{database_sources => database-sources}/MouseLight/metadata.json (100%) create mode 100644 examples/database-sources/NeuroElectro/jsonld_context.json rename examples/{database_sources => database-sources}/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson (100%) rename examples/{database_sources => database-sources}/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson (100%) rename examples/{database_sources => database-sources}/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson (100%) rename examples/{database_sources => database-sources}/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson (100%) rename examples/{database_sources => database-sources}/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson (100%) rename examples/{database_sources => database-sources}/NeuroElectro/metadata.json (100%) rename examples/{database_sources => database-sources}/UniProt/jsonld_context.json (100%) rename examples/{database_sources => database-sources}/UniProt/mappings/DictionaryMapping/Gene.hjson (100%) rename examples/{database_sources => database-sources}/UniProt/mappings/DictionaryMapping/Protein.hjson (100%) rename examples/{database_sources => database-sources}/UniProt/metadata.json (100%) rename examples/{database_sources => database-sources}/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson (100%) rename examples/{database_sources => database-sources}/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson (100%) rename examples/{database_sources => database-sources}/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson (100%) rename examples/{database_sources => database-sources}/scientists-database/DictionaryMapping/Association.hjson (100%) rename examples/{database_sources => database-sources}/scientists-database/DictionaryMapping/Contribution.hjson (100%) diff --git a/examples/database_sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson similarity index 100% rename from examples/database_sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson rename to examples/database-sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson diff --git a/examples/database_sources/MouseLight/metadata.json b/examples/database-sources/MouseLight/metadata.json similarity index 100% rename from examples/database_sources/MouseLight/metadata.json rename to examples/database-sources/MouseLight/metadata.json diff --git a/examples/database-sources/NeuroElectro/jsonld_context.json b/examples/database-sources/NeuroElectro/jsonld_context.json new file mode 100644 index 00000000..e69de29b diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson similarity index 100% rename from examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson rename to examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson similarity index 100% rename from examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson rename to examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson similarity index 100% rename from examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson rename to examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson similarity index 100% rename from examples/database_sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson rename to examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson diff --git a/examples/database_sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson similarity index 100% rename from examples/database_sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson rename to examples/database-sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson diff --git a/examples/database_sources/NeuroElectro/metadata.json b/examples/database-sources/NeuroElectro/metadata.json similarity index 100% rename from examples/database_sources/NeuroElectro/metadata.json rename to examples/database-sources/NeuroElectro/metadata.json diff --git a/examples/database_sources/UniProt/jsonld_context.json b/examples/database-sources/UniProt/jsonld_context.json similarity index 100% rename from examples/database_sources/UniProt/jsonld_context.json rename to examples/database-sources/UniProt/jsonld_context.json diff --git a/examples/database_sources/UniProt/mappings/DictionaryMapping/Gene.hjson b/examples/database-sources/UniProt/mappings/DictionaryMapping/Gene.hjson similarity index 100% rename from examples/database_sources/UniProt/mappings/DictionaryMapping/Gene.hjson rename to examples/database-sources/UniProt/mappings/DictionaryMapping/Gene.hjson diff --git a/examples/database_sources/UniProt/mappings/DictionaryMapping/Protein.hjson b/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson similarity index 100% rename from examples/database_sources/UniProt/mappings/DictionaryMapping/Protein.hjson rename to examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson diff --git a/examples/database_sources/UniProt/metadata.json b/examples/database-sources/UniProt/metadata.json similarity index 100% rename from examples/database_sources/UniProt/metadata.json rename to examples/database-sources/UniProt/metadata.json diff --git a/examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson similarity index 100% rename from examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson rename to examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson diff --git a/examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson similarity index 100% rename from examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson rename to examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson diff --git a/examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson similarity index 100% rename from examples/database_sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson rename to examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson diff --git a/examples/database_sources/scientists-database/DictionaryMapping/Association.hjson b/examples/database-sources/scientists-database/DictionaryMapping/Association.hjson similarity index 100% rename from examples/database_sources/scientists-database/DictionaryMapping/Association.hjson rename to examples/database-sources/scientists-database/DictionaryMapping/Association.hjson diff --git a/examples/database_sources/scientists-database/DictionaryMapping/Contribution.hjson b/examples/database-sources/scientists-database/DictionaryMapping/Contribution.hjson similarity index 100% rename from examples/database_sources/scientists-database/DictionaryMapping/Contribution.hjson rename to examples/database-sources/scientists-database/DictionaryMapping/Contribution.hjson diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 622ef042..b7b35a7b 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -33,25 +33,7 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "I am in BBNexus store\n", - "I am in BBNexus store\n", - "I am in BBNexus store\n", - "I am in BBNexus store\n", - "Configuration {'origin': 'store', 'source': 'SPARQLStore', 'endpoint': 'http://purl.uniprot.org/core/', 'searchendpoints': {'sparql': {'endpoint': 'https://sparql.uniprot.org/sparql'}}, 'model': {'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', 'context': {'iri': 'https://bbp.epfl.ch/jsonldcontext/db/uniprot', 'bucket': 'jsonld_context.json'}}}\n", - "store config {'endpoint': 'http://purl.uniprot.org/core/', 'searchendpoints': {'sparql': {'endpoint': 'https://sparql.uniprot.org/sparql'}}, 'model_context': }\n", - "inside SPARQL Store\n", - "initializing service\n", - "Configuration {'origin': 'store', 'source': 'BlueBrainNexus', 'bucket': 'bbp/neuroelectro', 'model': {'origin': 'directory', 'source': '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro', 'context': {'bucket': 'jsonld_context.json'}}, 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjYwMjkyNjEsImlhdCI6MTY2NjAxODIzNSwiYXV0aF90aW1lIjoxNjY2MDAwNDYxLCJqdGkiOiIxNjQwNTBkOS04OTAzLTQzMjItOWQyZi0xOGQ0ODg3MTYzYWEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwOWM0NDA5YjExZjRkNjI4YjM2NmM2N2MxOGU0MDk5Iiwic2Vzc2lvbl9zdGF0ZSI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.XHZ55pSp2zRQcgcvsXHYWS2adj12K411PtQaZesiABtKH9tgsjz0dgY_4ijneBHdHd1zgmqNRFqnHHd78EMD7XoHtmbpZOI5h1t12wK2YbI35mOwLfS1RcRnT6kKMuiFe5m4AzW1XWwHn9AcO_xeZTbixL2vqPLfdPg1jzk5uW-rgs3Sg-77Kr3LEBVTzVSHSokhtTqjHNY2EQlBljmpJpGCno-6keLX5LrNERFek303oJrAIgvfGSCcpKoHYYWCPiReD4-cjd8l-zMCVrfxCeuRhihVSpNGgPX67F4O7u8EnU_EMeA39eg9HSp5KahhrGwecCzqPMVg4R37cysz_Q', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'model_context': , 'name': 'BlueBrainNexus'}\n", - "store config {'bucket': 'bbp/neuroelectro', 'endpoint': 'https://staging.nise.bbp.epfl.ch/nexus/v1', 'searchendpoints': {'sparql': {'endpoint': 'https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex'}, 'elastic': {'endpoint': 'https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset', 'mapping': 'https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset', 'default_str_keyword_field': 'keyword'}}, 'vocabulary': {'metadata': {'iri': 'https://bluebrain.github.io/nexus/contexts/metadata.json', 'local_iri': 'https://bluebrainnexus.io/contexts/metadata.json'}, 'namespace': 'https://bluebrain.github.io/nexus/vocabulary/', 'deprecated_property': 'https://bluebrain.github.io/nexus/vocabulary/deprecated', 'project_property': 'https://bluebrain.github.io/nexus/vocabulary/project'}, 'max_connection': 50, 'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjYwMjkyNjEsImlhdCI6MTY2NjAxODIzNSwiYXV0aF90aW1lIjoxNjY2MDAwNDYxLCJqdGkiOiIxNjQwNTBkOS04OTAzLTQzMjItOWQyZi0xOGQ0ODg3MTYzYWEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwOWM0NDA5YjExZjRkNjI4YjM2NmM2N2MxOGU0MDk5Iiwic2Vzc2lvbl9zdGF0ZSI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.XHZ55pSp2zRQcgcvsXHYWS2adj12K411PtQaZesiABtKH9tgsjz0dgY_4ijneBHdHd1zgmqNRFqnHHd78EMD7XoHtmbpZOI5h1t12wK2YbI35mOwLfS1RcRnT6kKMuiFe5m4AzW1XWwHn9AcO_xeZTbixL2vqPLfdPg1jzk5uW-rgs3Sg-77Kr3LEBVTzVSHSokhtTqjHNY2EQlBljmpJpGCno-6keLX5LrNERFek303oJrAIgvfGSCcpKoHYYWCPiReD4-cjd8l-zMCVrfxCeuRhihVSpNGgPX67F4O7u8EnU_EMeA39eg9HSp5KahhrGwecCzqPMVg4R37cysz_Q', 'versioned_id_template': '{x.id}?rev={x._store_metadata._rev}', 'file_resource_mapping': 'https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson', 'model_context': None}\n", - "I am in BBNexus store\n" - ] - } - ], + "outputs": [], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"neurosciencegraph/datamodels\"\n", @@ -76,7 +58,8 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n" + "NeuroElectro\n", + "MouseLight\n" ] } ], @@ -115,15 +98,7 @@ "cell_type": "code", "execution_count": 6, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "store config {'model_context': None}\n" - ] - } - ], + "outputs": [], "source": [ "from kgforge.specializations.databases import StoreDatabase\n", "ds = StoreDatabase(forge, name=\"DemoDB\", **data)" @@ -135,7 +110,7 @@ "metadata": {}, "outputs": [], "source": [ - "# print(ds)" + "forge.add_db_source(ds)" ] }, { @@ -149,7 +124,9 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n" + "NeuroElectro\n", + "MouseLight\n", + "DemoDB\n" ] } ], @@ -322,18 +299,16 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'UniProt': StoreDatabase(context=, type='Database', _dirpath='/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt', _forge=, name='UniProt', service=SPARQLStore(context=None, bucket=None, endpoint='http://purl.uniprot.org/core/', file_mapping=None, metadata_context=None, model_context=, service=, token=None, versioned_id_template=None), source='SPARQLStore')}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Available Database sources:\n", + "UniProt\n" + ] } ], "source": [ - "forge.db_sources(mappings='Gene')" + "forge.db_sources(mappings='Gene', pretty=True)" ] }, { @@ -348,42 +323,79 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " search\n", - " ValueError: context model missing\n", + "Submitted query:\n", + " PREFIX bmc: \n", + " PREFIX bmo: \n", + " PREFIX commonshapes: \n", + " PREFIX datashapes: \n", + " PREFIX dc: \n", + " PREFIX dcat: \n", + " PREFIX dcterms: \n", + " PREFIX mba: \n", + " PREFIX nsg: \n", + " PREFIX nxv: \n", + " PREFIX oa: \n", + " PREFIX obo: \n", + " PREFIX owl: \n", + " PREFIX prov: \n", + " PREFIX rdf: \n", + " PREFIX rdfs: \n", + " PREFIX schema: \n", + " PREFIX sh: \n", + " PREFIX shsh: \n", + " PREFIX skos: \n", + " PREFIX vann: \n", + " PREFIX void: \n", + " PREFIX xml: \n", + " PREFIX xsd: \n", + " PREFIX : \n", + " SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id rdf:type schema:ScholarlyArticle;\n", + " ?_constrainedBy;\n", + " ?_createdAt;\n", + " ?_createdBy;\n", + " ?_deprecated;\n", + " ?_incoming;\n", + " ?_outgoing;\n", + " ?_project;\n", + " ?_rev;\n", + " ?_schemaProject;\n", + " ?_self;\n", + " ?_updatedAt;\n", + " ?_updatedBy . \n", + " Filter (?_deprecated = 'false'^^xsd:boolean)\n", + " Filter (?_project = )}} LIMIT 2\n", "\n" ] } ], "source": [ - "# Type, source or target brain region, \n", "filters = {\"type\":\"ScholarlyArticle\"}\n", "#map=True, use_cache=True, # download=True\n", - "resources = forge.search(filters, db_source=\"NeuroElectro\", limit=2) \n", - "# ADd function for checking datsource health => reqsuire health url from db\n" + "resources = forge.search(filters, db_source=\"NeuroElectro\", limit=2, debug=True) \n", + "# Add function for checking datsource health => reqsuire health url from db\n" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 17, "metadata": {}, "outputs": [ { - "ename": "TypeError", - "evalue": "object of type 'NoneType' has no len()", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_81136/3721108832.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresources\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()" - ] + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -392,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -401,68 +413,68 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/14353\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Cerebellar Purkinje cells (PCs) from spinocerebellar ataxia type 1 (SCA1) transgenic mice develop dendritic and somatic atrophy with age. Inositol 1,4,5-trisphosphate receptor type 1 and the sarco/endoplasmic reticulum Ca(2+) ATPase pump, which regulate [Ca(2+)](i), are expressed at lower levels in these cells compared with the levels in cells from wild-type (WT) mice. To examine PCs in SCA1 mice, we used whole-cell patch clamp recording combined with fluorometric [Ca(2+)](i) and [Na(+)](i) measurements in cerebellar slices. PCs in SCA1 mice had Na(+) spikes, Ca(2+) spikes, climbing fiber (CF) electrical responses, parallel fiber (PF) electrical responses, and metabotropic glutamate receptor (mGluR)-mediated, PF-evoked Ca(2+) release from intracellular stores that were qualitatively similar to those recorded from WT mice. Under our experimental conditions, it was easier to evoke the mGluR-mediated secondary [Ca(2+)](i) increase in SCA1 PCs. The membrane resistance of SCA1 PCs was 3.3 times higher than that of WT cells, which correlated with the 1.7 times smaller cell body size. Most SCA1 PCs (but not WT) had a delayed onset (about 50--200 ms) to Na(+) spike firing induced by current injection. This delay was increased by hyperpolarizing prepulses and was eliminated by 4-aminopyridine, which suggests that this delay was due to enhancement of the A-like K(+) conductance in the SCA1 PCs. In response to CF stimulation, most PCs in mutant and WT mice had rapid, widespread [Ca(2+)](i) changes that recovered in <200 ms. Some SCA1 PCs showed a slow, localized, secondary Ca(2+) transient following the initial CF Ca(2+) transient, which may reflect release of Ca(2+) from intracellular stores. Thus, with these exceptions, the basic physiological properties of mutant PCs are similar to those of WT neurons, even with dramatic alteration of their morphology and downregulation of Ca(2+) handling molecules.\n", + " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Inoue\n", - " givenName: T\n", + " familyName: Henderson\n", + " givenName: Z\n", " }\n", " {\n", " type: Person\n", - " familyName: Lin\n", - " givenName: X\n", + " familyName: Morris\n", + " givenName: N P\n", " }\n", " {\n", " type: Person\n", - " familyName: Kohlmeier\n", - " givenName: K A\n", + " familyName: Grimwood\n", + " givenName: P\n", " }\n", " {\n", " type: Person\n", - " familyName: Orr\n", - " givenName: H T\n", + " familyName: Fiddler\n", + " givenName: G\n", " }\n", " {\n", " type: Person\n", - " familyName: Zoghbi\n", - " givenName: H Y\n", + " familyName: Yang\n", + " givenName: H W\n", " }\n", " {\n", " type: Person\n", - " familyName: Ross\n", - " givenName: W N\n", + " familyName: Appenteng\n", + " givenName: K\n", " }\n", " ]\n", - " datePublished: 2001-4\n", + " datePublished: 2001-2-12\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 11287496\n", + " value: 11169477\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1152/jn.2001.85.4.1750\n", + " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0022-3077\n", - " name: Journal of neurophysiology\n", + " issn: 0021-9967\n", + " name: The Journal of comparative neurology\n", " }\n", - " name: article_14353\n", - " sameAs: http://jn.physiology.org/content/85/4/1750.long\n", - " title: Calcium dynamics and electrophysiological properties of cerebellar Purkinje cells in SCA1 transgenic mice.\n", - " url: http://jn.physiology.org/content/85/4/1750.long\n", + " name: article_91941\n", + " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", + " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", "}\n" ] } @@ -473,7 +485,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -489,21 +501,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "query in sparql \n", - "PREFIX up: \n", - "SELECT ?protein\n", - "WHERE {\n", - " ?protein a up:Protein ;\n", - " up:reviewed true.\n", - "}\n", - "\n", "Submitted query:\n", " \n", " PREFIX up: \n", @@ -513,8 +517,7 @@ " up:reviewed true.\n", " }\n", " LIMIT 10\n", - "\n", - "amount of results = 10\n" + "\n" ] } ], @@ -524,7 +527,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -533,7 +536,7 @@ "10" ] }, - "execution_count": 24, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -544,16 +547,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein=Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0A131MCZ8', _inner_sync=False, annotationScore=5.0, comments=[{'texts': [{'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'Probable metal transporter. Probably acts redundantly with the other metal transport proteins cnnm-1, cnnm-2, cnnm-4 and cnnm-5 to regulate Mg(2+) homeostasis. Promotes postembryonic gonad development by regulating Mg(2+) levels, probably via AMPK signaling'}], 'commentType': 'FUNCTION'}, {'commentType': 'SUBCELLULAR LOCATION', 'subcellularLocations': [{'location': {'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'Basolateral cell membrane', 'id': 'SL-0026'}, 'topology': {'evidences': [{'evidenceCode': 'ECO:0000255'}], 'value': 'Multi-pass membrane protein', 'id': 'SL-9909'}}]}, {'commentType': 'ALTERNATIVE PRODUCTS', 'events': ['Alternative splicing'], 'isoforms': [{'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'a'}, 'isoformIds': ['A0A131MCZ8-1'], 'isoformSequenceStatus': 'Displayed'}, {'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2b'}], 'value': 'b'}, 'isoformIds': ['A0A131MCZ8-2'], 'sequenceIds': ['VSP_058662'], 'isoformSequenceStatus': 'Described'}, {'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2c'}], 'value': 'c'}, 'isoformIds': ['A0A131MCZ8-3'], 'sequenceIds': ['VSP_058663', 'VSP_058664'], 'isoformSequenceStatus': 'Described'}, {'name': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2d'}], 'value': 'd'}, 'isoformIds': ['A0A131MCZ8-4'], 'sequenceIds': ['VSP_058662', 'VSP_058663', 'VSP_058664'], 'isoformSequenceStatus': 'Described'}]}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'Highly expressed in the intestine and in neurons, but it is also expressed in a variety of tissues including the pharynx, hypodermis, rectum and in muscles'}], 'commentType': 'TISSUE SPECIFICITY'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000269', 'source': 'PubMed', 'id': '27564576'}], 'value': 'No visible phenotype. Double knockout with cnnm-1 results in increased levels of intestinal Mg(2+) and reduced levels in other tissues. This Mg(2+) deficiency in tissues leads to a reduced lifespan, 100% sterility, and smaller animals that exhibit a developmental delay with defective gonad development and which therefore do not produce oocytes or form vulva. In addition, the gonad development defect in the cnnm-1 and cnnm-3 double knockout is rescued when the AMPK alpha subunit aak-2 is also knocked out. Double knockout with cnnm-2 results in 22% sterility. Quintuple knockout with cnnm-1, cnnm-2, cnnm-4 and cnnm-5 results in a reduced lifespan and 100% sterility'}], 'commentType': 'DISRUPTION PHENOTYPE'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000305'}], 'value': 'Belongs to the ACDP family'}], 'commentType': 'SIMILARITY'}], entryAudit={'firstPublicDate': '2016-11-30', 'lastAnnotationUpdateDate': '2022-08-03', 'lastSequenceUpdateDate': '2016-05-11', 'entryVersion': 37, 'sequenceVersion': 1}, entryType='UniProtKB reviewed (Swiss-Prot)', extraAttributes={'countByCommentType': {'FUNCTION': 1, 'SUBCELLULAR LOCATION': 1, 'ALTERNATIVE PRODUCTS': 4, 'TISSUE SPECIFICITY': 1, 'DISRUPTION PHENOTYPE': 1, 'SIMILARITY': 1}, 'countByFeatureType': {'Signal': 1, 'Chain': 1, 'Topological domain': 5, 'Transmembrane': 4, 'Domain': 3, 'Glycosylation': 7, 'Alternative sequence': 3}, 'uniParcId': 'UPI00077FE610'}, features=[{'type': 'Signal', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 23, 'modifier': 'EXACT'}}, 'description': '', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Chain', 'location': {'start': {'value': 24, 'modifier': 'EXACT'}, 'end': {'value': 797, 'modifier': 'EXACT'}}, 'description': 'Metal transporter cnnm-3', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'PRO_5007283697'}, {'type': 'Topological domain', 'location': {'start': {'value': 24, 'modifier': 'EXACT'}, 'end': {'value': 200, 'modifier': 'EXACT'}}, 'description': 'Extracellular', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 201, 'modifier': 'EXACT'}, 'end': {'value': 221, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 222, 'modifier': 'EXACT'}, 'end': {'value': 255, 'modifier': 'EXACT'}}, 'description': 'Cytoplasmic', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 256, 'modifier': 'EXACT'}, 'end': {'value': 276, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 277, 'modifier': 'EXACT'}, 'end': {'value': 280, 'modifier': 'EXACT'}}, 'description': 'Extracellular', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 281, 'modifier': 'EXACT'}, 'end': {'value': 301, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 302, 'modifier': 'EXACT'}, 'end': {'value': 322, 'modifier': 'EXACT'}}, 'description': 'Cytoplasmic', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Transmembrane', 'location': {'start': {'value': 323, 'modifier': 'EXACT'}, 'end': {'value': 343, 'modifier': 'EXACT'}}, 'description': 'Helical', 'evidences': [{'evidenceCode': 'ECO:0000255'}]}, {'type': 'Topological domain', 'location': {'start': {'value': 344, 'modifier': 'EXACT'}, 'end': {'value': 797, 'modifier': 'EXACT'}}, 'description': 'Extracellular', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}, {'type': 'Domain', 'location': {'start': {'value': 193, 'modifier': 'EXACT'}, 'end': {'value': 373, 'modifier': 'EXACT'}}, 'description': 'CNNM transmembrane', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU01193'}]}, {'type': 'Domain', 'location': {'start': {'value': 393, 'modifier': 'EXACT'}, 'end': {'value': 454, 'modifier': 'EXACT'}}, 'description': 'CBS 1', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00703'}]}, {'type': 'Domain', 'location': {'start': {'value': 461, 'modifier': 'EXACT'}, 'end': {'value': 527, 'modifier': 'EXACT'}}, 'description': 'CBS 2', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00703'}]}, {'type': 'Glycosylation', 'location': {'start': {'value': 31, 'modifier': 'EXACT'}, 'end': {'value': 31, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 42, 'modifier': 'EXACT'}, 'end': {'value': 42, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 676, 'modifier': 'EXACT'}, 'end': {'value': 676, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 692, 'modifier': 'EXACT'}, 'end': {'value': 692, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 724, 'modifier': 'EXACT'}, 'end': {'value': 724, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 731, 'modifier': 'EXACT'}, 'end': {'value': 731, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Glycosylation', 'location': {'start': {'value': 761, 'modifier': 'EXACT'}, 'end': {'value': 761, 'modifier': 'EXACT'}}, 'description': 'N-linked (GlcNAc...) asparagine', 'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'PROSITE-ProRule', 'id': 'PRU00498'}], 'featureId': ''}, {'type': 'Alternative sequence', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 547, 'modifier': 'EXACT'}}, 'description': 'in isoform b and isoform d', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'VSP_058662', 'alternativeSequence': {}}, {'type': 'Alternative sequence', 'location': {'start': {'value': 753, 'modifier': 'EXACT'}, 'end': {'value': 766, 'modifier': 'EXACT'}}, 'description': 'in isoform c and isoform d', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'VSP_058663', 'alternativeSequence': {'originalSequence': 'DAVSTPIRNGSVKL', 'alternativeSequences': ['KYLIGRWKRRGTYV']}}, {'type': 'Alternative sequence', 'location': {'start': {'value': 767, 'modifier': 'EXACT'}, 'end': {'value': 797, 'modifier': 'EXACT'}}, 'description': 'in isoform c and isoform d', 'evidences': [{'evidenceCode': 'ECO:0000305'}], 'featureId': 'VSP_058664', 'alternativeSequence': {}}], genes=[{'geneName': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'cnnm-3'}, 'orfNames': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'C33D12.2'}]}], keywords=[{'id': 'KW-0025', 'category': 'Coding sequence diversity', 'name': 'Alternative splicing'}, {'id': 'KW-0129', 'category': 'Domain', 'name': 'CBS domain'}, {'id': 'KW-1003', 'category': 'Cellular component', 'name': 'Cell membrane'}, {'id': 'KW-0325', 'category': 'PTM', 'name': 'Glycoprotein'}, {'id': 'KW-0406', 'category': 'Biological process', 'name': 'Ion transport'}, {'id': 'KW-0472', 'category': 'Cellular component', 'name': 'Membrane'}, {'id': 'KW-1185', 'category': 'Technical term', 'name': 'Reference proteome'}, {'id': 'KW-0677', 'category': 'Domain', 'name': 'Repeat'}, {'id': 'KW-0732', 'category': 'Domain', 'name': 'Signal'}, {'id': 'KW-0812', 'category': 'Domain', 'name': 'Transmembrane'}, {'id': 'KW-1133', 'category': 'Domain', 'name': 'Transmembrane helix'}, {'id': 'KW-0813', 'category': 'Biological process', 'name': 'Transport'}], organism={'scientificName': 'Caenorhabditis elegans', 'taxonId': 6239, 'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'Proteomes', 'id': 'UP000001940'}], 'lineage': ['Eukaryota', 'Metazoa', 'Ecdysozoa', 'Nematoda', 'Chromadorea', 'Rhabditida', 'Rhabditina', 'Rhabditomorpha', 'Rhabditoidea', 'Rhabditidae', 'Peloderinae', 'Caenorhabditis']}, primaryAccession='A0A131MCZ8', proteinDescription={'recommendedName': {'fullName': {'evidences': [{'evidenceCode': 'ECO:0000305'}], 'value': 'Metal transporter cnnm-3'}}, 'alternativeNames': [{'fullName': {'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'WormBase', 'id': 'C33D12.2a'}], 'value': 'CNNM family homolog 3'}}], 'flag': 'Precursor'}, proteinExistence='2: Evidence at transcript level', references=[{'citation': {'id': '9851916', 'citationType': 'journal article', 'authoringGroup': ['The C. elegans sequencing consortium'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '9851916'}, {'database': 'DOI', 'id': '10.1126/science.282.5396.2012'}], 'title': 'Genome sequence of the nematode C. elegans: a platform for investigating biology.', 'publicationDate': '1998', 'journal': 'Science', 'firstPage': '2012', 'lastPage': '2018', 'volume': '282'}, 'referencePositions': ['NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]'], 'referenceComments': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'Proteomes', 'id': 'UP000001940'}], 'value': 'Bristol N2', 'type': 'STRAIN'}], 'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'Proteomes', 'id': 'UP000001940'}]}, {'citation': {'id': '27564576', 'citationType': 'journal article', 'authors': ['Ishii T.', 'Funato Y.', 'Hashizume O.', 'Yamazaki D.', 'Hirata Y.', 'Nishiwaki K.', 'Kono N.', 'Arai H.', 'Miki H.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '27564576'}, {'database': 'DOI', 'id': '10.1371/journal.pgen.1006276'}], 'title': 'Mg2+ extrusion from intestinal epithelia by CNNM proteins is essential for gonadogenesis via AMPK-TORC1 signaling in Caenorhabditis elegans.', 'publicationDate': '2016', 'journal': 'PLoS Genet.', 'firstPage': 'E1006276', 'lastPage': 'E1006276', 'volume': '12'}, 'referencePositions': ['FUNCTION', 'SUBCELLULAR LOCATION', 'TISSUE SPECIFICITY', 'DISRUPTION PHENOTYPE'], 'evidences': [{'evidenceCode': 'ECO:0000305'}]}], secondaryAccessions=['A0A131MBV5', 'A0A131MD56', 'Q21469'], sequence={'value': 'MSKTPWALGLLIFLLTFTSPLSSSPVRSTDNSTSSKGLLNVNSSVILEPSILPSSASKPESLHLSKVRVSGLRLEAHASSTENIVLGHNKKHNVVVVPNKNVRVVLFGQNFQDIGALTFTADGSCKDLAHFFEADFSSMTPIRVVVEMSFPKTTESKDSFKLCVSEKFYANPQFVIVEDPFTMVTTEIPPVDEYMPKWLSWICLLILLCFSGLFSGLNLGLMTLSPYELQLYIASGTEQEKRDAGRILPIRKKGNQLLCTLLIGNVVVNVGVSLLMDQLVGSGFAVLVAATSCIVVFGEIIPQALCVKLGLPIGARTIPITQVLLFLMYPLTWPISKVLDIFLKEELTRSLERNKLVEMLKLSEKSIIGGQSDEFKMVLGALELYDKTVAHAMTRYEDIFMLPHTLTLGAGMVTQILDMGYTRIPIYENDRKNIVALLFVKDLALLDPDDNHNVMKIASIYNHEVRRVLVDMPLRNMLEEFKRGEYHMALVERLVEQEDKDPIYELCGLITLEDIIEEIIQCEIIDETDAVCDNVHRKKRQRKRNHDMSQIVNTAHAKCAINIQMLAVTIQVMSTCHKIFSSNYILPTILEKLIRKNCKKVETTQFSCLKEVGVVQPKPAVLFTKGEFSNKFIMILSGRAVVTIGKEEMRLEAGAWHSFGTEVLDAMAEAIERSLNQSTSRSTVSLNTEITNNSIGFIPDFDTVILYECVFCEITAADLLLAYNSSQIMQNNTKMQVVRSNSRISLIEEIPKDAVSTPIRNGSVKLRTVSEGETVHLLPKNMECHFNKQEKYEEEEE', 'length': 797, 'molWeight': 89288, 'crc64': '25AD3EA350EB4E85', 'md5': 'CDB9455358FB9B5496A1D047EF2A79FD'}, uniProtKBCrossReferences=[{'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CCD66487.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CZR14596.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CZR14597.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'EMBL', 'id': 'BX284606', 'properties': [{'key': 'ProteinId', 'value': 'CZR14608.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'PIR', 'id': 'T16631', 'properties': [{'key': 'EntryName', 'value': 'T16631'}]}, {'database': 'RefSeq', 'id': 'NP_001309670.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_001322613.1'}], 'isoformId': 'A0A131MCZ8-1'}, {'database': 'RefSeq', 'id': 'NP_001309671.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_001322614.1'}]}, {'database': 'RefSeq', 'id': 'NP_001309682.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_001322615.1'}]}, {'database': 'RefSeq', 'id': 'NP_508520.1', 'properties': [{'key': 'NucleotideSequenceId', 'value': 'NM_076119.3'}]}, {'database': 'AlphaFoldDB', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'SMR', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'IntAct', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Interactions', 'value': '1'}]}, {'database': 'STRING', 'id': '6239.C33D12.2', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'PaxDb', 'id': 'A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2a.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2a.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-1'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.2', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.2'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.3', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.3'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.4', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.4'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.5', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.5'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2b.6', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2b.6'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2c.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2c.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-3'}, {'database': 'EnsemblMetazoa', 'id': 'C33D12.2d.1', 'properties': [{'key': 'ProteinId', 'value': 'C33D12.2d.1'}, {'key': 'GeneId', 'value': 'WBGene00016343'}], 'isoformId': 'A0A131MCZ8-4'}, {'database': 'GeneID', 'id': '180591', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'KEGG', 'id': 'cel:CELE_C33D12.2', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'UCSC', 'id': 'M02F4.3', 'properties': [{'key': 'OrganismName', 'value': 'c. elegans'}]}, {'database': 'CTD', 'id': '180591', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'WormBase', 'id': 'C33D12.2a', 'properties': [{'key': 'ProteinId', 'value': 'CE51453'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-1'}, {'database': 'WormBase', 'id': 'C33D12.2b', 'properties': [{'key': 'ProteinId', 'value': 'CE04764'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-2'}, {'database': 'WormBase', 'id': 'C33D12.2c', 'properties': [{'key': 'ProteinId', 'value': 'CE51420'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-3'}, {'database': 'WormBase', 'id': 'C33D12.2d', 'properties': [{'key': 'ProteinId', 'value': 'CE51314'}, {'key': 'GeneId', 'value': 'WBGene00016343'}, {'key': 'GeneName', 'value': 'cnnm-3'}], 'isoformId': 'A0A131MCZ8-4'}, {'database': 'eggNOG', 'id': 'KOG2118', 'properties': [{'key': 'ToxonomicScope', 'value': 'Eukaryota'}]}, {'database': 'GeneTree', 'id': 'ENSGT00940000169533', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'OMA', 'id': 'MSTCHKI', 'properties': [{'key': 'Fingerprint', 'value': '-'}]}, {'database': 'OrthoDB', 'id': '1446644at2759', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'PRO', 'id': 'PR:A0A131MCZ8', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'Proteomes', 'id': 'UP000001940', 'properties': [{'key': 'Component', 'value': 'Chromosome X'}]}, {'database': 'Bgee', 'id': 'WBGene00016343', 'properties': [{'key': 'ExpressionPatterns', 'value': 'Expressed in larva and 3 other tissues'}]}, {'database': 'GO', 'id': 'GO:0016323', 'properties': [{'key': 'GoTerm', 'value': 'C:basolateral plasma membrane'}, {'key': 'GoEvidenceType', 'value': 'IDA:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000314', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0016021', 'properties': [{'key': 'GoTerm', 'value': 'C:integral component of membrane'}, {'key': 'GoEvidenceType', 'value': 'IEA:UniProtKB-KW'}]}, {'database': 'GO', 'id': 'GO:0043231', 'properties': [{'key': 'GoTerm', 'value': 'C:intracellular membrane-bounded organelle'}, {'key': 'GoEvidenceType', 'value': 'IBA:GO_Central'}], 'evidences': [{'evidenceCode': 'ECO:0000318', 'source': 'PubMed', 'id': '21873635'}]}, {'database': 'GO', 'id': 'GO:0005886', 'properties': [{'key': 'GoTerm', 'value': 'C:plasma membrane'}, {'key': 'GoEvidenceType', 'value': 'IBA:GO_Central'}], 'evidences': [{'evidenceCode': 'ECO:0000318', 'source': 'PubMed', 'id': '21873635'}]}, {'database': 'GO', 'id': 'GO:0022857', 'properties': [{'key': 'GoTerm', 'value': 'F:transmembrane transporter activity'}, {'key': 'GoEvidenceType', 'value': 'IBA:GO_Central'}], 'evidences': [{'evidenceCode': 'ECO:0000318', 'source': 'PubMed', 'id': '21873635'}]}, {'database': 'GO', 'id': 'GO:0008340', 'properties': [{'key': 'GoTerm', 'value': 'P:determination of adult lifespan'}, {'key': 'GoEvidenceType', 'value': 'IMP:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000315', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0045087', 'properties': [{'key': 'GoTerm', 'value': 'P:innate immune response'}, {'key': 'GoEvidenceType', 'value': 'HEP:WormBase'}], 'evidences': [{'evidenceCode': 'ECO:0007007', 'source': 'PubMed', 'id': '16968778'}]}, {'database': 'GO', 'id': 'GO:0010960', 'properties': [{'key': 'GoTerm', 'value': 'P:magnesium ion homeostasis'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0015693', 'properties': [{'key': 'GoTerm', 'value': 'P:magnesium ion transport'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:1905941', 'properties': [{'key': 'GoTerm', 'value': 'P:positive regulation of gonad development'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}, {'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}, {'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0040018', 'properties': [{'key': 'GoTerm', 'value': 'P:positive regulation of multicellular organism growth'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0040026', 'properties': [{'key': 'GoTerm', 'value': 'P:positive regulation of vulval development'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'GO', 'id': 'GO:0032026', 'properties': [{'key': 'GoTerm', 'value': 'P:response to magnesium ion'}, {'key': 'GoEvidenceType', 'value': 'IGI:UniProtKB'}], 'evidences': [{'evidenceCode': 'ECO:0000316', 'source': 'PubMed', 'id': '27564576'}]}, {'database': 'CDD', 'id': 'cd04590', 'properties': [{'key': 'EntryName', 'value': 'CBS_pair_CorC_HlyC_assoc'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'Gene3D', 'id': '3.10.580.10', 'properties': [{'key': 'EntryName', 'value': '-'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'InterPro', 'id': 'IPR045095', 'properties': [{'key': 'EntryName', 'value': 'ACDP'}]}, {'database': 'InterPro', 'id': 'IPR000644', 'properties': [{'key': 'EntryName', 'value': 'CBS_dom'}]}, {'database': 'InterPro', 'id': 'IPR046342', 'properties': [{'key': 'EntryName', 'value': 'CBS_dom_sf'}]}, {'database': 'InterPro', 'id': 'IPR002550', 'properties': [{'key': 'EntryName', 'value': 'CNNM'}]}, {'database': 'InterPro', 'id': 'IPR044751', 'properties': [{'key': 'EntryName', 'value': 'Ion_transp-like_CBS'}]}, {'database': 'PANTHER', 'id': 'PTHR12064', 'properties': [{'key': 'EntryName', 'value': 'PTHR12064'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'Pfam', 'id': 'PF01595', 'properties': [{'key': 'EntryName', 'value': 'DUF21'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'SUPFAM', 'id': 'SSF54631', 'properties': [{'key': 'EntryName', 'value': 'SSF54631'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'PROSITE', 'id': 'PS51371', 'properties': [{'key': 'EntryName', 'value': 'CBS'}, {'key': 'MatchStatus', 'value': '2'}]}, {'database': 'PROSITE', 'id': 'PS51846', 'properties': [{'key': 'EntryName', 'value': 'CNNM'}, {'key': 'MatchStatus', 'value': '1'}]}], uniProtkbId='CNNM3_CAEEL'))" + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/A0B137')" ] }, - "execution_count": 42, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -564,16 +567,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "# uresources" + "uniprot = sources['UniProt']" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "complete_query = \"\"\"\n", + "PREFIX up: \n", + "SELECT ?protein ?gene\n", + "WHERE {\n", + " ?protein a up:Protein ;\n", + " up:reviewed true ;\n", + " up:encodedBy ?gene; .\n", + " ?gene prefLabel ?gene_label\n", + " \n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -582,17 +604,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "query in sparql SELECT ?id WHERE {?id type Protein;\n", - " up:reviewed ?v1 . \n", - " FILTER(?v1 = 'true'^^xsd:boolean)\n", - "}\n", "Submitted query:\n", " PREFIX up: \n", " PREFIX owl: \n", @@ -610,8 +628,7 @@ " up:reviewed ?v1 . \n", " FILTER(?v1 = 'true'^^xsd:boolean)\n", " } LIMIT 10\n", - "\n", - "amount of results = 10\n" + "\n" ] } ], @@ -623,76 +640,54 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "uniprot = sources['UniProt']" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'up': 'http://purl.uniprot.org/core/',\n", - " 'owl': 'http://www.w3.org/2002/07/owl#',\n", - " 'owl2xml': 'http://www.w3.org/2006/12/owl2-xml#',\n", - " 'swrlb': 'http://www.w3.org/2003/11/swrlb#',\n", - " 'protege': 'http://protege.stanford.edu/plugins/owl/protege#',\n", - " 'swrl': 'http://www.w3.org/2003/11/swrl#',\n", - " 'xsd': 'http://www.w3.org/2001/XMLSchema#',\n", - " 'skos': 'http://www.w3.org/2004/02/skos/core#',\n", - " 'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',\n", - " 'dc11': 'http://purl.org/dc/terms/',\n", - " 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',\n", - " 'foaf': 'http://xmlns.com/foaf/0.1/'}" + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0B137', _inner_sync=False)" ] }, - "execution_count": 29, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "uniprot._store.context.prefixes" + "proteins[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "query in sparql SELECT ?id WHERE {?id type Gene . \n", - " \n", - "}\n", - "amount of results = 10\n" - ] - } - ], + "outputs": [], + "source": [ + "uniprot = sources['UniProt']" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], "source": [ "genes = forge.search({'type': 'Gene'}, db_source='UniProt', limit=10)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id=Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/H5SQ95#gene-MD58301D33AF640374C84A4DA4CAF383BE6', _inner_sync=False, annotationScore=2.0, comments=[{'texts': [{'evidences': [{'evidenceCode': 'ECO:0000305', 'source': 'PubMed', 'id': '28087277'}], 'value': 'May function as a protein modifier covalently attached to lysine residues of substrate proteins. This may serve to target the modified proteins for degradation by proteasomes'}], 'commentType': 'FUNCTION'}, {'texts': [{'evidences': [{'evidenceCode': 'ECO:0000255', 'source': 'HAMAP-Rule', 'id': 'MF_02133'}], 'value': 'Belongs to the ubiquitin-like protein UBact family'}], 'commentType': 'SIMILARITY'}], entryAudit={'firstPublicDate': '2017-10-25', 'lastAnnotationUpdateDate': '2022-05-25', 'lastSequenceUpdateDate': '2012-04-18', 'entryVersion': 15, 'sequenceVersion': 1}, entryType='UniProtKB reviewed (Swiss-Prot)', extraAttributes={'countByCommentType': {'FUNCTION': 1, 'SIMILARITY': 1}, 'countByFeatureType': {'Chain': 1, 'Region': 1, 'Compositional bias': 1, 'Cross-link': 1}, 'uniParcId': 'UPI00024DBA4F'}, features=[{'type': 'Chain', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Prokaryotic ubiquitin-like protein UBact', 'featureId': 'PRO_0000441772'}, {'type': 'Region', 'location': {'start': {'value': 1, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Disordered', 'evidences': [{'evidenceCode': 'ECO:0000256', 'source': 'SAM', 'id': 'MobiDB-lite'}]}, {'type': 'Compositional bias', 'location': {'start': {'value': 26, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Basic and acidic residues', 'evidences': [{'evidenceCode': 'ECO:0000256', 'source': 'SAM', 'id': 'MobiDB-lite'}]}, {'type': 'Cross-link', 'location': {'start': {'value': 56, 'modifier': 'EXACT'}, 'end': {'value': 56, 'modifier': 'EXACT'}}, 'description': 'Isoglutamyl lysine isopeptide (Glu-Lys) (interchain with K-? in acceptor proteins)', 'evidences': [{'evidenceCode': 'ECO:0000305'}]}], genes=[{'geneName': {'evidences': [{'evidenceCode': 'ECO:0000303', 'source': 'PubMed', 'id': '28087277'}], 'value': 'ubact'}, 'orfNames': [{'evidences': [{'evidenceCode': 'ECO:0000312', 'source': 'EMBL', 'id': 'BAL58331.1'}], 'value': 'HGMM_OP1C026'}]}], keywords=[{'id': 'KW-1017', 'category': 'PTM', 'name': 'Isopeptide bond'}, {'id': 'KW-0833', 'category': 'Biological process', 'name': 'Ubl conjugation pathway'}], organism={'scientificName': 'Acetothermus autotrophicum', 'taxonId': 1446466, 'lineage': ['Bacteria', 'Candidatus Bipolaricaulota', 'Candidatus Acetothermum']}, primaryAccession='H5SQ95', proteinDescription={'recommendedName': {'fullName': {'evidences': [{'evidenceCode': 'ECO:0000303', 'source': 'PubMed', 'id': '28087277'}], 'value': 'Prokaryotic ubiquitin-like protein UBact'}}}, proteinExistence='3: Inferred from homology', references=[{'citation': {'id': '22303444', 'citationType': 'journal article', 'authors': ['Takami H.', 'Noguchi H.', 'Takaki Y.', 'Uchiyama I.', 'Toyoda A.', 'Nishi S.', 'Chee G.J.', 'Arai W.', 'Nunoura T.', 'Itoh T.', 'Hattori M.', 'Takai K.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '22303444'}, {'database': 'DOI', 'id': '10.1371/journal.pone.0030559'}], 'title': 'A deeply branching thermophilic bacterium with an ancient acetyl-CoA pathway dominates a subsurface ecosystem.', 'publicationDate': '2012', 'journal': 'PLoS ONE', 'firstPage': 'E30559', 'lastPage': 'E30559', 'volume': '7'}, 'referencePositions': ['NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]']}, {'citation': {'id': '28087277', 'citationType': 'journal article', 'authors': ['Lehmann G.', 'Udasin R.G.', 'Livneh I.', 'Ciechanover A.'], 'citationCrossReferences': [{'database': 'PubMed', 'id': '28087277'}, {'database': 'DOI', 'id': '10.1016/j.bbrc.2017.01.037'}], 'title': 'Identification of UBact, a ubiquitin-like protein, along with other homologous components of a conjugation system and the proteasome in different gram-negative bacteria.', 'publicationDate': '2017', 'journal': 'Biochem. Biophys. Res. Commun.', 'firstPage': '946', 'lastPage': '950', 'volume': '483'}, 'referencePositions': ['PREDICTED FUNCTION']}], sequence={'value': 'MPERIVKPMPQDPVTKPGDEGPRTPNVPKPDTERLLERMRRVDPRQAQRYRQRSGE', 'length': 56, 'molWeight': 6594, 'crc64': '323CA0429FBA02D0', 'md5': 'EF137CE5E7D3D2D873E21B0ECFB25FF6'}, uniProtKBCrossReferences=[{'database': 'EMBL', 'id': 'AP011800', 'properties': [{'key': 'ProteinId', 'value': 'BAL58331.1'}, {'key': 'Status', 'value': '-'}, {'key': 'MoleculeType', 'value': 'Genomic_DNA'}]}, {'database': 'AlphaFoldDB', 'id': 'H5SQ95', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'SMR', 'id': 'H5SQ95', 'properties': [{'key': 'Description', 'value': '-'}]}, {'database': 'GO', 'id': 'GO:0031386', 'properties': [{'key': 'GoTerm', 'value': 'F:protein tag'}, {'key': 'GoEvidenceType', 'value': 'IEA:UniProtKB-UniRule'}]}, {'database': 'HAMAP', 'id': 'MF_02133', 'properties': [{'key': 'EntryName', 'value': 'UBact'}, {'key': 'MatchStatus', 'value': '1'}]}, {'database': 'InterPro', 'id': 'IPR037543', 'properties': [{'key': 'EntryName', 'value': 'UBact'}]}], uniProtkbId='UBACT_ACEAU'), _inner_sync=False)" + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A1KV59#gene-MD5C1D876E2DCF48FF8A37D8833A1B756B4', _inner_sync=False)" ] }, - "execution_count": 31, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -708,15 +703,6 @@ "# Save in BBP KG (Nexus)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# forge.register(resources)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -733,7 +719,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -750,7 +736,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -778,7 +764,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -1102,7 +1088,7 @@ "9 Sim1-Cre " ] }, - "execution_count": 35, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1114,249 +1100,6 @@ "forge.as_dataframe(reshaped_data)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Download" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " _download\n", - " DownloadingError: Downloading from neurosciencegraph/datamodels:404, message='Not Found', url=URL('https://staging.nise.bbp.epfl.ch/nexus/v1/files/neurosciencegraph/datamodels/665642bf-4d60-45be-853b-ace965f0057f')\n", - "\n" - ] - } - ], - "source": [ - "dirpath = \"./downloaded/\"\n", - "forge.download(data, \"distribution.contentUrl\", dirpath)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Try query" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Submitted query:\n", - " \n", - " # PREFIXES\n", - " SELECT ?id WHERE {\n", - " ?id a nsg:DetailedCircuit\n", - " } LIMIT 100\n", - "\n", - " _sparql\n", - " QueryingError: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/views/neurosciencegraph/datamodels/https%3A%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FdefaultSparqlIndex/sparql\n", - "\n" - ] - } - ], - "source": [ - "mquery = \"\"\"\n", - "# PREFIXES\n", - "SELECT ?id WHERE {\n", - " ?id a nsg:DetailedCircuit\n", - "} LIMIT 100\n", - "\"\"\"\n", - "\n", - "forge.sparql(mquery, debug=True, rewrite=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Submitted query:\n", - " \n", - " # PREFIXES\n", - " SELECT ?id WHERE {\n", - " ?id a nsg:DetailedCircuit\n", - " } LIMIT 100\n", - "\n", - " _sparql\n", - " QueryingError: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/views/neurosciencegraph/datamodels/https%3A%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FdefaultSparqlIndex/sparql\n", - "\n" - ] - } - ], - "source": [ - "forge.sparql(mquery, debug=True, rewrite=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_region = forge.retrieve(\"http://api.brain-map.org/api/v2/data/Structure/614454282\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"\"\"\n", - "SELECT ?id ?label ?preflabel\n", - "WHERE {\n", - " ?id subClassOf \"nsg:BrainRegion\" .\n", - "}\n", - " \"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Submitted query:\n", - " PREFIX bmc: \n", - " PREFIX bmo: \n", - " PREFIX commonshapes: \n", - " PREFIX datashapes: \n", - " PREFIX dc: \n", - " PREFIX dcat: \n", - " PREFIX dcterms: \n", - " PREFIX mba: \n", - " PREFIX nsg: \n", - " PREFIX nxv: \n", - " PREFIX oa: \n", - " PREFIX obo: \n", - " PREFIX owl: \n", - " PREFIX prov: \n", - " PREFIX rdf: \n", - " PREFIX rdfs: \n", - " PREFIX schema: \n", - " PREFIX sh: \n", - " PREFIX shsh: \n", - " PREFIX skos: \n", - " PREFIX vann: \n", - " PREFIX void: \n", - " PREFIX xml: \n", - " PREFIX xsd: \n", - " PREFIX : \n", - " \n", - " SELECT ?id ?label ?preflabel\n", - " WHERE {\n", - " ?id rdfs:subClassOf \"nsg:BrainRegion\" .\n", - " }\n", - " LIMIT 100\n", - "\n" - ] - } - ], - "source": [ - "bregions = forge.sparql(query, limit=100, debug=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 67, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bregions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'context': 'https://neuroshapes.org',\n", - " 'id': 'http://api.brain-map.org/api/v2/data/Structure/614454282',\n", - " 'type': 'Class',\n", - " 'atlas_id': 966,\n", - " 'color_hex_triplet': '1F9D5A',\n", - " 'graph_order': 20,\n", - " 'hemisphere_id': 3,\n", - " 'identifier': '614454282',\n", - " 'isDefinedBy': 'http://bbp.epfl.ch/neurosciencegraph/ontologies/core/brainregion',\n", - " 'isPartOf': 'mba:943',\n", - " 'notation': 'MOp2',\n", - " 'rdfs:label': 'Primary motor area, Layer 2',\n", - " 'skos:prefLabel': 'Primary motor area, Layer 2',\n", - " 'st_level': 11,\n", - " 'subClassOf': ['nsg:BrainRegion'],\n", - " '_last_action': Action(error=None, message=None, operation='retrieve', succeeded=True),\n", - " '_validated': False,\n", - " '_inner_sync': True,\n", - " '_synchronized': True,\n", - " '_store_metadata': {'id': 'mba:614454282',\n", - " '_constrainedBy': 'https://neuroshapes.org/dash/ontologyentity',\n", - " '_createdAt': '2022-05-27T07:24:49.109Z',\n", - " '_createdBy': 'https://staging.nise.bbp.epfl.ch/nexus/v1/realms/serviceaccounts/users/service-account-brain-modeling-ontology-ci-cd',\n", - " '_deprecated': False,\n", - " '_incoming': 'https://staging.nise.bbp.epfl.ch/nexus/v1/resources/neurosciencegraph/datamodels/datashapes:ontologyentity/http:%2F%2Fapi.brain-map.org%2Fapi%2Fv2%2Fdata%2FStructure%2F614454282/incoming',\n", - " '_outgoing': 'https://staging.nise.bbp.epfl.ch/nexus/v1/resources/neurosciencegraph/datamodels/datashapes:ontologyentity/http:%2F%2Fapi.brain-map.org%2Fapi%2Fv2%2Fdata%2FStructure%2F614454282/outgoing',\n", - " '_project': 'https://staging.nise.bbp.epfl.ch/nexus/v1/projects/neurosciencegraph/datamodels',\n", - " '_rev': 8,\n", - " '_schemaProject': 'https://staging.nise.bbp.epfl.ch/nexus/v1/projects/neurosciencegraph/datamodels',\n", - " '_self': 'https://staging.nise.bbp.epfl.ch/nexus/v1/resources/neurosciencegraph/datamodels/datashapes:ontologyentity/http:%2F%2Fapi.brain-map.org%2Fapi%2Fv2%2Fdata%2FStructure%2F614454282',\n", - " '_updatedAt': '2022-06-17T22:48:31.217Z',\n", - " '_updatedBy': 'https://staging.nise.bbp.epfl.ch/nexus/v1/realms/serviceaccounts/users/service-account-nexus-sa'}}" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "example_region.__dict__" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index c2b43012..dbf757ad 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -15,14 +15,14 @@ from abc import ABC, abstractmethod import json from pathlib import Path -from typing import Any, Optional, Callable, Dict, List +from typing import Any, Optional, Callable, Dict, List, Union from kgforge.core import Resource from kgforge.core.commons.context import Context from kgforge.core.archetypes import Mapping, Model from kgforge.core.commons.attributes import repr_class from kgforge.core.commons.exceptions import ConfigurationError -from kgforge.core.commons.execution import not_supported +from kgforge.core.commons.dictionaries import with_defaults from kgforge.core.commons.imports import import_class from kgforge.core.commons.dictionaries import with_defaults @@ -35,9 +35,10 @@ class Database(ABC): # POLICY Implementations should not add methods but private functions in the file. # POLICY Implementations should pass tests/specializations/databases/test_databases.py. - def __init__(self, source: str, **config) -> None: + def __init__(self, forge : Optional["KnowledgeGraphForge"], source: str, **config) -> None: # POLICY Resolver data access should be lazy, unless it takes less than a second. # POLICY There could be data caching but it should be aware of changes made in the source. + self._forge: Optional["KnowledgeGraphForge"] = forge # Model model_config = config.pop("model") if model_config.get('origin') == 'directory': @@ -51,9 +52,10 @@ def __init__(self, source: str, **config) -> None: with open(context_path, 'r') as jfile: context_file = json.load(jfile) self.context = Context(context_file, iri) + if 'model_context' not in config: + config['model_context'] = self.context except Exception: self.context = None - config['model_context'] = self.context elif model_config["origin"] == "store": with_defaults( model_config, @@ -65,7 +67,8 @@ def __init__(self, source: str, **config) -> None: model_name = model_config.pop("name") model = import_class(model_name, "models") self._model: Model = model(**model_config) - config['model_context'] = self._model.context() + if 'model_context' not in config: + config['model_context'] = self._model.context() else: raise NotImplementedError('DB Model not yet implemented.') self.source: str = source @@ -74,10 +77,6 @@ def __init__(self, source: str, **config) -> None: def __repr__(self) -> str: return repr_class(self) - def datatypes(self): - # TODO: add other datatypes used, for instance, inside the mappings - return self.mappings(pretty=False).keys() - def _mappings(self) -> Dict[str, List[str]]: try: dirpath = Path(self._dirpath, "mappings") @@ -107,6 +106,42 @@ def mapping(self, entity: str, type: Callable) -> Mapping: except AttributeError: raise ConfigurationError('No directory path was found from the configuration.') + def map_resources(self, resources : Union[List[Resource], Resource], + resource_type : Optional[str] = None) -> Optional[Union[Resource, List[Resource]]]: + datatypes = self.types + mappings = self.mappings() + mapped_resources = [] + for resource in resources: + if resource_type is None: + try: + resource_type = resource.type + except AttributeError: + mapped_resources.append(resource) + if resource_type in datatypes: + mapping_class : Mapping = import_class(mappings[resource_type][0], "mappings") + mapping = self.mapping(resource_type, mapping_class) + mapped_resources.append(self._forge.map(self._forge.as_json(resource), mapping)) + else: + mapped_resources.append(resource) + return mapped_resources + + def datatypes(self): + # TODO: add other datatypes used, for instance, inside the mappings + return list(self.mappings().keys()) + + @abstractmethod + def search(self, resolvers, *filters, **params) -> Resource: + pass + + @abstractmethod + def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, + offset: Optional[int] = None,**params) -> Resource: + pass + + @abstractmethod + def elastic(self, **params) -> Resource: + pass + @property @abstractmethod def health(self) -> Callable: @@ -127,6 +162,8 @@ def _initialize_service(self, source: str, **source_config) -> Any: return self._service_from_web_service(source, **source_config) elif origin == "store": store = import_class(source, "stores") + if source != 'DemoStore': + source_config['store_context'] = self.context return self._service_from_store(store, **source_config) else: raise ConfigurationError(f"unrecognized DataBase origin '{origin}'") diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index bdd5a3ff..184efe8b 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -600,7 +600,7 @@ def replace(match: Match) -> str: return f"{pfx}\n{qr}" -def build_construct_query(data, context): +def resources_from_construct_query(data, context): subject_triples = {} for r in data["results"]["bindings"]: subject = r["subject"]["value"] diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 819fe989..ba4bd15a 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -524,21 +524,23 @@ def db_sources(self, mappings: Optional[List[str]] = None, :param pretty: a boolean :return: Optional[List[str]] """ - if mappings is not None: + if mappings is None: + sources = self._db_sources + else: sources = {} if isinstance(mappings, list): for type in mappings: for db in self._db_sources: - if type in self._db_sources[db].datatypes(): + types = self._db_sources[db].datatypes() + if type in types: sources[db] = self._db_sources[db] else: for db in self._db_sources: - if mappings in self._db_sources[db].datatypes(): + types = self._db_sources[db].datatypes() + if mappings in types: sources[db] = self._db_sources[db] if not sources: - print("No Database sources were found for the given datatype(s)") - else: - sources = self._db_sources + print("No Database sources were found for the given type(s)") if pretty: print(*["Available Database sources:", *sources], sep="\n") else: @@ -672,7 +674,10 @@ def search(self, *filters, **params) -> List[Resource]: ) db_source = params.pop('db_source', None) if db_source: - return self._db_sources[db_source].search(resolvers, *filters, **params) + if db_source in self.db_sources(): + return self._db_sources[db_source].search(resolvers, *filters, **params) + else: + raise AttributeError('Selected database was not declared within forge.') else: return self._store.search(resolvers, *filters, **params) @@ -697,7 +702,10 @@ def sparql( """ db_source = params.pop('db_source', None) if db_source: - return self._db_sources[db_source].sparql(query, debug, limit, offset, **params) + if db_source in self.db_sources(): + return self._db_sources[db_source].sparql(query, debug, limit, offset, **params) + else: + raise AttributeError('Selected database was not declared within forge.') else: return self._store.sparql(query, debug, limit, offset, **params) @@ -720,7 +728,10 @@ def elastic( :return: List[Resource] """ if db_source: - return self._db_sources[db_source].elastic(query, debug, limit, offset) + if db_source in self.db_sources(): + return self._db_sources[db_source].elastic(query, debug, limit, offset) + else: + raise AttributeError('Selected database was not declared within forge.') else: return self._store.elastic(query, debug, limit, offset) @@ -992,7 +1003,7 @@ def get_model_context(self): def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], store_config : Optional[Dict[str, Dict[str, str]]], - model_config : Optional[Dict[str, Dict[str, str]]] + model_context: Context ) -> Union[Database, List[Database]]: names = all_config.keys() dbs = {} @@ -1001,17 +1012,15 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], origin = config.get('origin') if origin == 'store': source = config.get('source') - # Provide store and model configuration to the database sources - if "model" not in config: - config.update(model=deepcopy(model_config)) - # Complete configuration of the db store in case is the same - if source == store_config['name']: + # Reuse complete configuration of the store when Nexus is called + if source == store_config['name'] == 'BlueBrainNexus': store_copy = deepcopy(store_config) with_defaults(config, store_copy, "source", "name", store_copy.keys()) + config['model_context'] = model_context + config.update(origin=origin) config.update(origin=origin) - print('Configuration', config) config['name'] = name dbs[name] = StoreDatabase(self, **config) else: diff --git a/kgforge/specializations/databases/store_database.py b/kgforge/specializations/databases/store_database.py index 58227887..6e4953bb 100644 --- a/kgforge/specializations/databases/store_database.py +++ b/kgforge/specializations/databases/store_database.py @@ -18,11 +18,12 @@ import copy from typing import Callable, Optional, Union, Dict, List, Any -from kgforge.core.archetypes import Mapping, Store, Database -from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.core import Resource +from kgforge.core.archetypes import Store, Database from kgforge.core.commons.execution import not_supported -from kgforge.core.commons.dictionaries import with_defaults -from kgforge.core.commons.imports import import_class +from kgforge.core.wrappings.paths import FilterOperator +from kgforge.specializations.mappers.dictionaries import DictionaryMapper +from kgforge.specializations.stores.bluebrain_nexus import BlueBrainNexus class StoreDatabase(Database): @@ -31,13 +32,13 @@ class StoreDatabase(Database): _REQUIRED = ("name", "origin", "source", "model") - def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Database", + def __init__(self, forge: Optional["KnowledgeGraphForge"], **config) -> None: """ The properties defining the StoreDatabase are: + :param forge: To use forge utilities name: - REQUIRED - type: Database - REQUIRED origin: <'store'> - REQUIRED source: - REQUIRED bucket: @@ -52,10 +53,8 @@ def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Database """ self._check_properties(**config) self.name = config.pop('name') - self.type: str = type - self._forge: Optional["KnowledgeGraphForge"] = forge source = config.pop('source') - super().__init__(source, **config) + super().__init__(forge, source, **config) def _check_properties(self, **info): properties = info.keys() @@ -63,16 +62,43 @@ def _check_properties(self, **info): if r not in properties: raise ValueError(f'Missing {r} from the properties to define the DatabasSource') - def datatypes(self): + @property + def types(self): # TODO: add other datatypes used, for instance, inside the mappings return self.mappings().keys() - - def search(self, *filters, **params) -> Any: - self.service.search(*filters, **params) - def sparql(self, query: str, debug: bool, limit: int = None, offset: int = None, **params) -> Any: - self.service.sparql(query, debug, limit, offset, **params) + def search(self, resolvers, *filters, **params): + """Search within the database. + + :param keep_original: bool + """ + keep_original = params.pop('keep_original', True) + unmapped_resources = self.service.search(resolvers, *filters, **params) + if isinstance(self.service, BlueBrainNexus) or keep_original: + return unmapped_resources + else: + # Try to find the type of the resources within the filters + resource_type = type_from_filters(filters) + return self.map_resources(unmapped_resources, resource_type=resource_type) + + return resource_type + + def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, + offset: Optional[int] = None,**params): + """Use SPARQL within the database. + :param keep_original: bool + """ + keep_original = params.pop('keep_original', True) + unmapped_resources = self.service.sparql(query, debug, limit, offset, **params) + if keep_original: + return unmapped_resources + else: + return self.map_resources(unmapped_resources) + + def elastic(**params): + not_supported() + @staticmethod def _service_from_directory(dirpath: Path, **source_config) -> Any: not_supported() @@ -83,9 +109,19 @@ def _service_from_web_service(endpoint: str, **source_config) -> Any: @staticmethod def _service_from_store(store: Callable, **store_config) -> Store: - # Store. - print('store config', store_config) return store(**store_config) def health(self) -> Callable: not_supported() + +def type_from_filters(filters): + resource_type = None + if isinstance(filters[0], dict): + if 'type' in filters[0]: + resource_type = filters[0]['type'] + else: + for filter in filters: + if 'type' in filter.path and filter.operator is FilterOperator.EQUAL: + resource_type = filter.value + break + return resource_type \ No newline at end of file diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index 322696ac..92eaaeec 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -64,7 +64,7 @@ from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping from kgforge.specializations.stores.nexus.service import BatchAction, Service, _error_message -from kgforge.core.archetypes.store import build_construct_query +from kgforge.core.archetypes.store import resources_from_construct_query from kgforge.core.commons.es_query_builder import ESQueryBuilder class CategoryDataType(Enum): @@ -876,7 +876,7 @@ def _sparql(self, query: str) -> List[Resource]: _, q_comp = Query.parseString(query) if q_comp.name == "ConstructQuery": context = self.model_context or context - return build_construct_query(data, context) + return resources_from_construct_query(data, context) else: # SELECT QUERY results = data["results"]["bindings"] diff --git a/kgforge/specializations/stores/databases/service.py b/kgforge/specializations/stores/databases/service.py index 4a0d3eff..8a495bf4 100644 --- a/kgforge/specializations/stores/databases/service.py +++ b/kgforge/specializations/stores/databases/service.py @@ -34,17 +34,9 @@ from requests import HTTPError from kgforge.core import Resource -from kgforge.core.commons.actions import ( - Action, - collect_lazy_actions, - execute_lazy_actions, - LazyAction, -) from kgforge.core.commons.context import Context from kgforge.core.conversions.rdf import ( _from_jsonld_one, - _remove_ld_keys, - as_jsonld, recursive_resolve, ) from kgforge.core.wrappings.dict import wrap_dict @@ -56,6 +48,7 @@ def __init__( self, endpoint: str, model_context: Context, + store_context: Context, max_connection: int, searchendpoints: Dict, content_type: str, @@ -66,6 +59,7 @@ def __init__( self.endpoint = endpoint self.model_context = model_context self.context_cache: Dict = dict() + self.context = store_context self.max_connection = max_connection self.params = copy.deepcopy(params) @@ -90,13 +84,6 @@ def __init__( self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] self.sparql_endpoint["type"] = "sparql" - # The following code is for async to work on jupyter notebooks - try: - asyncio.get_event_loop() - nest_asyncio.apply() - except RuntimeError: - pass - def resolve_context(self, iri: str) -> Dict: if iri in self.context_cache: return self.context_cache[iri] @@ -108,51 +95,4 @@ def resolve_context(self, iri: str) -> Dict: else: document = context.document["@context"] self.context_cache.update({context_to_resolve: document}) - return document - - def to_resource( - self, payload: Dict, sync_metadata: bool = True, **kwargs - ) -> Resource: - # Use JSONLD context defined in Model if no context is retrieved from payload - # Todo: BlueBrainNexus store is not indexing in ES the JSONLD context, user provided context can be changed to Model defined one - data_context = deepcopy(payload.get("@context", self.model_context.iri if self.model_context else None)) - if not isinstance(data_context, list): - data_context = [data_context] - if self.store_context in data_context: - data_context.remove(self.store_context) - data_context = data_context[0] if len(data_context) == 1 else data_context - metadata = dict() - data = dict() - for k, v in payload.items(): - if k in self.metadata_context.terms.keys(): - metadata[k] = v - else: - data[k] = v - - if ( - self.model_context - and data_context is not None - and data_context == self.model_context.iri - ): - resolved_ctx = self.model_context.document["@context"] - elif data_context is not None: - resolved_ctx = recursive_resolve( - data_context, - self.resolve_context, - already_loaded=[self.store_local_context, self.store_context], - ) - else: - resolved_ctx = None - if resolved_ctx: - data["@context"] = resolved_ctx - resource = _from_jsonld_one(data) - resource.context = data_context - else: - resource = Resource.from_json(data) - - if len(metadata) > 0 and sync_metadata: - metadata.update(kwargs) - self.sync_metadata(resource, metadata) - if not hasattr(resource, "id") and kwargs and 'id' in kwargs.keys(): - resource.id = kwargs.get("id") - return resource + return document \ No newline at end of file diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py index 6376fea9..a7ebb5db 100644 --- a/kgforge/specializations/stores/sparql.py +++ b/kgforge/specializations/stores/sparql.py @@ -23,7 +23,7 @@ from kgforge.core import Resource from kgforge.core.archetypes import Resolver, Store -from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql, build_construct_query +from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql, resources_from_construct_query from kgforge.core.commons.context import Context from kgforge.core.wrappings.dict import DictWrapper from kgforge.specializations.stores.databases import Service @@ -44,10 +44,9 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, token: Optional[str] = None, versioned_id_template: Optional[str] = None, file_resource_mapping: Optional[str] = None, model_context: Optional[Context] = None, - searchendpoints: Optional[Dict] = None,) -> None: - print('inside SPARQL Store') + searchendpoints: Optional[Dict] = None, **store_config) -> None: super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, - model_context, searchendpoints) + model_context, searchendpoints, **store_config) # [C]RUD @@ -135,7 +134,6 @@ def search( # pass # if self.model_context is None: # raise ValueError("context model missing") - debug = params.get("debug", False) limit = params.get("limit", 100) offset = params.get("offset", None) @@ -182,9 +180,8 @@ def sparql( self, query: str, debug: bool, limit: int = None, offset: int = None, **params ) -> List[Resource]: rewrite = params.get("rewrite", True) - print('query in sparql', query) qr = ( - rewrite_sparql(query, self.context, self.service.metadata_context) + rewrite_sparql(query, self.context, self.service.context) if self.context is not None and rewrite else query ) @@ -209,7 +206,7 @@ def _sparql(self, query: str, limit: int, offset: int = None, **params) -> List[ _, q_comp = Query.parseString(query) if q_comp.name == "ConstructQuery": context = self.model_context or self.context - return build_construct_query(data, context) + return resources_from_construct_query(data, context) else: # SELECT QUERY results = data["results"]["bindings"] @@ -251,7 +248,6 @@ def _initialize_service( searchendpoints: Optional[Dict] = None, **store_config, ) -> Any: - print('initializing service') try: max_connection = store_config.pop("max_connection", 50) if max_connection <= 0: @@ -261,10 +257,11 @@ def _initialize_service( content_type = store_config.pop("Content-Type", "application/ld+json") accept = store_config.pop("Accept", "application/ld+json") params = store_config.pop("params", {}) + store_context = store_config.pop('store_context', None) except Exception as ve: raise ValueError(f"Store configuration error: {ve}") else: - return Service(endpoint=endpoint, model_context=self.model_context, max_connection=max_connection, + return Service(endpoint=endpoint, model_context=self.model_context, store_context=store_context, max_connection=max_connection, searchendpoints=searchendpoints, content_type=content_type, accept=accept, **params) @staticmethod From 79b44bc8603664fdba23fc24a6c182afe66aab52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 19 Oct 2022 18:20:07 +0200 Subject: [PATCH 16/30] Improved Protein mappings. --- .../prod-nexus-sources_progress.yml | 15 +- .../mappings/DictionaryMapping/Protein.hjson | 24 +- .../17 - Database-sources.ipynb | 463 +++++++++--------- kgforge/core/archetypes/database.py | 1 + 4 files changed, 256 insertions(+), 247 deletions(-) diff --git a/examples/configurations/database-sources/prod-nexus-sources_progress.yml b/examples/configurations/database-sources/prod-nexus-sources_progress.yml index 9e87a3ec..1c8fdccc 100644 --- a/examples/configurations/database-sources/prod-nexus-sources_progress.yml +++ b/examples/configurations/database-sources/prod-nexus-sources_progress.yml @@ -23,7 +23,6 @@ Store: deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" project_property: "https://bluebrain.github.io/nexus/vocabulary/project" max_connection: 50 - token: eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJleHAiOjE2NjYwMjkyNjEsImlhdCI6MTY2NjAxODIzNSwiYXV0aF90aW1lIjoxNjY2MDAwNDYxLCJqdGkiOiIxNjQwNTBkOS04OTAzLTQzMjItOWQyZi0xOGQ0ODg3MTYzYWEiLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJhdWQiOiJodHRwczovL3NsYWNrLmNvbSIsInN1YiI6ImY6MGZkYWRlZjctYjJiOS00OTJiLWFmNDYtYzY1NDkyZDQ1OWMyOmNnb256YWxlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYmJwLW5pc2Utc3RhZ2luZy1uZXh1cy1mdXNpb24iLCJub25jZSI6IjEwOWM0NDA5YjExZjRkNjI4YjM2NmM2N2MxOGU0MDk5Iiwic2Vzc2lvbl9zdGF0ZSI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImFjciI6IjAiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiYmJwLXBhbS1hdXRoZW50aWNhdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidXNlciIsImRlZmF1bHQtcm9sZXMtYmJwIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiaHR0cHM6Ly9zbGFjay5jb20iOnsicm9sZXMiOlsic2xhY2tfdXNlcl9yb2xlIl19fSwic2NvcGUiOiJvcGVuaWQgbmV4dXMgcHJvZmlsZSBsb2NhdGlvbiBlbWFpbCIsInNpZCI6ImE4OGU5OWI4LTdmNDQtNGVmNS05YzQ2LWU0ZGJhZmEzZTJhYSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IEVzcGlub3phIiwiZ3JvdXBzIjpbIi9iYnAtZGV2LXByb2oxMDkiLCIvYmJwLWRrZS1kZXYiLCIvYmJwLWRldi1wcm9qNjQiLCIvYmJwLWRldi1wcm9qMTE2IiwiL2JicC1vdS1ka2UiLCIvYmJwLWRldi1wcm9qODQiLCIvYmJwLWRrZS1zdGFnaW5nIiwiL2JicC11c2VyLXByb2ozOSIsIi9iYnAtdXNlci1wcm9qNjQiLCIvYmJwLW91LW5ldXJvaW5mb3JtYXRpY3MiLCIvYmJwLXVzZXItcmVsZWFzZS1sMiIsIi9iYnAtb3UtbmV4dXMiLCIvYmJwLXVzZXItY29yZWJsdXJvbiIsIi9iYnAtZGV2LXByb2oxNDEiLCIvYmJwLWRldi1wcm9qMTE1IiwiL2JicC1vdS1zaW11bGF0aW9ubmV1cm9zY2llbmNlIiwiL2JicC1kcy1leHBlcmltZW50IiwiL2JicC1zdmMtbHh2aXoiLCIvYmJwLXVzZXItcHJvajEwNiIsIi9iYnAtc3ZjLXZpeiIsIi9iYnAtdXNlci1yZWxlYXNlLWwxIiwiL2JicC1kZXYtcHJvajU1IiwiL2JicC1kZXYtcHJvajEzNCIsIi9iYnAtZnVsbCIsIi9iYnAtc3ZjLWdpdGxhYiIsIi9iYnAtY29sbGFiLWxpbmthYmxlLWFjY291bnRzIiwiL2JicC1kZXYtcmVmaW5lbWVudHByb3AiLCIvYmJwLWRldi1wcm9qODMiLCIvYmJwLWl0Yy1kYXRhaW50ZWdyYXRpb24iLCIvYmJwLWRldi1wcm9qMTA1IiwiL2JicC1zdmMtYmc0IiwiL2JicC1kZXYtcHJvajgyIiwiL2JicC1zcnYtbGluc3J2MiIsIi9iYnAtdXNlci1wcm9qOTQiLCIvYmJwLWRldi1wcm9qNzIiLCIvYmJwLXVzZXItcHJvajEyMyIsIi9iYnAtZHMtaHBjIiwiL2JicC1ka2UtcHJvZHVjdGlvbiIsIi9iYnAtdXNlci1wcm9qMTE2IiwiL2JicC1kZXYtcHJvajEzNiIsIi9iYnAtdXNlci1yZWxlYXNlLWwwIiwiL2JicF9jb25mbHVlbmNlIiwiL2JicC1vdS1lcGZsIiwiL2JicC11c2VyLXByb2oxMTQiLCIvYmJwLXN2Yy1ocGMiLCIvYmJwLXVzZXItcHJvajQyIiwiL2JicC1zdmMtY29kZSIsIi9iYnAtZGV2LXByb2oxMzIiLCIvYmJwLXN2Yy1zbGFjayIsIi9iYnAtbmV4dXMtZGV2IiwiL2JicC1zdGFmZiIsIi9iYnAtZGV2LWhicHZpeiIsIi9iYnAtZGV2LXByb2plY3Rwcm9wIiwiL2JicC1zdmMta3ViZXJuZXRlcyIsIi9iYnAtc3ZjLXN0YXR1cyIsIi9iYnAtcHVibGljYXRpb25zIiwiL2JicC11c2VyLXByb2ozIl0sImxvY2F0aW9uIjoiQjEgMyAyODQuMDUyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2dvbnphbGUiLCJnaXZlbl9uYW1lIjoiQ3Jpc3RpbmEgRWxpemFiZXRoIEdvbnphbGV6IiwiZmFtaWx5X25hbWUiOiJFc3Bpbm96YSIsImVtYWlsIjoiY3Jpc3RpbmEuZ29uemFsZXplc3Bpbm96YUBlcGZsLmNoIn0.XHZ55pSp2zRQcgcvsXHYWS2adj12K411PtQaZesiABtKH9tgsjz0dgY_4ijneBHdHd1zgmqNRFqnHHd78EMD7XoHtmbpZOI5h1t12wK2YbI35mOwLfS1RcRnT6kKMuiFe5m4AzW1XWwHn9AcO_xeZTbixL2vqPLfdPg1jzk5uW-rgs3Sg-77Kr3LEBVTzVSHSokhtTqjHNY2EQlBljmpJpGCno-6keLX5LrNERFek303oJrAIgvfGSCcpKoHYYWCPiReD4-cjd8l-zMCVrfxCeuRhihVSpNGgPX67F4O7u8EnU_EMeA39eg9HSp5KahhrGwecCzqPMVg4R37cysz_Q versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson @@ -66,17 +65,25 @@ Databases: endpoint: "https://sparql.uniprot.org/sparql" model: origin: directory - source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/UniProt + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/UniProt context: iri: "https://bbp.epfl.ch/jsonldcontext/db/uniprot" bucket: jsonld_context.json - NeuroElectro: origin: store source: BlueBrainNexus bucket: bbp/neuroelectro model: origin: directory - source: /Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/NeuroElectro + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/NeuroElectro + context: + bucket: jsonld_context.json + MouseLight: + origin: store + source: BlueBrainNexus + bucket: dke/kgforge + model: + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/MouseLight context: bucket: jsonld_context.json \ No newline at end of file diff --git a/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson b/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson index 1b637039..d5b6aee1 100644 --- a/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson +++ b/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson @@ -1,20 +1,18 @@ { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/proteins/4f5b79f8-1abe-48e0-95d1-e017d621b058" + "id": forge.format('identifier', "proteins", x.id.split('/')[-1]) "type": [ "Entity" "Protein" ] - "encodedBy": { #genes - "id": "https://bbp.epfl.ch/neurosciencegraph/data/genes/98d04429-b003-4fc4-9e66-817546fd2f5f" - "type": "Gene" + "encodedBy": { + "id": x.gene + "label": x.gene_label } - "identifier": x.identifier #primaryAccession - "name": x.name #proteinDescription/recommendedName/fullName - "label": x.label #proteinDescription/recommendedName/fullName/value - "altLabel": x.altlabel #proteinDescription/alternativeNames/0/fullName/value - "structureAvailable": x.hasstructure ??? - "subject": x.subject #resolve organism/scientificName or taxonID - #optional - "sequence": x.sequence #sequence/value - "references": ... + "identifier": { + "propertyID": "UniProtKB" + "value": x.id.split('/')[-1] + } + "name": f"Protein {x.id.split('/')[-1]} from UniProtKB" + "label": x.label + "subject": forge.resolve(x.subject, scope='ontology') if forge.resolve(x.subject, scope='ontology') is not None else {"label": x.subject} } \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index b7b35a7b..74f73b9a 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -413,68 +413,59 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/72136\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", + " abstract: We investigated the effects of muscarinic acetylcholine receptor (mAChR) activation on GABAergic synaptic transmission in rat hippocampal neurons. Current-clamp recordings revealed that methacholine produced membrane depolarization and action potential firing. Methacholine augmented the bicuculline-sensitive and GABA(A) -mediated frequency of spontaneous inhibitory postsynaptic currents (sIPSCs); the action of methacholine had a slow onset and longer duration. The increase in methacholine-evoked sIPSCs was completely inhibited by atropine and was insensitive to glutamatergic receptor blockers. Interestingly, methacholine action was not inhibited by intracellular perfusion with GDP-β-S, suggesting that muscarinic effects on membrane excitability and sIPSC frequency are mainly presynaptic. McN-A-343 and pirenzepine, selective agonist and antagonist of the m1 mAChR subtype, respectively, neither enhanced sIPSCs nor inhibited the methacholine effect. However, the m3-m5 mAChR antagonist 4-DAMP, and the m2-m4 mAChR antagonist himbacine inhibited the methacholine effect. U73122, an IP(3) production inhibitor, and 2APB, an IP(3) receptor blocker, drastically decreased the methacholine effect. Recording of miniature events revealed that besides the effect exerted by methacholine on membrane firing properties and sIPSC frequency, muscarinic receptors also enhanced the frequency of mIPSCs with no effect on their amplitude, possibly modulating the molecular machinery subserving vesicle docking and fusion and suggesting a tight colocalization at the active zone of the presynaptic terminals. These data strongly suggest that by activating presynaptic m2, m3, m4 and m5 mAChRs, methacholine can increase membrane excitability and enhance efficiency in the GABA release machinery, perhaps through a mechanism involving the release of calcium from the endoplasmic reticulum.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Henderson\n", - " givenName: Z\n", + " familyName: González\n", + " givenName: J C\n", " }\n", " {\n", " type: Person\n", - " familyName: Morris\n", - " givenName: N P\n", + " familyName: Albiñana\n", + " givenName: E\n", " }\n", " {\n", " type: Person\n", - " familyName: Grimwood\n", + " familyName: Baldelli\n", " givenName: P\n", " }\n", " {\n", " type: Person\n", - " familyName: Fiddler\n", - " givenName: G\n", + " familyName: García\n", + " givenName: A G\n", " }\n", " {\n", " type: Person\n", - " familyName: Yang\n", - " givenName: H W\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Appenteng\n", - " givenName: K\n", + " familyName: Hernández-Guijo\n", + " givenName: J M\n", " }\n", " ]\n", - " datePublished: 2001-2-12\n", + " datePublished: 2011-1\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 11169477\n", - " }\n", - " {\n", - " propertyID: doi\n", - " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", + " value: 21091801\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0021-9967\n", - " name: The Journal of comparative neurology\n", + " issn: 0953-816X\n", + " name: The European journal of neuroscience\n", " }\n", - " name: article_91941\n", - " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", - " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", - " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " name: article_72136\n", + " sameAs: http://onlinelibrary.wiley.com/doi/10.1111/j.1460-9568.2010.07475.x/abstract;jsessionid=E294450D87316113388F9E234F3F8A39.d02t01\n", + " title: Presynaptic muscarinic receptor subtypes involved in the enhancement of spontaneous GABAergic postsynaptic currents in hippocampal neurons.\n", + " url: http://onlinelibrary.wiley.com/doi/10.1111/j.1460-9568.2010.07475.x/abstract;jsessionid=E294450D87316113388F9E234F3F8A39.d02t01\n", "}\n" ] } @@ -566,36 +557,15 @@ ] }, { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "uniprot = sources['UniProt']" - ] - }, - { - "cell_type": "code", - "execution_count": 24, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "complete_query = \"\"\"\n", - "PREFIX up: \n", - "SELECT ?protein ?gene\n", - "WHERE {\n", - " ?protein a up:Protein ;\n", - " up:reviewed true ;\n", - " up:encodedBy ?gene; .\n", - " ?gene prefLabel ?gene_label\n", - " \n", - "}\n", - "\"\"\"" + "## Use Filters to search" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -604,7 +574,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -633,14 +603,13 @@ } ], "source": [ - "\n", "\n", "proteins = forge.search({'type': 'Protein', 'up:reviewed': True}, db_source='UniProt', limit=10, debug=True)" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -649,7 +618,7 @@ "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0B137', _inner_sync=False)" ] }, - "execution_count": 27, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -658,9 +627,16 @@ "proteins[0]" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Map resources" + ] + }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -669,31 +645,80 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "complete_query = \"\"\"\n", + "PREFIX up: \n", + "SELECT ?id ?gene ?label ?subject ?gene_label\n", + "WHERE {\n", + " ?id a up:Protein ;\n", + " up:reviewed true ;\n", + " up:encodedBy ?gene ;\n", + " up:recommendedName / up:fullName ?label ;\n", + " up:organism / up:scientificName ?subject .\n", + " ?gene skos:prefLabel ?gene_label . \n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ - "genes = forge.search({'type': 'Gene'}, db_source='UniProt', limit=10)" + "raw_proteins = uniprot.sparql(complete_query)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, + "outputs": [], + "source": [ + "new_resource = uniprot.map_resources(raw_proteins[0], 'Protein')" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A1KV59#gene-MD5C1D876E2DCF48FF8A37D8833A1B756B4', _inner_sync=False)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/proteins/P0DJN9\",\n", + " \"@type\": [\n", + " \"Entity\",\n", + " \"Protein\"\n", + " ],\n", + " \"encodedBy\": {\n", + " \"@id\": \"http://purl.uniprot.org/uniprot/P0DJN9#gene-MD5A00DD99270221B359AB0AE338E423668\",\n", + " \"label\": \"acsF\"\n", + " },\n", + " \"identifier\": {\n", + " \"propertyID\": \"UniProtKB\",\n", + " \"value\": \"P0DJN9\"\n", + " },\n", + " \"name\": \"Protein P0DJN9 from UniProtKB\",\n", + " \"label\": \"Aerobic magnesium-protoporphyrin IX monomethyl ester [oxidative] cyclase\",\n", + " \"subject\": {\n", + " \"label\": \"Rubrivivax gelatinosus\"\n", + " }\n", + " }\n", + "]\n" + ] } ], "source": [ - "genes[0]" + "\n", + "print(json.dumps(forge.as_jsonld(new_resource), indent=4))" ] }, { @@ -719,7 +744,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -791,18 +816,16 @@ " id\n", " brainLocation.brainRegion.id\n", " brainLocation.brainRegion.label\n", + " brainLocation.layer\n", " contribution.type\n", " contribution.agent.id\n", " contribution.agent.type\n", - " contribution.agent.label\n", " distribution.contentUrl\n", " distribution.encodingFormat\n", " distribution.name\n", " name\n", + " subject.id\n", " subject.type\n", - " subject.species.id\n", - " subject.species.label\n", - " subject.strain.label\n", " \n", " \n", " \n", @@ -810,181 +833,161 @@ " 0\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Primary motor area Layer 5\n", + " VISp5\n", + " 5\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1050.swc\n", - " AA1050\n", + " reconstruction.swc\n", + " Htr3a-Cre_NO152;Ai14-314467.03.02.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 1\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Primary somatosensory area mouth layer 5\n", + " VISp5\n", + " 5\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1049.swc\n", - " AA1049\n", + " reconstruction.swc\n", + " Pvalb-IRES-Cre;Ai14-185362.03.01.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 2\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Retrosplenial area ventral part layer 5\n", + " VISli6a\n", + " 6a\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1045.swc\n", - " AA1045\n", + " reconstruction.swc\n", + " Chrna2-Cre_OE25;Ai14(BT)-280154.04.01.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 3\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Parafascicular nucleus\n", + " MTG\n", + " 5\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1046.swc\n", - " AA1046\n", + " reconstruction.swc\n", + " H16.06.008.01.26.04\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 4\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Medial mammillary nucleus\n", + " VISp5\n", + " 5\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1048.swc\n", - " AA1048\n", + " reconstruction.swc\n", + " Rorb-IRES2-Cre-D;Ai14-168053.05.01.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 5\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Primary somatosensory area mouth layer 5\n", + " VISp6b\n", + " 6b\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1051.swc\n", - " AA1051\n", + " reconstruction.swc\n", + " Ctgf-2A-dgCre;Ai14(IVSCC)-230665.02.02.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 6\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", - " http://api.brain-map.org/api/v2/data/Structure...\n", - " Entorhinal area lateral part\n", + " http://api.brain-map.org/api/v2/data/Structure/33\n", + " VISp6a\n", + " 6a\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1047.swc\n", - " AA1047\n", + " reconstruction.swc\n", + " Nos1-CreERT2;Sst-IRES-FlpO;Ai65-304714.02.01.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 7\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Primary motor area Layer 6a\n", + " MTG\n", + " 3\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1043.swc\n", - " AA1043\n", + " reconstruction.swc\n", + " H16.06.010.01.03.04.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 8\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Medial mammillary nucleus\n", + " VISp4\n", + " 4\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1041.swc\n", - " AA1041\n", + " reconstruction.swc\n", + " Nr5a1-Cre;Ai14-187780.03.02.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", " 9\n", " https://bbp.epfl.ch/neurosciencegraph/data/neu...\n", " http://api.brain-map.org/api/v2/data/Structure...\n", - " Medial mammillary nucleus\n", + " VISp2/3\n", + " 2/3\n", " Contribution\n", - " https://www.grid.ac/institutes/grid.443970.d\n", + " https://www.grid.ac/institutes/grid.417881.3\n", " Organization\n", - " Janelia Research Campus\n", - " https://staging.nise.bbp.epfl.ch/nexus/v1/file...\n", + " https://staging.nexus.ocp.bbp.epfl.ch/v1/files...\n", " application/swc\n", - " AA1039.swc\n", - " AA1039\n", + " reconstruction.swc\n", + " Slc17a6-IRES-Cre;Ai14-190263.04.01.01\n", + " https://bbp.epfl.ch/neurosciencegraph/data/sub...\n", " Subject\n", - " http://purl.obolibrary.org/obo/NCBITaxon_10090\n", - " Mus musculus\n", - " Sim1-Cre\n", " \n", " \n", "\n", @@ -1010,82 +1013,82 @@ "3 http://api.brain-map.org/api/v2/data/Structure... \n", "4 http://api.brain-map.org/api/v2/data/Structure... \n", "5 http://api.brain-map.org/api/v2/data/Structure... \n", - "6 http://api.brain-map.org/api/v2/data/Structure... \n", + "6 http://api.brain-map.org/api/v2/data/Structure/33 \n", "7 http://api.brain-map.org/api/v2/data/Structure... \n", "8 http://api.brain-map.org/api/v2/data/Structure... \n", "9 http://api.brain-map.org/api/v2/data/Structure... \n", "\n", - " brainLocation.brainRegion.label contribution.type \\\n", - "0 Primary motor area Layer 5 Contribution \n", - "1 Primary somatosensory area mouth layer 5 Contribution \n", - "2 Retrosplenial area ventral part layer 5 Contribution \n", - "3 Parafascicular nucleus Contribution \n", - "4 Medial mammillary nucleus Contribution \n", - "5 Primary somatosensory area mouth layer 5 Contribution \n", - "6 Entorhinal area lateral part Contribution \n", - "7 Primary motor area Layer 6a Contribution \n", - "8 Medial mammillary nucleus Contribution \n", - "9 Medial mammillary nucleus Contribution \n", + " brainLocation.brainRegion.label brainLocation.layer contribution.type \\\n", + "0 VISp5 5 Contribution \n", + "1 VISp5 5 Contribution \n", + "2 VISli6a 6a Contribution \n", + "3 MTG 5 Contribution \n", + "4 VISp5 5 Contribution \n", + "5 VISp6b 6b Contribution \n", + "6 VISp6a 6a Contribution \n", + "7 MTG 3 Contribution \n", + "8 VISp4 4 Contribution \n", + "9 VISp2/3 2/3 Contribution \n", "\n", " contribution.agent.id contribution.agent.type \\\n", - "0 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "1 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "2 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "3 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "4 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "5 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "6 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "7 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "8 https://www.grid.ac/institutes/grid.443970.d Organization \n", - "9 https://www.grid.ac/institutes/grid.443970.d Organization \n", + "0 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "1 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "2 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "3 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "4 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "5 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "6 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "7 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "8 https://www.grid.ac/institutes/grid.417881.3 Organization \n", + "9 https://www.grid.ac/institutes/grid.417881.3 Organization \n", "\n", - " contribution.agent.label distribution.contentUrl \\\n", - "0 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "1 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "2 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "3 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "4 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "5 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "6 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "7 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "8 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", - "9 Janelia Research Campus https://staging.nise.bbp.epfl.ch/nexus/v1/file... \n", + " distribution.contentUrl \\\n", + "0 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "1 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "2 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "3 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "4 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "5 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "6 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "7 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "8 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", + "9 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", "\n", - " distribution.encodingFormat distribution.name name subject.type \\\n", - "0 application/swc AA1050.swc AA1050 Subject \n", - "1 application/swc AA1049.swc AA1049 Subject \n", - "2 application/swc AA1045.swc AA1045 Subject \n", - "3 application/swc AA1046.swc AA1046 Subject \n", - "4 application/swc AA1048.swc AA1048 Subject \n", - "5 application/swc AA1051.swc AA1051 Subject \n", - "6 application/swc AA1047.swc AA1047 Subject \n", - "7 application/swc AA1043.swc AA1043 Subject \n", - "8 application/swc AA1041.swc AA1041 Subject \n", - "9 application/swc AA1039.swc AA1039 Subject \n", + " distribution.encodingFormat distribution.name \\\n", + "0 application/swc reconstruction.swc \n", + "1 application/swc reconstruction.swc \n", + "2 application/swc reconstruction.swc \n", + "3 application/swc reconstruction.swc \n", + "4 application/swc reconstruction.swc \n", + "5 application/swc reconstruction.swc \n", + "6 application/swc reconstruction.swc \n", + "7 application/swc reconstruction.swc \n", + "8 application/swc reconstruction.swc \n", + "9 application/swc reconstruction.swc \n", "\n", - " subject.species.id subject.species.label \\\n", - "0 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "1 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "2 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "3 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "4 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "5 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "6 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "7 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "8 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", - "9 http://purl.obolibrary.org/obo/NCBITaxon_10090 Mus musculus \n", + " name \\\n", + "0 Htr3a-Cre_NO152;Ai14-314467.03.02.01 \n", + "1 Pvalb-IRES-Cre;Ai14-185362.03.01.01 \n", + "2 Chrna2-Cre_OE25;Ai14(BT)-280154.04.01.01 \n", + "3 H16.06.008.01.26.04 \n", + "4 Rorb-IRES2-Cre-D;Ai14-168053.05.01.01 \n", + "5 Ctgf-2A-dgCre;Ai14(IVSCC)-230665.02.02.01 \n", + "6 Nos1-CreERT2;Sst-IRES-FlpO;Ai65-304714.02.01.01 \n", + "7 H16.06.010.01.03.04.01 \n", + "8 Nr5a1-Cre;Ai14-187780.03.02.01 \n", + "9 Slc17a6-IRES-Cre;Ai14-190263.04.01.01 \n", "\n", - " subject.strain.label \n", - "0 Sim1-Cre \n", - "1 Sim1-Cre \n", - "2 Sim1-Cre \n", - "3 Sim1-Cre \n", - "4 Sim1-Cre \n", - "5 Sim1-Cre \n", - "6 Sim1-Cre \n", - "7 Sim1-Cre \n", - "8 Sim1-Cre \n", - "9 Sim1-Cre " + " subject.id subject.type \n", + "0 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "1 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "2 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "3 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "4 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "5 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "6 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "7 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "8 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", + "9 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject " ] }, "execution_count": 34, diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index dbf757ad..1db89451 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -111,6 +111,7 @@ def map_resources(self, resources : Union[List[Resource], Resource], datatypes = self.types mappings = self.mappings() mapped_resources = [] + resources = (resources if isinstance(resources, list) else [resources]) for resource in resources: if resource_type is None: try: From 2cfabf341902f528d8a74c6a7886d73a5aa5516a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 24 Oct 2022 09:27:14 +0200 Subject: [PATCH 17/30] Implemented mappings methods in RdfModel and update Database classes accordingly. --- .../database-sources/prod-nexus-sources.yml | 40 +- .../prod-nexus-sources_progress.yml | 89 --- .../NeuroElectro/jsonld_context.json | 158 +++++ .../UniProt/jsonld_context.json | 2 +- .../17 - Database-sources.ipynb | 583 +++--------------- kgforge/core/archetypes/database.py | 99 +-- kgforge/core/commons/dictionaries.py | 14 +- kgforge/core/forge.py | 18 +- .../databases/store_database.py | 9 - kgforge/specializations/models/rdf_model.py | 24 +- 10 files changed, 333 insertions(+), 703 deletions(-) delete mode 100644 examples/configurations/database-sources/prod-nexus-sources_progress.yml diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml index e680cc9f..32476adf 100644 --- a/examples/configurations/database-sources/prod-nexus-sources.yml +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -55,26 +55,26 @@ Formatters: identifier_neuroelectro: http://neuroelectro.org/api/1/{}/{} identifier_neurolex: http://neurolex.org/wiki/{} -DatabaseSources: +Databases: UniProt: - store: - name: UniProtStore - endpoint: "https://www.uniprot.org/" - searchendpoints: - sparql: - endpoint: "https://sparql.uniprot.org/sparql" + origin: store + source: SPARQLStore + searchendpoints: + sparql: + endpoint: "https://sparql.uniprot.org/sparql" + model: + name: RdfModel + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/UniProt + context: + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/UniProt/jsonld_context.json NeuroElectro: - store: - name: BlueBrainNexus - bucket: bbp/neuroelectro - MouseLight: - store: - name: BlueBrainNexus - bucket: dke/kgforge - protocol: 'https://www.janelia.org/project-team/mouselight/resources' - license: - id: 'https://creativecommons.org/licenses/by-nc/4.0' - label: 'CC BY-NC 4.0' - definition: + origin: store + source: BlueBrainNexus + bucket: bbp/neuroelectro + model: + name: RdfModel origin: directory - source: '/Users/cgonzale/Documents/code/nexus-forge/examples/database_sources/MouseLight/' \ No newline at end of file + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/NeuroElectro + context: + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/models/neuroshapes_context.json \ No newline at end of file diff --git a/examples/configurations/database-sources/prod-nexus-sources_progress.yml b/examples/configurations/database-sources/prod-nexus-sources_progress.yml deleted file mode 100644 index 1c8fdccc..00000000 --- a/examples/configurations/database-sources/prod-nexus-sources_progress.yml +++ /dev/null @@ -1,89 +0,0 @@ -Model: - name: RdfModel - origin: store - source: BlueBrainNexus - context: - iri: "https://bbp.neuroshapes.org" - bucket: "neurosciencegraph/datamodels" -Store: - name: BlueBrainNexus - endpoint: https://bbp.epfl.ch/nexus/v1 - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - elastic: - endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" - mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" - default_str_keyword_field: "keyword" - vocabulary: - metadata: - iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" - local_iri: "https://bluebrainnexus.io/contexts/metadata.json" - namespace: "https://bluebrain.github.io/nexus/vocabulary/" - deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" - project_property: "https://bluebrain.github.io/nexus/vocabulary/project" - max_connection: 50 - versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" - file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson - -Resolvers: - ontology: - - resolver: OntologyResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: terms - bucket: neurosciencegraph/datamodels - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson - agent: - - resolver: AgentResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: agents - bucket: bbp/agents - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson - -Formatters: - identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} - identifier_neuroelectro: http://neuroelectro.org/api/1/{}/{} - identifier_neurolex: http://neurolex.org/wiki/{} - -Databases: - UniProt: - origin: store - source: SPARQLStore - endpoint: "http://purl.uniprot.org/core/" - searchendpoints: - sparql: - endpoint: "https://sparql.uniprot.org/sparql" - model: - origin: directory - source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/UniProt - context: - iri: "https://bbp.epfl.ch/jsonldcontext/db/uniprot" - bucket: jsonld_context.json - NeuroElectro: - origin: store - source: BlueBrainNexus - bucket: bbp/neuroelectro - model: - origin: directory - source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/NeuroElectro - context: - bucket: jsonld_context.json - MouseLight: - origin: store - source: BlueBrainNexus - bucket: dke/kgforge - model: - origin: directory - source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/MouseLight - context: - bucket: jsonld_context.json \ No newline at end of file diff --git a/examples/database-sources/NeuroElectro/jsonld_context.json b/examples/database-sources/NeuroElectro/jsonld_context.json index e69de29b..1dfd8b46 100644 --- a/examples/database-sources/NeuroElectro/jsonld_context.json +++ b/examples/database-sources/NeuroElectro/jsonld_context.json @@ -0,0 +1,158 @@ +{ + "@context": { + "@vocab": "http://example.org/vocab/", + "Activity": { + "@id": "prov:Activity" + }, + "Agent": { + "@id": "prov:Agent" + }, + "Class": { + "@id": "owl:Class" + }, + "Entity": { + "@id": "prov:Entity" + }, + "Association": { + "@id": "schema:Association" + }, + "BrainRegion": { + "@id": "nsg:BrainRegion" + }, + "Building": { + "@id": "http://schema.org/Building" + }, + "Employee": { + "@id": "http://www.example.com/Employee" + }, + "Organization": { + "@id": "schema:Organization" + }, + "Person": { + "@id": "schema:Person" + }, + "PostalAddress": { + "@id": "schema:PostalAddress" + }, + "address": { + "@id": "schema:address" + }, + "agent": { + "@id": "schema:agent" + }, + "birthDate": { + "@id": "schema:birthDate", + "@type": "xsd:date" + }, + "citation": { + "@id": "schema:citation" + }, + "contractor": { + "@id": "schema:contractor" + }, + "deathDate": { + "@id": "schema:deathDate", + "@type": "xsd:date" + }, + "department": { + "@id": "schema:department" + }, + "description": { + "@id": "schema:description" + }, + "endedAtTime": { + "@id": "prov:endedAtTime" + }, + "familyName": { + "@id": "schema:familyName" + }, + "founder": { + "@id": "schema:founder" + }, + "gender": { + "@id": "schema:gender" + }, + "generated": { + "@id": "prov:generated" + }, + "geo": { + "@id": "schema:geo" + }, + "givenName": { + "@id": "schema:givenName" + }, + "image": { + "@id": "http://schema.org/image", + "@type": "@id" + }, + "isDefinedBy": { + "@id": "rdfs:isDefinedBy", + "@type": "@id" + }, + "label": "rdfs:label", + "latitude": { + "@id": "schema:latitude", + "@type": "xsd:float" + }, + "longitude": { + "@id": "schema:longitude" + }, + "name": { + "@id": "schema:name" + }, + "notation": "skos:notation", + "nsg": "https://neuroshapes.org/", + "owl": "http://www.w3.org/2002/07/owl#", + "postalCode": { + "@id": "schema:postalCode" + }, + "prefLabel": "skos:prefLabel", + "prov": "http://www.w3.org/ns/prov#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "schema": "http://schema.org/", + "skos": "http://www.w3.org/2004/02/skos/core#", + "startDate": { + "@id": "schema:startDate", + "@type": "xsd:date" + }, + "startedAtTime": { + "@id": "prov:startedAtTime", + "@type": "xsd:dateTime" + }, + "status": { + "@id": "schema:status" + }, + "streetAddress": { + "@id": "schema:streetAddress" + }, + "subClassOf": { + "@id": "rdfs:subClassOf", + "@type": "@id" + }, + "supervisor": { + "@id": "schema:supervisor", + "@type": "@id" + }, + "type": { + "@id": "rdf:type", + "@type": "@id" + }, + "used": { + "@id": "prov:used" + }, + "validated": { + "@id": "schema:validated" + }, + "value": { + "@id": "schema:value" + }, + "wasStartedBy": { + "@id": "prov:wasStartedBy", + "@type": "@id" + }, + "xsd": "http://www.w3.org/2001/XMLSchema#", + "foaf": "http://xmlns.com/foaf/0.1/" + }, + "@id": "http://context.example.org" + } \ No newline at end of file diff --git a/examples/database-sources/UniProt/jsonld_context.json b/examples/database-sources/UniProt/jsonld_context.json index 3b104d2e..0d12ce3e 100644 --- a/examples/database-sources/UniProt/jsonld_context.json +++ b/examples/database-sources/UniProt/jsonld_context.json @@ -24,6 +24,6 @@ "type": { "@id": "rdf:type" }, - "@id": "https://bbp.epfl.ch/jsonldcontext/db/uniprot" + "@id": "https://bbp.epfl.ch/jsonldcontext/db/uniprot" } } \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 74f73b9a..19803e1a 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -58,8 +58,7 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n", - "MouseLight\n" + "NeuroElectro\n" ] } ], @@ -125,7 +124,6 @@ "Available Database sources:\n", "UniProt\n", "NeuroElectro\n", - "MouseLight\n", "DemoDB\n" ] } @@ -166,22 +164,25 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'ElectrophysiologicalFeatureAnnotation': ['DictionaryMapping'],\n", - " 'ParameterAnnotation': ['DictionaryMapping'],\n", - " 'ParameterBody': ['DictionaryMapping'],\n", - " 'ScholarlyArticle': ['DictionaryMapping'],\n", - " 'SeriesBody': ['DictionaryMapping']}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Managed mappings for the data source per entity type and mapping type:\n", + " - ElectrophysiologicalFeatureAnnotation:\n", + " * DictionaryMapping\n", + " - ParameterAnnotation:\n", + " * DictionaryMapping\n", + " - ParameterBody:\n", + " * DictionaryMapping\n", + " - ScholarlyArticle:\n", + " * DictionaryMapping\n", + " - SeriesBody:\n", + " * DictionaryMapping\n" + ] } ], "source": [ - "forge.mappings(\"NeuroElectro\")" + "forge.mappings(source=\"NeuroElectro\")" ] }, { @@ -190,14 +191,15 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'Gene': ['DictionaryMapping'], 'Protein': ['DictionaryMapping']}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Managed mappings for the data source per entity type and mapping type:\n", + " - Gene:\n", + " * DictionaryMapping\n", + " - Protein:\n", + " * DictionaryMapping\n" + ] } ], "source": [ @@ -211,8 +213,7 @@ "outputs": [], "source": [ "from kgforge.specializations.mappings import DictionaryMapping\n", - "mapping = forge.mapping(\"ScholarlyArticle\", \"NeuroElectro\")\n", - "direct_mapping = neuroelectro.mapping(\"ScholarlyArticle\", type=DictionaryMapping)" + "mapping = forge.mapping(entity=\"ScholarlyArticle\", source=\"NeuroElectro\")" ] }, { @@ -263,52 +264,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " id: forge.format(\"identifier\", \"scholarlyarticles\", x.id)\n", - " type:\n", - " [\n", - " Entity\n", - " ScholarlyArticle\n", - " ]\n", - " abstract: x.abstract\n", - " author: x.authors_shaped\n", - " datePublished: x.date_issued\n", - " identifier: x.identifiers\n", - " isPartOf:\n", - " {\n", - " type: Periodical\n", - " issn: x.issn\n", - " name: x.journal\n", - " publisher: x.publisher\n", - " }\n", - " name: f\"article_{x.id}\"\n", - " sameAs: x.full_text_link\n", - " title: x.title\n", - " url: x.full_text_link\n", - "}\n" - ] - } - ], - "source": [ - "print(direct_mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Available Database sources:\n", - "UniProt\n" + " db_sources\n", + " AttributeError: 'StoreDatabase' object has no attribute 'datatypes'\n", + "\n" ] } ], "source": [ - "forge.db_sources(mappings='Gene', pretty=True)" + "forge.db_sources(type_='Gene', pretty=True)" ] }, { @@ -323,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -384,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -393,7 +356,7 @@ "2" ] }, - "execution_count": 17, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -404,7 +367,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -413,59 +376,74 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/72136\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/3449\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: We investigated the effects of muscarinic acetylcholine receptor (mAChR) activation on GABAergic synaptic transmission in rat hippocampal neurons. Current-clamp recordings revealed that methacholine produced membrane depolarization and action potential firing. Methacholine augmented the bicuculline-sensitive and GABA(A) -mediated frequency of spontaneous inhibitory postsynaptic currents (sIPSCs); the action of methacholine had a slow onset and longer duration. The increase in methacholine-evoked sIPSCs was completely inhibited by atropine and was insensitive to glutamatergic receptor blockers. Interestingly, methacholine action was not inhibited by intracellular perfusion with GDP-β-S, suggesting that muscarinic effects on membrane excitability and sIPSC frequency are mainly presynaptic. McN-A-343 and pirenzepine, selective agonist and antagonist of the m1 mAChR subtype, respectively, neither enhanced sIPSCs nor inhibited the methacholine effect. However, the m3-m5 mAChR antagonist 4-DAMP, and the m2-m4 mAChR antagonist himbacine inhibited the methacholine effect. U73122, an IP(3) production inhibitor, and 2APB, an IP(3) receptor blocker, drastically decreased the methacholine effect. Recording of miniature events revealed that besides the effect exerted by methacholine on membrane firing properties and sIPSC frequency, muscarinic receptors also enhanced the frequency of mIPSCs with no effect on their amplitude, possibly modulating the molecular machinery subserving vesicle docking and fusion and suggesting a tight colocalization at the active zone of the presynaptic terminals. These data strongly suggest that by activating presynaptic m2, m3, m4 and m5 mAChRs, methacholine can increase membrane excitability and enhance efficiency in the GABA release machinery, perhaps through a mechanism involving the release of calcium from the endoplasmic reticulum.\n", + " abstract: The voltage-gated potassium channel subunit Kv3.1 confers fast firing characteristics to neurones. Kv3.1b subunit immunoreactivity (Kv3.1b-IR) was widespread throughout the medulla oblongata, with labelled neurones in the gracile, cuneate and spinal trigeminal nuclei. In the nucleus of the solitary tract (NTS), Kv3.1b-IR neurones were predominantly located close to the tractus solitarius (TS) and could be GABAergic or glutamatergic. Ultrastructurally, Kv3.1b-IR was detected in NTS terminals, some of which were vagal afferents. Whole-cell current-clamp recordings from neurones near the TS revealed electrophysiological characteristics consistent with the presence of Kv3.1b subunits: short duration action potentials (4.2 +/- 1.4 ms) and high firing frequencies (68.9 +/- 5.3 Hz), both sensitive to application of TEA (0.5 mm) and 4-aminopyridine (4-AP; 30 mum). Intracellular dialysis of an anti-Kv3.1b antibody mimicked and occluded the effects of TEA and 4-AP in NTS and dorsal column nuclei neurones, but not in dorsal vagal nucleus or cerebellar Purkinje cells (which express other Kv3 subunits, but not Kv3.1b). Voltage-clamp recordings from outside-out patches from NTS neurones revealed an outward K(+) current with the basic characteristics of that carried by Kv3 channels. In NTS neurones, electrical stimulation of the TS evoked EPSPs and IPSPs, and TEA and 4-AP increased the average amplitude and decreased the paired pulse ratio, consistent with a presynaptic site of action. Synaptic inputs evoked by stimulation of a region lacking Kv3.1b-IR neurones were not affected, correlating the presence of Kv3.1b in the TS with the pharmacological effects.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: González\n", - " givenName: J C\n", + " familyName: Dallas\n", + " givenName: Mark L\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Atkinson\n", + " givenName: Lucy\n", " }\n", " {\n", " type: Person\n", - " familyName: Albiñana\n", - " givenName: E\n", + " familyName: Milligan\n", + " givenName: Carol J\n", " }\n", " {\n", " type: Person\n", - " familyName: Baldelli\n", - " givenName: P\n", + " familyName: Morris\n", + " givenName: Neil P\n", " }\n", " {\n", " type: Person\n", - " familyName: García\n", - " givenName: A G\n", + " familyName: Lewis\n", + " givenName: David I\n", " }\n", " {\n", " type: Person\n", - " familyName: Hernández-Guijo\n", - " givenName: J M\n", + " familyName: Deuchars\n", + " givenName: Susan A\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Deuchars\n", + " givenName: Jim\n", " }\n", " ]\n", - " datePublished: 2011-1\n", + " datePublished: 2005-2-1\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 21091801\n", + " value: 15528247\n", + " }\n", + " {\n", + " propertyID: doi\n", + " value: 10.1113/jphysiol.2004.073338\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0953-816X\n", - " name: The European journal of neuroscience\n", + " issn: 0022-3751\n", + " name: The Journal of physiology\n", + " publisher: Blackwell Science Inc\n", " }\n", - " name: article_72136\n", - " sameAs: http://onlinelibrary.wiley.com/doi/10.1111/j.1460-9568.2010.07475.x/abstract;jsessionid=E294450D87316113388F9E234F3F8A39.d02t01\n", - " title: Presynaptic muscarinic receptor subtypes involved in the enhancement of spontaneous GABAergic postsynaptic currents in hippocampal neurons.\n", - " url: http://onlinelibrary.wiley.com/doi/10.1111/j.1460-9568.2010.07475.x/abstract;jsessionid=E294450D87316113388F9E234F3F8A39.d02t01\n", + " name: article_3449\n", + " sameAs: http://jp.physoc.org/content/562/3/655.long\n", + " title: Localization and function of the Kv3.1b subunit in the rat medulla oblongata: focus on the nucleus tractus solitarii.\n", + " url: http://jp.physoc.org/content/562/3/655.long\n", "}\n" ] } @@ -476,7 +454,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -492,7 +470,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -518,7 +496,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -527,7 +505,7 @@ "10" ] }, - "execution_count": 21, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -538,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -547,7 +525,7 @@ "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/A0B137')" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -565,7 +543,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -574,7 +552,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -609,7 +587,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -618,7 +596,7 @@ "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0B137', _inner_sync=False)" ] }, - "execution_count": 25, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -636,7 +614,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -645,7 +623,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -665,7 +643,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -674,16 +652,16 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ - "new_resource = uniprot.map_resources(raw_proteins[0], 'Protein')" + "new_resource = uniprot.map(raw_proteins[0], 'Protein')" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -720,395 +698,6 @@ "\n", "print(json.dumps(forge.as_jsonld(new_resource), indent=4))" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Save in BBP KG (Nexus)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Access" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Set filters" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "_type = \"NeuronMorphology\"\n", - "filters = {\"type\": _type}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Run Query" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 dataset(s) of type NeuronMorphology found\n" - ] - } - ], - "source": [ - "limit = 10 # You can limit the number of results, pass `None` to fetch all the results\n", - "\n", - "data = forge.search(filters, db_source='MouseLight', limit=limit)\n", - "\n", - "print(f\"{str(len(data))} dataset(s) of type {_type} found\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Display the results as pandas dataframe" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idbrainLocation.brainRegion.idbrainLocation.brainRegion.labelbrainLocation.layercontribution.typecontribution.agent.idcontribution.agent.typedistribution.contentUrldistribution.encodingFormatdistribution.namenamesubject.idsubject.type
0https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISp55Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcHtr3a-Cre_NO152;Ai14-314467.03.02.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
1https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISp55Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcPvalb-IRES-Cre;Ai14-185362.03.01.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
2https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISli6a6aContributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcChrna2-Cre_OE25;Ai14(BT)-280154.04.01.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
3https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...MTG5Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcH16.06.008.01.26.04https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
4https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISp55Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcRorb-IRES2-Cre-D;Ai14-168053.05.01.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
5https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISp6b6bContributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcCtgf-2A-dgCre;Ai14(IVSCC)-230665.02.02.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
6https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure/33VISp6a6aContributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcNos1-CreERT2;Sst-IRES-FlpO;Ai65-304714.02.01.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
7https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...MTG3Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcH16.06.010.01.03.04.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
8https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISp44Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcNr5a1-Cre;Ai14-187780.03.02.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
9https://bbp.epfl.ch/neurosciencegraph/data/neu...http://api.brain-map.org/api/v2/data/Structure...VISp2/32/3Contributionhttps://www.grid.ac/institutes/grid.417881.3Organizationhttps://staging.nexus.ocp.bbp.epfl.ch/v1/files...application/swcreconstruction.swcSlc17a6-IRES-Cre;Ai14-190263.04.01.01https://bbp.epfl.ch/neurosciencegraph/data/sub...Subject
\n", - "
" - ], - "text/plain": [ - " id \\\n", - "0 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "1 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "2 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "3 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "4 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "5 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "6 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "7 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "8 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "9 https://bbp.epfl.ch/neurosciencegraph/data/neu... \n", - "\n", - " brainLocation.brainRegion.id \\\n", - "0 http://api.brain-map.org/api/v2/data/Structure... \n", - "1 http://api.brain-map.org/api/v2/data/Structure... \n", - "2 http://api.brain-map.org/api/v2/data/Structure... \n", - "3 http://api.brain-map.org/api/v2/data/Structure... \n", - "4 http://api.brain-map.org/api/v2/data/Structure... \n", - "5 http://api.brain-map.org/api/v2/data/Structure... \n", - "6 http://api.brain-map.org/api/v2/data/Structure/33 \n", - "7 http://api.brain-map.org/api/v2/data/Structure... \n", - "8 http://api.brain-map.org/api/v2/data/Structure... \n", - "9 http://api.brain-map.org/api/v2/data/Structure... \n", - "\n", - " brainLocation.brainRegion.label brainLocation.layer contribution.type \\\n", - "0 VISp5 5 Contribution \n", - "1 VISp5 5 Contribution \n", - "2 VISli6a 6a Contribution \n", - "3 MTG 5 Contribution \n", - "4 VISp5 5 Contribution \n", - "5 VISp6b 6b Contribution \n", - "6 VISp6a 6a Contribution \n", - "7 MTG 3 Contribution \n", - "8 VISp4 4 Contribution \n", - "9 VISp2/3 2/3 Contribution \n", - "\n", - " contribution.agent.id contribution.agent.type \\\n", - "0 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "1 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "2 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "3 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "4 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "5 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "6 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "7 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "8 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "9 https://www.grid.ac/institutes/grid.417881.3 Organization \n", - "\n", - " distribution.contentUrl \\\n", - "0 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "1 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "2 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "3 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "4 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "5 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "6 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "7 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "8 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "9 https://staging.nexus.ocp.bbp.epfl.ch/v1/files... \n", - "\n", - " distribution.encodingFormat distribution.name \\\n", - "0 application/swc reconstruction.swc \n", - "1 application/swc reconstruction.swc \n", - "2 application/swc reconstruction.swc \n", - "3 application/swc reconstruction.swc \n", - "4 application/swc reconstruction.swc \n", - "5 application/swc reconstruction.swc \n", - "6 application/swc reconstruction.swc \n", - "7 application/swc reconstruction.swc \n", - "8 application/swc reconstruction.swc \n", - "9 application/swc reconstruction.swc \n", - "\n", - " name \\\n", - "0 Htr3a-Cre_NO152;Ai14-314467.03.02.01 \n", - "1 Pvalb-IRES-Cre;Ai14-185362.03.01.01 \n", - "2 Chrna2-Cre_OE25;Ai14(BT)-280154.04.01.01 \n", - "3 H16.06.008.01.26.04 \n", - "4 Rorb-IRES2-Cre-D;Ai14-168053.05.01.01 \n", - "5 Ctgf-2A-dgCre;Ai14(IVSCC)-230665.02.02.01 \n", - "6 Nos1-CreERT2;Sst-IRES-FlpO;Ai65-304714.02.01.01 \n", - "7 H16.06.010.01.03.04.01 \n", - "8 Nr5a1-Cre;Ai14-187780.03.02.01 \n", - "9 Slc17a6-IRES-Cre;Ai14-190263.04.01.01 \n", - "\n", - " subject.id subject.type \n", - "0 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "1 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "2 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "3 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "4 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "5 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "6 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "7 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "8 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject \n", - "9 https://bbp.epfl.ch/neurosciencegraph/data/sub... Subject " - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "property_to_display = [\"id\",\"name\",\"subject\",\"brainLocation.brainRegion.id\",\"brainLocation.brainRegion.label\",\"brainLocation.layer.id\",\"brainLocation.layer.label\", \"contribution\",\"brainLocation.layer.id\",\"brainLocation.layer.label\",\"distribution.name\",\"distribution.contentUrl\",\"distribution.encodingFormat\"]\n", - "reshaped_data = forge.reshape(data, keep=property_to_display)\n", - "\n", - "forge.as_dataframe(reshaped_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index 1db89451..9f629912 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -19,12 +19,12 @@ from kgforge.core import Resource from kgforge.core.commons.context import Context +from kgforge.core.commons.execution import not_supported from kgforge.core.archetypes import Mapping, Model from kgforge.core.commons.attributes import repr_class from kgforge.core.commons.exceptions import ConfigurationError -from kgforge.core.commons.dictionaries import with_defaults +from kgforge.core.commons.dictionaries import use_values from kgforge.core.commons.imports import import_class -from kgforge.core.commons.dictionaries import with_defaults class Database(ABC): @@ -41,92 +41,52 @@ def __init__(self, forge : Optional["KnowledgeGraphForge"], source: str, **confi self._forge: Optional["KnowledgeGraphForge"] = forge # Model model_config = config.pop("model") - if model_config.get('origin') == 'directory': - dirpath = model_config.get('source') - self._dirpath = dirpath - bucket = model_config.get('bucket', 'jsonld_context.json') - iri = model_config.get('iri', None) - context_path = Path(dirpath, bucket) - try: - # load context from file - with open(context_path, 'r') as jfile: - context_file = json.load(jfile) - self.context = Context(context_file, iri) - if 'model_context' not in config: - config['model_context'] = self.context - except Exception: - self.context = None - elif model_config["origin"] == "store": - with_defaults( + if model_config["origin"] == "store": + use_values( model_config, config, - "source", - "name", ["endpoint", "token", "bucket", "vocabulary"], ) - model_name = model_config.pop("name") - model = import_class(model_name, "models") - self._model: Model = model(**model_config) - if 'model_context' not in config: - config['model_context'] = self._model.context() - else: - raise NotImplementedError('DB Model not yet implemented.') + model_name = model_config.pop("name") + model = import_class(model_name, "models") + self._model: Model = model(**model_config) + self.context = self._model.context() + if 'model_context' not in config: + config['model_context'] = self._model.context() self.source: str = source self.service: Any = self._initialize_service(self.source, **config) def __repr__(self) -> str: return repr_class(self) - def _mappings(self) -> Dict[str, List[str]]: - try: - dirpath = Path(self._dirpath, "mappings") - mappings = {} - if dirpath.is_dir(): - for x in dirpath.glob("*/*.hjson"): - mappings.setdefault(x.stem, []).append(x.parent.name) - else: - raise ValueError("Mapping directory not found.") - return mappings - except AttributeError: - raise ConfigurationError('No directory path was found from the configuration.') - - def mappings(self) -> Optional[Dict[str, List[str]]]: - mappings = {k: sorted(v) for k, v in - sorted(self._mappings().items(), key=lambda kv: kv[0])} - return mappings - - def mapping(self, entity: str, type: Callable) -> Mapping: - filename = f"{entity}.hjson" - try: - filepath = Path(self._dirpath, "mappings", type.__name__, filename) - if filepath.is_file(): - return type.load(filepath) - else: - raise ValueError("unrecognized entity type or source file") - except AttributeError: - raise ConfigurationError('No directory path was found from the configuration.') - - def map_resources(self, resources : Union[List[Resource], Resource], - resource_type : Optional[str] = None) -> Optional[Union[Resource, List[Resource]]]: - datatypes = self.types - mappings = self.mappings() + def map(self, resources : Union[List[Union[Resource, str]], Union[Resource, str]], + type_ : Optional[Union[str, "DictionaryMapping"]] = None) -> Optional[Union[Resource, List[Resource]]]: + mappings = self._model.mappings(self._model.source, False) mapped_resources = [] resources = (resources if isinstance(resources, list) else [resources]) for resource in resources: - if resource_type is None: + if isinstance(resource, Resource): + resource_dict = self._forge.as_json(resource) + else: + resource_dict = resource + resource = self._forge.from_json(resource_dict) + if type_ is None: try: - resource_type = resource.type + type_ = resource.type except AttributeError: mapped_resources.append(resource) - if resource_type in datatypes: - mapping_class : Mapping = import_class(mappings[resource_type][0], "mappings") - mapping = self.mapping(resource_type, mapping_class) - mapped_resources.append(self._forge.map(self._forge.as_json(resource), mapping)) + elif isinstance(type_, Mapping): + mapped_resources.append(self._forge.map(resource_dict, type_)) + elif type_ in mappings: + # type_ is the entity here + mapping_class : Mapping = import_class(mappings[type_][0], "mappings") + mapping = self._model.mapping(type_, self._model.source, mapping_class) + mapped_resources.append(self._forge.map(resource_dict, mapping)) else: mapped_resources.append(resource) return mapped_resources - def datatypes(self): + def types(self): # TODO: add other datatypes used, for instance, inside the mappings return list(self.mappings().keys()) @@ -170,9 +130,8 @@ def _initialize_service(self, source: str, **source_config) -> Any: raise ConfigurationError(f"unrecognized DataBase origin '{origin}'") @staticmethod - @abstractmethod def _service_from_directory(dirpath: Path, **source_config) -> Any: - pass + not_supported() @staticmethod @abstractmethod diff --git a/kgforge/core/commons/dictionaries.py b/kgforge/core/commons/dictionaries.py index 1ac856e3..c5667df6 100644 --- a/kgforge/core/commons/dictionaries.py +++ b/kgforge/core/commons/dictionaries.py @@ -21,12 +21,10 @@ def with_defaults(original: Dict, other: Dict, original_key: str, other_key: str """Update 'original' with 'other' 'keys' unless keys value is different in both dictionaries.""" if original[original_key] == other[other_key]: - for x in keys: - if x not in original and x in other: - original[x] = other[x] + use_values(original, other, keys) - -def update_dict(original: Dict, other:Dict) -> Dict: - original_copy = copy.deepcopy(original) - original_copy.update(other) - return original_copy +def use_values(original: Dict, other: Dict, + keys: List[str]) -> None: + for x in keys: + if x not in original and x in other: + original[x] = other[x] diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index ba4bd15a..f6c2fb24 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -517,19 +517,19 @@ def sources(self, pretty: bool = True) -> Optional[List[str]]: return self._model.sources(pretty) @catch - def db_sources(self, mappings: Optional[List[str]] = None, + def db_sources(self, type_: Optional[List[str]] = None, pretty: bool = False) -> Optional[List[str]]: """ Print(pretty=True) or return (pretty=False) configured data sources. :param pretty: a boolean :return: Optional[List[str]] """ - if mappings is None: + if type_ is None: sources = self._db_sources else: sources = {} - if isinstance(mappings, list): - for type in mappings: + if isinstance(type_, list): + for type in type_: for db in self._db_sources: types = self._db_sources[db].datatypes() if type in types: @@ -537,7 +537,7 @@ def db_sources(self, mappings: Optional[List[str]] = None, else: for db in self._db_sources: types = self._db_sources[db].datatypes() - if mappings in types: + if type_ in types: sources[db] = self._db_sources[db] if not sources: print("No Database sources were found for the given type(s)") @@ -565,7 +565,8 @@ def mappings( :return: Optional[Dict[str, List[str]]] """ if source in self._db_sources: - return self._db_sources[source].mappings() + db = self._db_sources[source] + return db._model.mappings(db._model.source, pretty) else: return self._model.mappings(source, pretty) @@ -582,7 +583,8 @@ def mapping( :return: Mapping """ if source in self._db_sources: - return self._db_sources[source].mapping(entity, type) + db = self._db_sources[source] + return db._model.mapping(entity, db._model.source, type) else: return self._model.mapping(entity, source, type) @@ -1020,7 +1022,7 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], store_copy.keys()) config['model_context'] = model_context config.update(origin=origin) - config.update(origin=origin) + config.update(source=source) config['name'] = name dbs[name] = StoreDatabase(self, **config) else: diff --git a/kgforge/specializations/databases/store_database.py b/kgforge/specializations/databases/store_database.py index 6e4953bb..8973cdfa 100644 --- a/kgforge/specializations/databases/store_database.py +++ b/kgforge/specializations/databases/store_database.py @@ -11,7 +11,6 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . -import os import json from pathlib import Path, PurePath from re import I @@ -62,10 +61,6 @@ def _check_properties(self, **info): if r not in properties: raise ValueError(f'Missing {r} from the properties to define the DatabasSource') - @property - def types(self): - # TODO: add other datatypes used, for instance, inside the mappings - return self.mappings().keys() def search(self, resolvers, *filters, **params): """Search within the database. @@ -99,10 +94,6 @@ def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, def elastic(**params): not_supported() - @staticmethod - def _service_from_directory(dirpath: Path, **source_config) -> Any: - not_supported() - @staticmethod def _service_from_web_service(endpoint: str, **source_config) -> Any: not_supported() diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index d6edb49f..66db5478 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -21,11 +21,12 @@ from rdflib.namespace import XSD from kgforge.core import Resource +from kgforge.core.archetypes import Mapping +from kgforge.core.commons.execution import run from kgforge.core.archetypes import Model, Store from kgforge.core.commons.actions import Action from kgforge.core.commons.context import Context from kgforge.core.commons.exceptions import ValidationError -from kgforge.core.commons.execution import run from kgforge.specializations.models.rdf.collectors import NodeProperties from kgforge.specializations.models.rdf.directory_service import DirectoryService from kgforge.specializations.models.rdf.service import RdfService @@ -59,6 +60,7 @@ XSD.dateTime: datetime.datetime(9999, 12, 31).isoformat(), } + DEFAULT_TYPE_ORDER = [str, float, int, bool, datetime.date, datetime.time] @@ -87,6 +89,26 @@ def _generate_context(self) -> Context: if document: return Context(document) + # Mappings. + + def _mappings(self, source: str) -> Dict[str, List[str]]: + dirpath = Path(source, "mappings") + mappings = {} + if dirpath.is_dir(): + for x in dirpath.glob("*/*.hjson"): + mappings.setdefault(x.stem, []).append(x.parent.name) + else: + raise ValueError("unrecognized source") + return mappings + + def mapping(self, entity: str, source: str, type: Callable) -> Mapping: + filename = f"{entity}.hjson" + filepath = Path(source, "mappings", type.__name__, filename) + if filepath.is_file(): + return type.load(filepath) + else: + raise ValueError("unrecognized entity type or source file") + # Templates. def _template(self, type: str, only_required: bool) -> Dict: From a7da898787d01a8315f328aa7c3566b51e203a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 27 Oct 2022 11:06:39 +0200 Subject: [PATCH 18/30] Rebased and added unit tests for SPARQLStore and StoreDatabase. --- .../17 - Database-sources.ipynb | 120 +++++++++----- kgforge/core/archetypes/database.py | 1 + .../databases/store_database.py | 31 ++-- .../stores/databases/service.py | 45 ++---- kgforge/specializations/stores/sparql.py | 65 +------- tests/conftest.py | 36 ++++- tests/data/dbpedia/jsonld_context.json | 33 ++++ .../databases/test_database.py | 149 ++++++++++++++++++ tests/specializations/stores/test_sparql.py | 144 +++++++++++++++++ 9 files changed, 475 insertions(+), 149 deletions(-) create mode 100644 tests/data/dbpedia/jsonld_context.json create mode 100644 tests/specializations/databases/test_database.py create mode 100644 tests/specializations/stores/test_sparql.py diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 19803e1a..7474f808 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -31,13 +31,24 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "TOKEN = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"neurosciencegraph/datamodels\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources_progress.yml\", endpoint=endpoint, bucket=BUCKET)" + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\",\n", + "token=TOKEN, endpoint=endpoint, bucket=BUCKET)" ] }, { @@ -376,74 +387,49 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/3449\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/35463\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: The voltage-gated potassium channel subunit Kv3.1 confers fast firing characteristics to neurones. Kv3.1b subunit immunoreactivity (Kv3.1b-IR) was widespread throughout the medulla oblongata, with labelled neurones in the gracile, cuneate and spinal trigeminal nuclei. In the nucleus of the solitary tract (NTS), Kv3.1b-IR neurones were predominantly located close to the tractus solitarius (TS) and could be GABAergic or glutamatergic. Ultrastructurally, Kv3.1b-IR was detected in NTS terminals, some of which were vagal afferents. Whole-cell current-clamp recordings from neurones near the TS revealed electrophysiological characteristics consistent with the presence of Kv3.1b subunits: short duration action potentials (4.2 +/- 1.4 ms) and high firing frequencies (68.9 +/- 5.3 Hz), both sensitive to application of TEA (0.5 mm) and 4-aminopyridine (4-AP; 30 mum). Intracellular dialysis of an anti-Kv3.1b antibody mimicked and occluded the effects of TEA and 4-AP in NTS and dorsal column nuclei neurones, but not in dorsal vagal nucleus or cerebellar Purkinje cells (which express other Kv3 subunits, but not Kv3.1b). Voltage-clamp recordings from outside-out patches from NTS neurones revealed an outward K(+) current with the basic characteristics of that carried by Kv3 channels. In NTS neurones, electrical stimulation of the TS evoked EPSPs and IPSPs, and TEA and 4-AP increased the average amplitude and decreased the paired pulse ratio, consistent with a presynaptic site of action. Synaptic inputs evoked by stimulation of a region lacking Kv3.1b-IR neurones were not affected, correlating the presence of Kv3.1b in the TS with the pharmacological effects.\n", + " abstract: Rationally, an increased intrinsic excitability of dorsal horn neurons could be a factor contributing to alter the gain of the nociceptive system during central sensitization, however direct evidence is scarce. Here we have examined this hypothesis using current and voltage-clamp recordings from dorsal horn neurons in the spinal cord in vitro preparation obtained from mice pups of either sex. Cords were extracted from carrageenan-pretreated and control animals to allow for comparison. Dorsal horn neurons from treated animals showed significantly larger and faster synaptic responses. Synaptic changes started developing shortly after inflammation (1 h) and developed further after a longer-term inflammation (20 h). However, these neurons showed biphasic changes in membrane excitability with an increase shortly after inflammation and a decrease in the longer term. Concomitant changes were observed in transient (I(A)) and sustained potassium currents (I(DR)). Prolonged superfusion of naive spinal cords with NMDA led to a decreased neuronal excitability and to increased potassium currents. Results suggest that excitability plays a role more complex than expected during the process of central sensitization of dorsal horn neurons and that modulation of potassium currents may contribute to shape the changing states of excitability. The decreased excitability observed after long-term inflammation is interpreted as a homeostatic correction to an abnormal state of synaptic activity.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Dallas\n", - " givenName: Mark L\n", + " familyName: Rivera-Arconada\n", + " givenName: Ivan\n", " }\n", " {\n", " type: Person\n", - " familyName: Atkinson\n", - " givenName: Lucy\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Milligan\n", - " givenName: Carol J\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Morris\n", - " givenName: Neil P\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Lewis\n", - " givenName: David I\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Deuchars\n", - " givenName: Susan A\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Deuchars\n", - " givenName: Jim\n", + " familyName: Lopez-Garcia\n", + " givenName: Jose A\n", " }\n", " ]\n", - " datePublished: 2005-2-1\n", + " datePublished: 2010-4-14\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 15528247\n", + " value: 20392959\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1113/jphysiol.2004.073338\n", + " value: 10.1523/JNEUROSCI.4359-09.2010\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0022-3751\n", - " name: The Journal of physiology\n", - " publisher: Blackwell Science Inc\n", + " issn: 0270-6474\n", + " name: The Journal of neuroscience : the official journal of the Society for Neuroscience\n", + " publisher: Society for Neuroscience\n", " }\n", - " name: article_3449\n", - " sameAs: http://jp.physoc.org/content/562/3/655.long\n", - " title: Localization and function of the Kv3.1b subunit in the rat medulla oblongata: focus on the nucleus tractus solitarii.\n", - " url: http://jp.physoc.org/content/562/3/655.long\n", + " name: article_35463\n", + " sameAs: http://www.jneurosci.org/content/30/15/5376.long\n", + " title: Changes in membrane excitability and potassium currents in sensitized dorsal horn neurons of mice pups.\n", + " url: http://www.jneurosci.org/content/30/15/5376.long\n", "}\n" ] } @@ -698,6 +684,54 @@ "\n", "print(json.dumps(forge.as_jsonld(new_resource), indent=4))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### same result could be obtain from a dictionary and a DictionaryMapping instance" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/proteins/P0DJN9\",\n", + " \"@type\": [\n", + " \"Entity\",\n", + " \"Protein\"\n", + " ],\n", + " \"encodedBy\": {\n", + " \"@id\": \"http://purl.uniprot.org/uniprot/P0DJN9#gene-MD5A00DD99270221B359AB0AE338E423668\",\n", + " \"label\": \"acsF\"\n", + " },\n", + " \"identifier\": {\n", + " \"propertyID\": \"UniProtKB\",\n", + " \"value\": \"P0DJN9\"\n", + " },\n", + " \"name\": \"Protein P0DJN9 from UniProtKB\",\n", + " \"label\": \"Aerobic magnesium-protoporphyrin IX monomethyl ester [oxidative] cyclase\",\n", + " \"subject\": {\n", + " \"label\": \"Rubrivivax gelatinosus\"\n", + " }\n", + " }\n", + "]\n" + ] + } + ], + "source": [ + "dict_resource = forge.as_json(raw_proteins[0])\n", + "mapping = DictionaryMapping.load(\"../../database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson\")\n", + "print(json.dumps(forge.as_jsonld(uniprot.map(dict_resource, mapping)), indent=4))" + ] } ], "metadata": { diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index 9f629912..88223025 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -142,3 +142,4 @@ def _service_from_web_service(endpoint: str, **source_config) -> Any: @abstractmethod def _service_from_store(store: Callable, **store_config) -> Any: pass + diff --git a/kgforge/specializations/databases/store_database.py b/kgforge/specializations/databases/store_database.py index 8973cdfa..9f785e70 100644 --- a/kgforge/specializations/databases/store_database.py +++ b/kgforge/specializations/databases/store_database.py @@ -11,17 +11,10 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . -import json -from pathlib import Path, PurePath -from re import I -import copy -from typing import Callable, Optional, Union, Dict, List, Any +from typing import Callable, Optional, Any -from kgforge.core import Resource from kgforge.core.archetypes import Store, Database from kgforge.core.commons.execution import not_supported -from kgforge.core.wrappings.paths import FilterOperator -from kgforge.specializations.mappers.dictionaries import DictionaryMapper from kgforge.specializations.stores.bluebrain_nexus import BlueBrainNexus @@ -61,7 +54,6 @@ def _check_properties(self, **info): if r not in properties: raise ValueError(f'Missing {r} from the properties to define the DatabasSource') - def search(self, resolvers, *filters, **params): """Search within the database. @@ -73,10 +65,8 @@ def search(self, resolvers, *filters, **params): return unmapped_resources else: # Try to find the type of the resources within the filters - resource_type = type_from_filters(filters) + resource_type = type_from_filters(*filters) return self.map_resources(unmapped_resources, resource_type=resource_type) - - return resource_type def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, offset: Optional[int] = None,**params): @@ -105,14 +95,21 @@ def _service_from_store(store: Callable, **store_config) -> Store: def health(self) -> Callable: not_supported() -def type_from_filters(filters): +def type_from_filters(*filters) -> Optional[str]: + """Returns the first `type` found in filters.""" resource_type = None - if isinstance(filters[0], dict): - if 'type' in filters[0]: - resource_type = filters[0]['type'] + filters = filters[0] + if isinstance(filters, dict): + if 'type' in filters: + resource_type = filters['type'] else: + # check filters grouping + if isinstance(filters, (list, tuple)): + filters = [filter for filter in filters] + else: + filters = [filters] for filter in filters: - if 'type' in filter.path and filter.operator is FilterOperator.EQUAL: + if 'type' in filter.path and filter.operator is "__eq__": resource_type = filter.value break return resource_type \ No newline at end of file diff --git a/kgforge/specializations/stores/databases/service.py b/kgforge/specializations/stores/databases/service.py index 8a495bf4..2e8b9d79 100644 --- a/kgforge/specializations/stores/databases/service.py +++ b/kgforge/specializations/stores/databases/service.py @@ -65,34 +65,19 @@ def __init__( self.headers = {"Content-Type": content_type, "Accept": accept} - sparql_config = ( - searchendpoints["sparql"] - if searchendpoints and "sparql" in searchendpoints - else None - ) - - self.headers_sparql = { - "Content-Type": sparql_config["Content-Type"] - if sparql_config and "Content-Type" in sparql_config - else "text/plain", - "Accept": sparql_config["Accept"] - if sparql_config and "Accept" in sparql_config - else "application/sparql-results+json", - } - - self.sparql_endpoint = dict() - self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] - self.sparql_endpoint["type"] = "sparql" - - def resolve_context(self, iri: str) -> Dict: - if iri in self.context_cache: - return self.context_cache[iri] - context_to_resolve = iri try: - context = Context(context_to_resolve) - except URLError: - raise ValueError(f"{context_to_resolve} is not resolvable") - else: - document = context.document["@context"] - self.context_cache.update({context_to_resolve: document}) - return document \ No newline at end of file + sparql_config = searchendpoints["sparql"] + self.headers_sparql = { + "Content-Type": sparql_config["Content-Type"] + if sparql_config and "Content-Type" in sparql_config + else "text/plain", + "Accept": sparql_config["Accept"] + if sparql_config and "Accept" in sparql_config + else "application/sparql-results+json", + } + + self.sparql_endpoint = dict() + self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] + self.sparql_endpoint["type"] = "sparql" + except Exception: + raise ValueError(f"Store configuration error: sparql searchendpoint missing") \ No newline at end of file diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py index a7ebb5db..f3347739 100644 --- a/kgforge/specializations/stores/sparql.py +++ b/kgforge/specializations/stores/sparql.py @@ -34,6 +34,7 @@ from kgforge.specializations.stores.bluebrain_nexus import ( CategoryDataType, _create_select_query, _box_value_as_full_iri, sparql_operator_map, + build_sparql_query_statements, type_map, format_type) @@ -137,7 +138,6 @@ def search( debug = params.get("debug", False) limit = params.get("limit", 100) offset = params.get("offset", None) - deprecated = params.get("deprecated", False) distinct = params.get("distinct", False) includes = params.get("includes", None) excludes = params.get("excludes", None) @@ -177,7 +177,7 @@ def search( return resources def sparql( - self, query: str, debug: bool, limit: int = None, offset: int = None, **params + self, query: str, debug: bool = False, limit: int = None, offset: int = None, **params ) -> List[Resource]: rewrite = params.get("rewrite", True) qr = ( @@ -185,8 +185,9 @@ def sparql( if self.context is not None and rewrite else query ) - qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") - qr = _replace_in_sparql(qr, "OFFSET", offset, 0, r" OFFSET \d+") + if rewrite: + qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") + qr = _replace_in_sparql(qr, "OFFSET", offset, 0, r" OFFSET \d+") if debug: self._debug_query(qr) return self._sparql(qr, limit, offset, **params) @@ -245,7 +246,7 @@ def _initialize_service( endpoint: Optional[str], bucket: Optional[str], token: Optional[str], - searchendpoints: Optional[Dict] = None, + searchendpoints: Optional[Dict], **store_config, ) -> Any: try: @@ -258,6 +259,7 @@ def _initialize_service( accept = store_config.pop("Accept", "application/ld+json") params = store_config.pop("params", {}) store_context = store_config.pop('store_context', None) + except Exception as ve: raise ValueError(f"Store configuration error: {ve}") else: @@ -270,55 +272,4 @@ def _debug_query(query): print("Submitted query:", query) else: print(*["Submitted query:", *query.splitlines()], sep="\n ") - print() - - -def build_sparql_query_statements(context: Context, *conditions) -> Tuple[List, List]: - statements = list() - filters = list() - for index, f in enumerate(*conditions): - last_path = f.path[-1] - try: - last_term = context.terms[last_path] - except KeyError: - last_term = None - if last_path in ["id", "@id"]: - property_path = "/".join(f.path[:-1]) - elif last_path == "@type": - minus_last_path = f.path[:-1] - minus_last_path.append("type") - property_path = "/".join(minus_last_path) - else: - property_path = "/".join(f.path) - try: - if ( - last_path in ["type", "@type"] - or last_path in ["id", "@id"] - or (last_term is not None and last_term.type == "@id") - ): - if f.operator == "__eq__": - statements.append(f"{property_path} {_box_value_as_full_iri(f.value)}") - elif f.operator == "__ne__": - statements.append(f"{property_path} ?v{index}") - filters.append(f"FILTER(?v{index} != {f.value})") - else: - raise NotImplementedError( - f"supported operators are '==' and '!=' when filtering by type or id." - ) - else: - parsed_type, parsed_value = _parse_type(f.value, parse_str=False) - value_type = type_map[parsed_type] - value = format_type[value_type](parsed_value if parsed_value else f.value) - if value_type is CategoryDataType.LITERAL: - if f.operator not in ["__eq__", "__ne__"]: - raise NotImplementedError(f"supported operators are '==' and '!=' when filtering with a str.") - statements.append(f"{property_path} ?v{index}") - filters.append(f"FILTER(?v{index} = {_box_value_as_full_iri(value)})") - else: - statements.append(f"{property_path} ?v{index}") - filters.append( - f"FILTER(?v{index} {sparql_operator_map[f.operator]} {_box_value_as_full_iri(value)})" - ) - except NotImplementedError as nie: - raise ValueError(f"Operator '{sparql_operator_map[f.operator]}' is not supported with the value '{f.value}': {str(nie)}") - return statements, filters + print() \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 53304eb8..907c9d44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -384,7 +384,9 @@ def _make_jsonld( MODEL = "DemoModel" STORE = "DemoStore" RESOLVER = "DemoResolver" - +DATABASE = "StoreDatabase" +DBNAME = "DBpedia" +SPARQL_ENDPOINT = "http://dbpedia.org/sparql" @pytest.fixture( params=[ @@ -418,9 +420,18 @@ def store(request): def resolver(request): return request.param +@pytest.fixture( + params=[ + DATABASE, + f"{DATABASE} from kgforge.specializations.databases", + f"{DATABASE} from kgforge.specializations.databases.store_database", + ] +) +def database(request): + return request.param @pytest.fixture -def config(model, store, resolver): +def config(model, store, resolver, database): return { "Model": { "name": model, @@ -457,6 +468,27 @@ def config(model, store, resolver): }, ], }, + "Databases": { + DBNAME: + { + "origin": "store", + "source": "SPARQLStore", + "searchendpoints":{ + "sparql":{ + "endpoint": SPARQL_ENDPOINT + }, + + }, + "model": { + "name": model, + "origin": "directory", + "source": "tests/data/demo-model/", + "context":{ + "iri": "test/data/dbpedia" + } + }, + }, + } } diff --git a/tests/data/dbpedia/jsonld_context.json b/tests/data/dbpedia/jsonld_context.json new file mode 100644 index 00000000..09950988 --- /dev/null +++ b/tests/data/dbpedia/jsonld_context.json @@ -0,0 +1,33 @@ +{ + "@context": { + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "dbpedia2": "http://dbpedia.org/property/", + "dbpedia": "http://dbpedia.org/", + "dbo": "http://dbpedia.org/ontology/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "MusicalArtist": { + "@id": "dbo:MusicalArtist" + }, + "birthDate": { + "@id": "dbo:birthDate" + }, + "birthPlace": { + "@id": "dbo:birthPlace" + }, + "type": { + "@id": "rdf:type" + }, + "@id": "https://bbp.epfl.ch/jsonldcontext/db/dbpedia" + } +} \ No newline at end of file diff --git a/tests/specializations/databases/test_database.py b/tests/specializations/databases/test_database.py new file mode 100644 index 00000000..072d59bf --- /dev/null +++ b/tests/specializations/databases/test_database.py @@ -0,0 +1,149 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +# Placeholder for the test suite for actions. +import pytest + +from kgforge.core import Resource, KnowledgeGraphForge +from kgforge.core.commons.context import Context +from kgforge.core.wrappings.paths import Filter +from kgforge.specializations.databases.store_database import StoreDatabase, type_from_filters +from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.core.wrappings.dict import DictWrapper, wrap_dict +import json + +from tests.specializations.stores.test_sparql import dict_context + + +@pytest.mark.parametrize( + "filters,expected", + [ + pytest.param( + ({'type': 'Person'}), + ('Person'), + id="dictionary_type", + ), + pytest.param( + ({'id': 'my_id'}), + (None), + id="dictionary_notype", + ), + pytest.param( + (Filter(['id'], '__eq__', 'some_id')), + (None), + id="one_filter_notype", + ), + pytest.param( + (Filter(['type'], '__eq__', 'Person')), + ('Person'), + id="one_filter_type", + ), + pytest.param( + (Filter(['id'], '__eq__', 'some_id'), + Filter(['label'], '__eq__', 'some_label')), + None, + id="two_filters_notype", + ), + pytest.param( + (Filter(['id'], '__eq__', 'some_id'), + Filter(['type'], '__eq__', 'Person')), + 'Person', + id="two_filters_type", + ), + pytest.param( + (Filter(['type'], '__eq__', 'Protein'), + Filter(['type'], '__eq__', 'Person')), + 'Protein', + id="two_filters_two_types", + ), + ] +) +def test_type_from_filters(filters, expected): + assert type_from_filters(filters) == expected + + +@pytest.fixture +def context(dict_context): + return Context(dict_context) + + +@pytest.fixture +def model_config(): + return { + "name": "DemoModel", + "origin": "directory", + "source": "tests/data/demo-model/" + } + +@pytest.fixture +def db_config(): + return { + "origin": "store", + "source": "SPARQLStore", + "searchendpoints":{ + "sparql":{ + "endpoint": "http://dbpedia.org/sparql" + }, + + }, + "model": { + "name": "DemoModel", + "origin": "directory", + "source": "tests/data/demo-model/", + "context":{ + "iri": "test/data/dbpedia" + } + } + } + +@pytest.fixture +def forge(db_config, model_config): + config = { + "Model": model_config, + "Store": { + "name": "DemoStore", + }, + "Databases": { + 'dbpedia': db_config + } + } + return KnowledgeGraphForge(config) + + +def test_database_config(forge, db_config, model_config): + with pytest.raises(ValueError): + StoreDatabase(db_config) # Missing forge + with pytest.raises(ValueError): + StoreDatabase(forge, **db_config) # Missing name + with pytest.raises(ValueError): + # Missing model + StoreDatabase(forge, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/'}) + with pytest.raises(ConfigurationError): + StoreDatabase(forge, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/', + 'model': model_config}) + +def test_database_directory(forge, model_config): + with pytest.raises(Exception): + StoreDatabase(forge, **{'name': 'mydb', 'origin': 'directory', + 'source':'tests/data/demo-store/', + 'model': model_config}) + +def test_database_web_serice(forge, model_config): + with pytest.raises(Exception): + StoreDatabase(forge, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', + 'model': model_config}) + diff --git a/tests/specializations/stores/test_sparql.py b/tests/specializations/stores/test_sparql.py new file mode 100644 index 00000000..7b13a035 --- /dev/null +++ b/tests/specializations/stores/test_sparql.py @@ -0,0 +1,144 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import pytest + +from kgforge.core import Resource +from kgforge.core.wrappings.paths import Filter +from kgforge.core.commons.context import Context +from kgforge.specializations.stores.sparql import ( + SPARQLStore, + build_sparql_query_statements, +) + +SEARCH_ENDPOINT = {"sparql": {"endpoint": "http://dbpedia.org/sparql"}} + +@pytest.fixture +def dict_context(): + document = { + "@context": { + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "dbpedia2": "http://dbpedia.org/property/", + "dbpedia": "http://dbpedia.org/", + "dbo": "http://dbpedia.org/ontology/", + "dbr": "http://dbpedia.org/resource/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "MusicalArtist": { + "@id": "dbo:MusicalArtist" + }, + "birthDate": { + "@id": "dbo:birthDate" + }, + "birthPlace": { + "@id": "dbo:birthPlace" + }, + "type": { + "@id": "rdf:type" + }, + "Berlin": { + "@id": "dbr:Berlin" + }, + "@id": "https://bbp.epfl.ch/jsonldcontext/db/dbpedia" + } + } + return document + +@pytest.fixture +def store_context(dict_context): + return Context(dict_context) + +@pytest.fixture +def sparql_store(store_context): + return SPARQLStore( + model_context=store_context, + searchendpoints=SEARCH_ENDPOINT, + store_context=store_context + ) + +def test_config_error(): + with pytest.raises(ValueError): + SPARQLStore(endpoint="test", bucket="invalid") + +def test_config(sparql_store, store_context): + assert sparql_store.bucket == None + assert sparql_store.endpoint == None + assert sparql_store.context == store_context + +def test_search_params(sparql_store): + with pytest.raises(ValueError): + sparql_store.search(filters=[None]) + + +class TestQuery: + + @pytest.mark.parametrize( + "filters,expected", + [ + pytest.param( + ([Filter(['type'], '__eq__','MusicalArtist')]), + ([Resource.from_json({'id': "http://dbpedia.org/resource/1969_NCAA_University_Division_Men's_Cross_Country_Championships"}), + Resource.from_json({'id': "http://dbpedia.org/resource/1970_NCAA_University_Division_Men's_Cross_Country_Championships"}), + Resource.from_json({'id': "http://dbpedia.org/resource/1971_NCAA_University_Division_Men's_Cross_Country_Championships"})]), + id="search_names", + ) + ] + ) + def test_sparql_search(self, sparql_store, filters, expected): + resources = sparql_store.search(None, *filters, limit=3, debug=True) + assert resources == expected + + @pytest.mark.parametrize( + "query,expected", + [ + pytest.param( + ("""SELECT ?name WHERE { ?person a dbo:MusicalArtist . + ?person foaf:name ?name . + FILTER (LANG(?name) = 'en') . + }"""), + ([Resource.from_json({'name': '20 Dead Flower Children'}), + Resource.from_json({'name': '21 Savage'}), + Resource.from_json({'name': '220 Kid'})]), + id="query_names", + ), + pytest.param( + ("""PREFIX : + SELECT ?name ?birth WHERE { + ?person a dbo:MusicalArtist . + ?person dbo:birthPlace dbr:Berlin . + ?person dbo:birthDate ?birth . + ?person foaf:name ?name . + ?person rdfs:comment ?description . + FILTER (LANG(?description) = 'en') . + }"""), + ([Resource.from_json({'name': 'Monika Kruse', 'birth': "1971-07-23"}), + Resource.from_json({'name': 'Walter Gronostay', 'birth': "1906-07-29"}), + Resource.from_json({'name': 'Wilhelm Guttmann', 'birth': "1886-01-01"})]), + id="query_berlin_birthdates", + ), + ] + ) + def test_sparql_query(self, sparql_store, query, expected): + resources = sparql_store.sparql(query, limit=3, debug=True) + assert resources == expected \ No newline at end of file From b837bc8c4fff6919642c0f7764609ff52287fa40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 2 Nov 2022 17:27:23 +0100 Subject: [PATCH 19/30] Added WebServiceDatabase. Missing tests and improved download methods. --- .../database-sources/prod-nexus-sources.yml | 18 +- .../NeuroMorpho/jsonld_context.json | 164 ++++++ .../DictionaryMapping/NeuronMorphology.hjson | 53 ++ .../NeuroMorpho/metadata.json | 8 + .../17 - Database-sources.ipynb | 465 +++++++++++++++--- kgforge/core/archetypes/database.py | 1 + kgforge/core/forge.py | 11 +- kgforge/specializations/databases/__init__.py | 3 +- .../databases/store_database.py | 22 +- .../databases/webservice_database.py | 187 +++++++ kgforge/specializations/resources/datasets.py | 10 +- kgforge/specializations/stores/__init__.py | 3 +- .../stores/databases/__init__.py | 1 + .../stores/databases/web_service.py | 81 +++ .../specializations/stores/uniprot_store.py | 90 ---- .../databases/test_database.py | 3 +- 16 files changed, 939 insertions(+), 181 deletions(-) create mode 100644 examples/database-sources/NeuroMorpho/jsonld_context.json create mode 100644 examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson create mode 100644 examples/database-sources/NeuroMorpho/metadata.json create mode 100644 kgforge/specializations/databases/webservice_database.py create mode 100644 kgforge/specializations/stores/databases/web_service.py delete mode 100644 kgforge/specializations/stores/uniprot_store.py diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml index 32476adf..8928abd2 100644 --- a/examples/configurations/database-sources/prod-nexus-sources.yml +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -77,4 +77,20 @@ Databases: origin: directory source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/NeuroElectro context: - iri: /Users/cgonzale/Documents/code/nexus-forge/examples/models/neuroshapes_context.json \ No newline at end of file + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/models/neuroshapes_context.json + NeuroMorpho: + origin: web_service + source: http://neuromorpho.org/api/neuron + health: http://neuromorpho.org/api/health + files_download: + endpoint: https://neuromorpho.org/dableFiles + Accept: "*/*" + max_connection: 50 + content_type: application/json;charset=UTF-8 + accept: "*/*" + model: + name: RdfModel + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/Neuromorpho + context: + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/Neuromorpho/jsonld_context.json \ No newline at end of file diff --git a/examples/database-sources/NeuroMorpho/jsonld_context.json b/examples/database-sources/NeuroMorpho/jsonld_context.json new file mode 100644 index 00000000..8754369a --- /dev/null +++ b/examples/database-sources/NeuroMorpho/jsonld_context.json @@ -0,0 +1,164 @@ +{ + "@context": { + "@vocab": "http://example.org/vocab/", + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "schema": "http://schema.org/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "nsg": "https://neuroshapes.org/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "Activity": { + "@id": "prov:Activity" + }, + "Agent": { + "@id": "prov:Agent" + }, + "Class": { + "@id": "owl:Class" + }, + "Entity": { + "@id": "prov:Entity" + }, + "Association": { + "@id": "schema:Association" + }, + "BrainRegion": { + "@id": "nsg:BrainRegion" + }, + "Building": { + "@id": "http://schema.org/Building" + }, + "Employee": { + "@id": "http://www.example.com/Employee" + }, + "Organization": { + "@id": "schema:Organization" + }, + "Person": { + "@id": "schema:Person" + }, + "PostalAddress": { + "@id": "schema:PostalAddress" + }, + "address": { + "@id": "schema:address" + }, + "agent": { + "@id": "schema:agent" + }, + "birthDate": { + "@id": "schema:birthDate", + "@type": "xsd:date" + }, + "citation": { + "@id": "schema:citation" + }, + "contractor": { + "@id": "schema:contractor" + }, + "deathDate": { + "@id": "schema:deathDate", + "@type": "xsd:date" + }, + "department": { + "@id": "schema:department" + }, + "description": { + "@id": "schema:description" + }, + "endedAtTime": { + "@id": "prov:endedAtTime" + }, + "familyName": { + "@id": "schema:familyName" + }, + "founder": { + "@id": "schema:founder" + }, + "gender": { + "@id": "schema:gender" + }, + "generated": { + "@id": "prov:generated" + }, + "geo": { + "@id": "schema:geo" + }, + "givenName": { + "@id": "schema:givenName" + }, + "image": { + "@id": "http://schema.org/image", + "@type": "@id" + }, + "isDefinedBy": { + "@id": "rdfs:isDefinedBy", + "@type": "@id" + }, + "label": "rdfs:label", + "latitude": { + "@id": "schema:latitude", + "@type": "xsd:float" + }, + "longitude": { + "@id": "schema:longitude" + }, + "name": { + "@id": "schema:name" + }, + "notation": "skos:notation", + "postalCode": { + "@id": "schema:postalCode" + }, + "prefLabel": "skos:prefLabel", + "startDate": { + "@id": "schema:startDate", + "@type": "xsd:date" + }, + "startedAtTime": { + "@id": "prov:startedAtTime", + "@type": "xsd:dateTime" + }, + "status": { + "@id": "schema:status" + }, + "streetAddress": { + "@id": "schema:streetAddress" + }, + "subClassOf": { + "@id": "rdfs:subClassOf", + "@type": "@id" + }, + "supervisor": { + "@id": "schema:supervisor", + "@type": "@id" + }, + "type": { + "@id": "rdf:type", + "@type": "@id" + }, + "used": { + "@id": "prov:used" + }, + "validated": { + "@id": "schema:validated" + }, + "value": { + "@id": "schema:value" + }, + "wasStartedBy": { + "@id": "prov:wasStartedBy", + "@type": "@id" + } + }, + "@id": "http://context.example.org" + } \ No newline at end of file diff --git a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson new file mode 100644 index 00000000..48bc3708 --- /dev/null +++ b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -0,0 +1,53 @@ +{ + type: [ + Dataset + NeuronMorphology + ] + id: forge.format('identifier', 'neuronmorphologies/neuromorpho', f'{x.neuron_id:09}') + brainLocation: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_MATCH') else x.brain_region[0] + contribution: + { + type: Contribution + agent: + { + type: Organization + label: "George Mason University" + } + } + identifier: x.neuron_id + archive: x.archive + name: x.neuron_name + generation: + { + type: Generation + activity: + { + type: "nsg:NeuronMorphologyReconstruction" + hadProtocol:{ + + } + } + } + subject: + { + type: Subject + species: forge.resolve(x.species, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.species, scope='ontology', strategy='EXACT_MATCH') else {'label': x.species} + strain: forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') else {'label': x.scientific_name} + } + license: + { + type: License + id: https://neuromorpho.org + } + objectOfStudy: + { + type: ObjectOfStudy + id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells + label: Single Cell + } + note: x.note + dateCreated: x.deposition_date + dateUploaded: x.upload_date + description: f"Morphology of {x.neuron_name} obtained from NeuroMorpho API." + stain: x.stain +} \ No newline at end of file diff --git a/examples/database-sources/NeuroMorpho/metadata.json b/examples/database-sources/NeuroMorpho/metadata.json new file mode 100644 index 00000000..3eb312ce --- /dev/null +++ b/examples/database-sources/NeuroMorpho/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The goal of NeuroMorpho.Org is to provide dense coverage of available reconstruction data for the neuroscience community. Data sharing through NeuroMorpho.Org enables the full and continuing research potential of existing digital reconstruction data.", + "url": "https://neuromorpho.org", + "license" : { + "id": "https://creativecommons.org/licenses/by/4.0/", + "label": "CC BY 4.0" + } +} \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 7474f808..71535441 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -23,6 +23,7 @@ }, "outputs": [], "source": [ + "import os\n", "import json\n", "\n", "from kgforge.core import KnowledgeGraphForge\n", @@ -31,24 +32,23 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "import getpass\n", - "TOKEN = getpass.getpass()" + "# import getpass\n", + "# TOKEN = getpass.getpass()" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", - "BUCKET = \"neurosciencegraph/datamodels\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\",\n", - "token=TOKEN, endpoint=endpoint, bucket=BUCKET)" + "BUCKET = \"dke/kgforge\"\n", + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", endpoint=endpoint, bucket=BUCKET)" ] }, { @@ -60,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -69,7 +69,8 @@ "text": [ "Available Database sources:\n", "UniProt\n", - "NeuroElectro\n" + "NeuroElectro\n", + "NeuroMorpho\n" ] } ], @@ -79,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -88,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -106,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -116,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -125,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -135,6 +136,7 @@ "Available Database sources:\n", "UniProt\n", "NeuroElectro\n", + "NeuroMorpho\n", "DemoDB\n" ] } @@ -152,7 +154,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -171,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -198,7 +200,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -219,7 +221,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -229,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -268,7 +270,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -297,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -329,7 +331,7 @@ " PREFIX void: \n", " PREFIX xml: \n", " PREFIX xsd: \n", - " PREFIX : \n", + " PREFIX : \n", " SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id rdf:type schema:ScholarlyArticle;\n", " ?_constrainedBy;\n", " ?_createdAt;\n", @@ -358,7 +360,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -367,7 +369,7 @@ "2" ] }, - "execution_count": 16, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -378,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -387,49 +389,68 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/35463\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Rationally, an increased intrinsic excitability of dorsal horn neurons could be a factor contributing to alter the gain of the nociceptive system during central sensitization, however direct evidence is scarce. Here we have examined this hypothesis using current and voltage-clamp recordings from dorsal horn neurons in the spinal cord in vitro preparation obtained from mice pups of either sex. Cords were extracted from carrageenan-pretreated and control animals to allow for comparison. Dorsal horn neurons from treated animals showed significantly larger and faster synaptic responses. Synaptic changes started developing shortly after inflammation (1 h) and developed further after a longer-term inflammation (20 h). However, these neurons showed biphasic changes in membrane excitability with an increase shortly after inflammation and a decrease in the longer term. Concomitant changes were observed in transient (I(A)) and sustained potassium currents (I(DR)). Prolonged superfusion of naive spinal cords with NMDA led to a decreased neuronal excitability and to increased potassium currents. Results suggest that excitability plays a role more complex than expected during the process of central sensitization of dorsal horn neurons and that modulation of potassium currents may contribute to shape the changing states of excitability. The decreased excitability observed after long-term inflammation is interpreted as a homeostatic correction to an abnormal state of synaptic activity.\n", + " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Rivera-Arconada\n", - " givenName: Ivan\n", + " familyName: Henderson\n", + " givenName: Z\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Morris\n", + " givenName: N P\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Grimwood\n", + " givenName: P\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Fiddler\n", + " givenName: G\n", " }\n", " {\n", " type: Person\n", - " familyName: Lopez-Garcia\n", - " givenName: Jose A\n", + " familyName: Yang\n", + " givenName: H W\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Appenteng\n", + " givenName: K\n", " }\n", " ]\n", - " datePublished: 2010-4-14\n", + " datePublished: 2001-2-12\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 20392959\n", + " value: 11169477\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1523/JNEUROSCI.4359-09.2010\n", + " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0270-6474\n", - " name: The Journal of neuroscience : the official journal of the Society for Neuroscience\n", - " publisher: Society for Neuroscience\n", + " issn: 0021-9967\n", + " name: The Journal of comparative neurology\n", " }\n", - " name: article_35463\n", - " sameAs: http://www.jneurosci.org/content/30/15/5376.long\n", - " title: Changes in membrane excitability and potassium currents in sensitized dorsal horn neurons of mice pups.\n", - " url: http://www.jneurosci.org/content/30/15/5376.long\n", + " name: article_91941\n", + " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", + " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", "}\n" ] } @@ -440,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -456,7 +477,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -482,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -491,7 +512,7 @@ "10" ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -502,7 +523,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -511,7 +532,7 @@ "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/A0B137')" ] }, - "execution_count": 21, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -529,7 +550,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -538,7 +559,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -573,7 +594,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -582,7 +603,7 @@ "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0B137', _inner_sync=False)" ] }, - "execution_count": 24, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -600,7 +621,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -609,7 +630,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -629,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -638,7 +659,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -647,7 +668,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -732,6 +753,338 @@ "mapping = DictionaryMapping.load(\"../../database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson\")\n", "print(json.dumps(forge.as_jsonld(uniprot.map(dict_resource, mapping)), indent=4))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query the NeuroMorpho WebService" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "neuromorpho = sources['NeuroMorpho']" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "nmo_filters = {\"species\": \"rat,mouse,human\", \"response_loc\": [\"_embedded\", \"neuronResources\"]}" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n" + ] + } + ], + "source": [ + "nmo_resources = forge.search(nmo_filters, db_source='NeuroMorpho')" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "import uuid" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "new_morphology = neuromorpho.map(nmo_resources[0], 'NeuronMorphology')" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"@type\": [\n", + " \"Dataset\",\n", + " \"NeuronMorphology\"\n", + " ],\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/000000001\",\n", + " \"brainLocation\": {\n", + " \"@type\": \"Class\",\n", + " \"@id\": \"http://purl.obolibrary.org/obo/UBERON_0001950\",\n", + " \"label\": \"neocortex\",\n", + " \"subClassOf\": \"prov:Entity\",\n", + " \"isDefinedBy\": \"http://bbp.epfl.ch/neurosciencegraph/ontologies/mtypes\"\n", + " },\n", + " \"contribution\": {\n", + " \"@type\": \"Contribution\",\n", + " \"agent\": {\n", + " \"@type\": \"Organization\",\n", + " \"label\": \"George Mason University\"\n", + " }\n", + " },\n", + " \"identifier\": 1,\n", + " \"archive\": \"Wearne_Hof\",\n", + " \"name\": \"cnic_001\",\n", + " \"generation\": {\n", + " \"@type\": \"Generation\",\n", + " \"activity\": {\n", + " \"@type\": \"nsg:NeuronMorphologyReconstruction\",\n", + " \"hadProtocol\": {}\n", + " }\n", + " },\n", + " \"subject\": {\n", + " \"@type\": \"Subject\",\n", + " \"species\": {\n", + " \"label\": \"monkey\"\n", + " },\n", + " \"strain\": {\n", + " \"@type\": \"Class\",\n", + " \"@id\": \"http://purl.obolibrary.org/obo/NCBITaxon_9544\",\n", + " \"label\": \"Macaca mulatta\",\n", + " \"subClassOf\": \"obo:NCBITaxon_9443\"\n", + " }\n", + " },\n", + " \"license\": {\n", + " \"@type\": \"License\",\n", + " \"@id\": \"https://neuromorpho.org\"\n", + " },\n", + " \"objectOfStudy\": {\n", + " \"@type\": \"ObjectOfStudy\",\n", + " \"@id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", + " \"label\": \"Single Cell\"\n", + " },\n", + " \"note\": \"When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.1 (May 2015). The pre-6.1 version of the processed file is available for download here.\",\n", + " \"dateCreated\": \"2005-12-31\",\n", + " \"dateUploaded\": \"2006-08-01\",\n", + " \"description\": \"Morphology of cnic_001 obtained from NeuroMorpho API.\",\n", + " \"stain\": \"lucifer yellow\"\n", + " }\n", + "]\n" + ] + } + ], + "source": [ + "print(json.dumps(forge.as_jsonld(new_morphology), indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "format_file = neuromorpho.service.files_download['endpoint'] + \"/{}/Source-Version/{}.swc\"" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "example_url = format_file.format(new_morphology[0].archive.lower(), new_morphology[0].name)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "file_path = f\"./downloaded/{example_url.split('/')[-1]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n" + ] + } + ], + "source": [ + "neuromorpho._download_one(example_url, file_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "neuromorpho.attach_file(new_morphology[0], file_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"@type\": [\n", + " \"Dataset\",\n", + " \"NeuronMorphology\"\n", + " ],\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/000000001\",\n", + " \"brainLocation\": {\n", + " \"@type\": \"Class\",\n", + " \"@id\": \"http://purl.obolibrary.org/obo/UBERON_0001950\",\n", + " \"label\": \"neocortex\",\n", + " \"subClassOf\": \"prov:Entity\",\n", + " \"isDefinedBy\": \"http://bbp.epfl.ch/neurosciencegraph/ontologies/mtypes\"\n", + " },\n", + " \"contribution\": {\n", + " \"@type\": \"Contribution\",\n", + " \"agent\": {\n", + " \"@type\": \"Organization\",\n", + " \"label\": \"George Mason University\"\n", + " }\n", + " },\n", + " \"identifier\": 1,\n", + " \"archive\": \"Wearne_Hof\",\n", + " \"name\": \"cnic_001\",\n", + " \"generation\": {\n", + " \"@type\": \"Generation\",\n", + " \"activity\": {\n", + " \"@type\": \"nsg:NeuronMorphologyReconstruction\",\n", + " \"hadProtocol\": {}\n", + " }\n", + " },\n", + " \"subject\": {\n", + " \"@type\": \"Subject\",\n", + " \"species\": {\n", + " \"label\": \"monkey\"\n", + " },\n", + " \"strain\": {\n", + " \"@type\": \"Class\",\n", + " \"@id\": \"http://purl.obolibrary.org/obo/NCBITaxon_9544\",\n", + " \"label\": \"Macaca mulatta\",\n", + " \"subClassOf\": \"obo:NCBITaxon_9443\"\n", + " }\n", + " },\n", + " \"license\": {\n", + " \"@type\": \"License\",\n", + " \"@id\": \"https://neuromorpho.org\"\n", + " },\n", + " \"objectOfStudy\": {\n", + " \"@type\": \"ObjectOfStudy\",\n", + " \"@id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", + " \"label\": \"Single Cell\"\n", + " },\n", + " \"note\": \"When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.1 (May 2015). The pre-6.1 version of the processed file is available for download here.\",\n", + " \"dateCreated\": \"2005-12-31\",\n", + " \"dateUploaded\": \"2006-08-01\",\n", + " \"description\": \"Morphology of cnic_001 obtained from NeuroMorpho API.\",\n", + " \"stain\": \"lucifer yellow\",\n", + " \"distribution\": {\n", + " \"@type\": \"DataDownload\",\n", + " \"contentSize\": {\n", + " \"unitCode\": \"bytes\",\n", + " \"value\": 65525\n", + " },\n", + " \"digest\": {\n", + " \"algorithm\": \"SHA-256\",\n", + " \"value\": \"d7b494c3b2bc244c445283594dc1d4061d60013fe6abb4e9e302dd9798324e03\"\n", + " },\n", + " \"encodingFormat\": \"application/octet-stream\",\n", + " \"name\": \"cnic_001.swc\",\n", + " \"contentUrl\": \"https://staging.nise.bbp.epfl.ch/nexus/v1/files/dke/kgforge/9ee023ae-d0d5-4e6a-83b9-94f954cd970e\",\n", + " \"atLocation\": {\n", + " \"@type\": \"Location\",\n", + " \"store\": {\n", + " \"@id\": \"https://bbp.epfl.ch/dke/kgforge/gpfsStore\",\n", + " \"@type\": \"RemoteDiskStorage\",\n", + " \"_rev\": 1\n", + " },\n", + " \"location\": \"file:///gpfs/bbp.cscs.ch/data/project/nexustest/nexus-staging/dke/kgforge/8/e/6/8/b/a/6/c/cnic_001.swc\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(forge.as_jsonld(new_morphology[0]), indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _validate_one\n", + " False\n", + " ReportableRuntimeError: Evaluation path too deep!\n", + "->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>\n" + ] + } + ], + "source": [ + "forge.validate(new_morphology[0], type_=\"NeuronMorphology\")" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _register_one\n", + " False\n", + " RegistrationError: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/resources/dke/kgforge/datashapes%3Aneuronmorphology\n" + ] + } + ], + "source": [ + "forge.register(new_morphology[0], schema_id=\"datashapes:neuronmorphology\")" + ] } ], "metadata": { diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index 88223025..4b9e25a6 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -120,6 +120,7 @@ def _initialize_service(self, source: str, **source_config) -> Any: dirpath = Path(source) return self._service_from_directory(dirpath, **source_config) elif origin == "web_service": + source_config['service_context'] = self.context return self._service_from_web_service(source, **source_config) elif origin == "store": store = import_class(source, "stores") diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index f6c2fb24..b1367aa6 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -47,7 +47,7 @@ from kgforge.core.wrappings.paths import PathsWrapper, wrap_paths from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping -from kgforge.specializations.databases import StoreDatabase +from kgforge.specializations.databases import StoreDatabase, WebServiceDatabase class KnowledgeGraphForge: @@ -1011,9 +1011,9 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], dbs = {} for name in names: config = all_config[name] - origin = config.get('origin') + origin = config['origin'] + source = config['source'] if origin == 'store': - source = config.get('source') # Reuse complete configuration of the store when Nexus is called if source == store_config['name'] == 'BlueBrainNexus': store_copy = deepcopy(store_config) @@ -1021,10 +1021,11 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], "source", "name", store_copy.keys()) config['model_context'] = model_context - config.update(origin=origin) - config.update(source=source) config['name'] = name dbs[name] = StoreDatabase(self, **config) + elif origin == 'web_service': + config['name'] = name + dbs[name] = WebServiceDatabase(self, **config) else: raise NotImplementedError(f'Database from {origin} is not yet implemented.') return dbs diff --git a/kgforge/specializations/databases/__init__.py b/kgforge/specializations/databases/__init__.py index 081f712c..a4b12dc8 100644 --- a/kgforge/specializations/databases/__init__.py +++ b/kgforge/specializations/databases/__init__.py @@ -12,4 +12,5 @@ # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . -from .store_database import StoreDatabase \ No newline at end of file +from .store_database import StoreDatabase +from .webservice_database import WebServiceDatabase \ No newline at end of file diff --git a/kgforge/specializations/databases/store_database.py b/kgforge/specializations/databases/store_database.py index 9f785e70..e9bbb8e2 100644 --- a/kgforge/specializations/databases/store_database.py +++ b/kgforge/specializations/databases/store_database.py @@ -16,6 +16,7 @@ from kgforge.core.archetypes import Store, Database from kgforge.core.commons.execution import not_supported from kgforge.specializations.stores.bluebrain_nexus import BlueBrainNexus +from kgforge.specializations.databases.utils import type_from_filters class StoreDatabase(Database): @@ -93,23 +94,4 @@ def _service_from_store(store: Callable, **store_config) -> Store: return store(**store_config) def health(self) -> Callable: - not_supported() - -def type_from_filters(*filters) -> Optional[str]: - """Returns the first `type` found in filters.""" - resource_type = None - filters = filters[0] - if isinstance(filters, dict): - if 'type' in filters: - resource_type = filters['type'] - else: - # check filters grouping - if isinstance(filters, (list, tuple)): - filters = [filter for filter in filters] - else: - filters = [filters] - for filter in filters: - if 'type' in filter.path and filter.operator is "__eq__": - resource_type = filter.value - break - return resource_type \ No newline at end of file + not_supported() \ No newline at end of file diff --git a/kgforge/specializations/databases/webservice_database.py b/kgforge/specializations/databases/webservice_database.py new file mode 100644 index 00000000..445d010a --- /dev/null +++ b/kgforge/specializations/databases/webservice_database.py @@ -0,0 +1,187 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . +import copy +import requests +import asyncio +from aiohttp import ClientSession +from requests.exceptions import SSLError +from typing import Callable, List, Optional, Any + +from kgforge.core import Resource +from kgforge.core.archetypes import Database, Store +from kgforge.core.commons.exceptions import ConfigurationError, DownloadingError +from kgforge.core.commons.execution import not_supported, catch +from kgforge.specializations.databases.utils import type_from_filters +from kgforge.specializations.stores.bluebrain_nexus import BlueBrainNexus, _error_message +from kgforge.specializations.stores.databases import WebService +from kgforge.specializations.resources.datasets import _set + + +class WebServiceDatabase(Database): + """A high-level class to retrieve and create Database-related Resources.""" + + + _REQUIRED = ("name", "origin", "source", "model") + + def __init__(self, forge: Optional["KnowledgeGraphForge"], + **config) -> None: + """ + The properties defining the WebServiceDatabase are: + + :param forge: To use forge utilities + name: - REQUIRED + origin: <'web_service'> - REQUIRED + source: - REQUIRED + bucket: + endpoint: + token: + source: + context: <'directory', 'url', or 'store'> + bucket: + iri: + """ + self._check_properties(**config) + self.name = config.pop('name') + source = config.pop('source') + self._health = config.pop('health', None) + super().__init__(forge, source, **config) + + def _check_properties(self, **info): + properties = info.keys() + for r in self._REQUIRED: + if r not in properties: + raise ValueError(f'Missing {r} from the properties to define the DatabasSource') + + def search(self, resolvers, *filters, **params): + """Search within the database. + + :param keep_original: bool + """ + keep_original = params.pop('keep_original', True) + unmapped_resources = self.service.search(resolvers, *filters, **params) + if isinstance(self.service, BlueBrainNexus) or keep_original: + return unmapped_resources + else: + # Try to find the type of the resources within the filters + resource_type = type_from_filters(*filters) + return self.map_resources(unmapped_resources, resource_type=resource_type) + + def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, + offset: Optional[int] = None,**params): + not_supported() + + def elastic(**params): + not_supported() + + @catch + def attach_file(self, resource, path: str, url: str = None, + content_type: str = None, + as_part: bool=False, save: bool = False) -> None: + """Add (different) files as parts or distribution + + :param resource: Resource to which attach files + :param path: DirPath or URL - Location of the file + :param content_type: the file type + :param as_part: indicate how to add the file. + Default adds the file as distribution + :param save: If path is url, clean up the downloaded files + """ + if url: + self._download_one(url, path) + action = self._forge.attach(path, content_type) + if as_part: + distribution = Resource(distribution=action) + _set(resource, "hasPart", distribution) + else: + _set(resource, "distribution", action) + + @catch + def download(self, urls: str, paths: str, overwrite: bool = False) -> None: + # path: DirPath. + """Download files """ + pass + + def _download_many(self, urls: List[str], + paths: List[str]) -> None: + async def _bulk(): + loop = asyncio.get_event_loop() + semaphore = asyncio.Semaphore(self.service.max_connection) + async with ClientSession(headers=self.service.headers_download) as session: + tasks = ( + _create_task(x, y, loop, semaphore, session) + for x, y in zip(urls, paths) + ) + return await asyncio.gather(*tasks) + + def _create_task(url, path, loop, semaphore, session): + return loop.create_task( + _download(url, path, semaphore, session) + ) + + async def _download(url, path, semaphore, session): + async with semaphore: + params_download = copy.deepcopy(self.service.params.get('download', {})) + async with session.get(url, params=params_download) as response: + try: + response.raise_for_status() + except Exception as e: + raise DownloadingError( + f"Downloading:{_error_message(e)}" + ) + else: + with open(path, "wb") as f: + data = await response.read() + f.write(data) + + return asyncio.run(_bulk()) + + def _download_one(self, url: str, path: str) -> None: + try: + params_download = copy.deepcopy(self.service.params.get('download', {})) + response = requests.get( + url=url, + headers=self.service.headers_download, + params=params_download, + verify=False + ) + response.raise_for_status() + except Exception as e: + raise DownloadingError( + f"Downloading from failed :{_error_message(e)}" + ) + else: + with open(path, "wb") as f: + for chunk in response.iter_content(chunk_size=4096): + f.write(chunk) + + def health(self) -> Callable: + if self._health: + try: + response = requests.get(self._health) + except SSLError: + response = requests.get(self._health, verify=False) + return response.json() + else: + raise ConfigurationError('Health information not reachable with given configuration. \ + Define health in configuration arguments or set _health.') + + @staticmethod + def _service_from_web_service(endpoint: str, **source_config) -> Any: + return WebService(endpoint, **source_config) + + @staticmethod + def _service_from_store(store: Callable, **store_config) -> Store: + not_supported() \ No newline at end of file diff --git a/kgforge/specializations/resources/datasets.py b/kgforge/specializations/resources/datasets.py index 50f0f973..a9079910 100644 --- a/kgforge/specializations/resources/datasets.py +++ b/kgforge/specializations/resources/datasets.py @@ -167,9 +167,9 @@ def _(d): return [_(d) for d in data] if isinstance(data, List) else _(data) -def _set(dataset: Dataset, attr: str, data: Union[Resource, List[Resource], LazyAction]) -> None: - if hasattr(dataset, attr): - value = getattr(dataset, attr) +def _set(resource: Resource, attr: str, data: Union[Resource, List[Resource], LazyAction]) -> None: + if hasattr(resource, attr): + value = getattr(resource, attr) if isinstance(value, List): if isinstance(data, List): value.extend(data) @@ -180,6 +180,6 @@ def _set(dataset: Dataset, attr: str, data: Union[Resource, List[Resource], Lazy new = [value, *data] else: new = [value, data] - setattr(dataset, attr, new) + setattr(resource, attr, new) else: - setattr(dataset, attr, data) + setattr(resource, attr, data) diff --git a/kgforge/specializations/stores/__init__.py b/kgforge/specializations/stores/__init__.py index def3492d..0adf07d0 100644 --- a/kgforge/specializations/stores/__init__.py +++ b/kgforge/specializations/stores/__init__.py @@ -14,5 +14,4 @@ from .bluebrain_nexus import BlueBrainNexus from .demo_store import DemoStore -from .sparql import SPARQLStore -from .uniprot_store import UniProtStore +from .sparql import SPARQLStore \ No newline at end of file diff --git a/kgforge/specializations/stores/databases/__init__.py b/kgforge/specializations/stores/databases/__init__.py index 711555f6..8b95aec5 100644 --- a/kgforge/specializations/stores/databases/__init__.py +++ b/kgforge/specializations/stores/databases/__init__.py @@ -13,3 +13,4 @@ # along with Blue Brain Nexus Forge. If not, see . from .service import Service +from .web_service import WebService \ No newline at end of file diff --git a/kgforge/specializations/stores/databases/web_service.py b/kgforge/specializations/stores/databases/web_service.py new file mode 100644 index 00000000..58c34e11 --- /dev/null +++ b/kgforge/specializations/stores/databases/web_service.py @@ -0,0 +1,81 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + + +from logging import exception +import copy +import json +from pathlib import Path +from collections import namedtuple +from copy import deepcopy +from enum import Enum +from typing import Callable, Dict, List, Optional, Union, Tuple +from urllib.error import URLError +from urllib.parse import quote_plus, urlparse + +import requests +from numpy import nan +from requests import HTTPError + +from kgforge.core import Resource +from kgforge.core.commons.context import Context +from kgforge.core.commons.exceptions import DownloadingError +from kgforge.core.conversions.rdf import ( + _from_jsonld_one, + recursive_resolve, +) +from kgforge.core.wrappings.dict import DictWrapper, wrap_dict +from kgforge.specializations.databases.utils import request +from kgforge.specializations.stores.bluebrain_nexus import _error_message + + +class WebService: + + def __init__( + self, + endpoint: str, + model_context: Context, + service_context: Context, + max_connection: int, + content_type: str, + accept: str, + **params, + ): + + self.endpoint = endpoint + self.model_context = model_context + self.context_cache: Dict = dict() + self.context = service_context + self.max_connection = max_connection + self.files_download = params.pop('files_download', None) + self.params = copy.deepcopy(params) + + self.headers = {"Content-Type": content_type, "Accept": accept} + + self.headers_download = { + "Content-Type": self.files_download["Content-Type"] + if self.files_download and "Content-Type" in self.files_download + else "text/plain", + "Accept": self.files_download["Accept"] + if self.files_download and "Accept" in self.files_download + else "text/plain", + } + + def search(self, resolvers, *filters, **params): + # resolvers are not used, just passed because of previous methods shapes + filter_params = filters[0] + if not isinstance(filter_params, dict): + raise NotImplementedError('Currently only the use of a dictionary is implemented') + query_params = {**filter_params, **params} + return request(self.endpoint, self.headers, **query_params) \ No newline at end of file diff --git a/kgforge/specializations/stores/uniprot_store.py b/kgforge/specializations/stores/uniprot_store.py deleted file mode 100644 index 431a39f9..00000000 --- a/kgforge/specializations/stores/uniprot_store.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Blue Brain Nexus Forge is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser -# General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Blue Brain Nexus Forge. If not, see . - -import json -from pyld import jsonld -from copy import deepcopy -from pathlib import Path -from typing import Dict, List, Optional, Union, Any -from uuid import uuid4 -import requests -from rdflib import Graph -from rdflib.plugins.sparql.parser import Query -from requests import HTTPError -from SPARQLWrapper import SPARQLWrapper, JSON - -from kgforge.core import Resource -from kgforge.core.archetypes import Resolver, Store -from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql -# from kgforge.specializations.stores.bluebrain_nexus import ( -# CategoryDataType, -# build_sparql_query_statements, -# format_type, -# _create_select_query -# ) -from kgforge.specializations.stores import SPARQLStore -from kgforge.core.commons.context import Context -from kgforge.core.commons.exceptions import QueryingError -from kgforge.core.commons.execution import not_supported -from kgforge.core.conversions.rdf import as_jsonld, from_jsonld -# from kgforge.core.conversions.json import as_json, from_json -# from kgforge.core.wrappings.dict import wrap_dict -# from kgforge.core.wrappings.paths import create_filters_from_dict -# from urllib.parse import quote_plus, unquote, urlparse, parse_qs - - -UNIPROT_RESTAPI_URL = 'https://rest.uniprot.org/uniprotkb' -UNIPROT_ENTITY_PREFIX = "http://purl.uniprot.org" - - -class UniProtStore(SPARQLStore): - - """A Store specialized for SPARQL queries, supporting only Reading (searching) methods.""" - - def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, - token: Optional[str] = None, versioned_id_template: Optional[str] = None, - file_resource_mapping: Optional[str] = None, - model_context: Optional[Context] = None, - searchendpoints: Optional[Dict] = None,) -> None: - super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, - model_context, searchendpoints) - - @staticmethod - def resources_from_results(results): - resources = [] - for result in results: - resource = {} - for k, v in result.items(): - if v['type'] =='literal' and ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean'): - value = json.loads(str(v["value"]).lower()) - - elif v['type'] =='literal' and ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer'): - value = int(v["value"]) - # check if the value is an url, if so, get the metadata - elif v['type'] == 'uri' and UNIPROT_ENTITY_PREFIX in v['value']: - value = v["value"] - url = UNIPROT_RESTAPI_URL + f"/{value.split('/')[-1]}.json" - try: - response = requests.get(url) - response.raise_for_status() - except Exception as e: - raise QueryingError(e) - else: - metadata = response.json() - metadata['id'] = value - resource[k] = Resource(**metadata) - else: - resource[k] = v['value'] - resources.append(Resource(**resource)) - return resources \ No newline at end of file diff --git a/tests/specializations/databases/test_database.py b/tests/specializations/databases/test_database.py index 072d59bf..e0c36207 100644 --- a/tests/specializations/databases/test_database.py +++ b/tests/specializations/databases/test_database.py @@ -18,7 +18,8 @@ from kgforge.core import Resource, KnowledgeGraphForge from kgforge.core.commons.context import Context from kgforge.core.wrappings.paths import Filter -from kgforge.specializations.databases.store_database import StoreDatabase, type_from_filters +from kgforge.specializations.databases.store_database import StoreDatabase +from kgforge.specializations.databases.utils import type_from_filters from kgforge.core.commons.exceptions import ConfigurationError from kgforge.core.wrappings.dict import DictWrapper, wrap_dict import json From 9433b1f79fa95efaabbc54dd40c496a3b1baf411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 2 Nov 2022 17:40:49 +0100 Subject: [PATCH 20/30] Adding missing file. --- kgforge/specializations/databases/utils.py | 77 ++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 kgforge/specializations/databases/utils.py diff --git a/kgforge/specializations/databases/utils.py b/kgforge/specializations/databases/utils.py new file mode 100644 index 00000000..1402b279 --- /dev/null +++ b/kgforge/specializations/databases/utils.py @@ -0,0 +1,77 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import os +import json +import requests +from typing import Optional, Any + +from kgforge.core import Resource +from kgforge.core.commons.exceptions import DownloadingError, QueryingError + + +def type_from_filters(*filters) -> Optional[str]: + """Returns the first `type` found in filters.""" + resource_type = None + filters = filters[0] + if isinstance(filters, dict): + if 'type' in filters: + resource_type = filters['type'] + else: + # check filters grouping + if isinstance(filters, (list, tuple)): + filters = [filter for filter in filters] + else: + filters = [filters] + for filter in filters: + if 'type' in filter.path and filter.operator is "__eq__": + resource_type = filter.value + break + return resource_type + +def resources_from_results(results): + return [ + Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') + else (int(v["value"]) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer') + else v["value"] + ) + for k, v in x.items()} ) + for x in results + ] + +def request(url, headers, **params): + """Perform a HTTP request""" + response_location = params.pop('response_loc', None) + try: + response = requests.get( + url, params=params, headers=headers, verify=False + ) + response.raise_for_status() + except Exception as e: + raise QueryingError(e) + else: + data = response.json() + if response_location: + if isinstance(response_location, str): + results = data[response_location] + elif isinstance(response_location, (list, tuple)): + for inner in response_location: + data = data[inner] + results = data + return [Resource(**result) for result in results] + else: + results = data["results"]["bindings"] + return resources_from_results(results) \ No newline at end of file From 761e114b4a3af24517d01c2346529ee61538deeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 7 Nov 2022 10:23:35 +0100 Subject: [PATCH 21/30] Some changes. --- .../DictionaryMapping/NeuronMorphology.hjson | 26 +- .../17 - Database-sources.ipynb | 422 +++++++++++------- kgforge/core/archetypes/database.py | 2 +- kgforge/specializations/models/rdf_model.py | 4 +- 4 files changed, 287 insertions(+), 167 deletions(-) diff --git a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson index 48bc3708..55bec5b6 100644 --- a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson +++ b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -3,18 +3,24 @@ Dataset NeuronMorphology ] - id: forge.format('identifier', 'neuronmorphologies/neuromorpho', f'{x.neuron_id:09}') - brainLocation: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_MATCH') else x.brain_region[0] + id: forge.format('identifier', 'neuronmorphologies/neuromorpho', f'{x.bbpID}') + brainLocation: { + type: BrainLocation + brainRegion: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') if forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') else {'label': x.brain_region[0]} + } contribution: { type: Contribution agent: { type: Organization - label: "George Mason University" + label: George Mason University } } - identifier: x.neuron_id + identifier: { + propertyID: NeuroMorhoID + value: x.neuron_id + } archive: x.archive name: x.neuron_name generation: @@ -23,16 +29,13 @@ activity: { type: "nsg:NeuronMorphologyReconstruction" - hadProtocol:{ - - } } } subject: { type: Subject species: forge.resolve(x.species, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.species, scope='ontology', strategy='EXACT_MATCH') else {'label': x.species} - strain: forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') else {'label': x.scientific_name} + strain: forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') else None } license: { @@ -46,8 +49,11 @@ label: Single Cell } note: x.note - dateCreated: x.deposition_date - dateUploaded: x.upload_date + dateCreated: { + type: "xsd:dateTime" + value: x.deposition_date + } description: f"Morphology of {x.neuron_name} obtained from NeuroMorpho API." stain: x.stain + version: "8.4.7" } \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 71535441..2c5a3853 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -48,7 +48,7 @@ "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", "BUCKET = \"dke/kgforge\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", endpoint=endpoint, bucket=BUCKET)" + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", endpoint=endpoint, bucket=BUCKET, debug=True)" ] }, { @@ -277,9 +277,8 @@ "name": "stdout", "output_type": "stream", "text": [ - " db_sources\n", - " AttributeError: 'StoreDatabase' object has no attribute 'datatypes'\n", - "\n" + "Available Database sources:\n", + "UniProt\n" ] } ], @@ -389,68 +388,58 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/91941\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/34151\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: Neurons in the medial septal/diagonal band complex (MS/DB) in vivo exhibit rhythmic burst-firing activity that is phase-locked with the hippocampal theta rhythm. The aim was to assess the morphology of local axon collaterals of electrophysiologically identified MS/DB neurons using intracellular recording and biocytin injection in vitro. Cells were classified according to previous criteria into slow-firing, fast-spiking, regular-spiking, and burst-firing neurons; previous work has suggested that the slow-firing neurons are cholinergic and that the other types are GABAergic. A novel finding was the existence of two types of burst-firing neuron. Type I burst-firing neurons had significantly longer duration after hyperpolarisation potentials when held at -60 mV, and at -75 mV, type I neurons exhibited a low-threshold spike with more rapid activation and inactivation kinetics than those of type II neurons. We have, also for the first time, described the main features of the local axon collaterals of the five neuron types. All filled neurons possessed a main axon that gave forth 1-12 local primary axon collaterals. All electrophysiological types, except for the type I burst-firing neuron, had a main axon that coursed toward the fornix. Myelination of the main axon was a prominent feature of all but the slow-firing neurons. Branching of the primary axon collaterals of the fast-spiking and type I burst-firing neurons was more extensive than that of the other cell types, with those of the slow-firing neurons exhibiting the least branching. All cell types possessed axon collaterals of the en passant type, and some in addition had twiglike or basketlike axon terminals. All cell types made synapses on distal dendrites; a proportion of the fast-spiking and burst-firing cells in addition had basketlike terminals that made synaptic contacts on proximal dendrites and on somata. Two morphological types of somata were postsynaptic to the basket cells: large (20-30-microm) oval cells with dark cytoplasm, and large oval cells with paler cytoplasm, often with an apical dendrite. The presence of lamellar bodies in the large dark neurons suggests that they may be cholinergic neurons, because previous work has localised these structures in some neurons that stain for choline acetyltransferase. Our work suggests therefore that there may be GABAergic neurons in the MS/DB that form basket synaptic contacts on at least two types of target cell, possibly cholinergic and GABAergic neurons, which means that the basket cells could play a key role in the generation of rhythmic activity in the MS/DB.\n", + " abstract: This study examined the passive membrane and action potential properties of serotonergic and nonserotonergic neurons in the rostral ventromedial medulla (RVM) of the rat using whole cell patch-clamp recording techniques in the slice. Serotonergic neurons were identified by immunoreactivity for tryptophan hydroxylase (TrpH). Spinally projecting neurons were retrogradely labeled with 1'-dioactadecyl-3,3,3',3'-tetramethylindocarbodyanine perchlorate (DiI). Three types of neurons were identified within both spinally projecting serotonergic and nonserotonergic populations. Type 1 neurons exhibited irregular or sporadic spontaneous activity interspersed with periods of quiescence. Type 2 neurons were not spontaneously active and were additionally discriminated by a more negative resting membrane potential and a larger-amplitude action potential. Type 3 neurons fired repetitively without pause. Serotonergic neurons had a higher membrane resistance and greater action potential half-width than their nonserotonergic counterparts and rarely exhibited a fast afterhyperpolarization. Serotonergic type 3 neurons also fired more slowly and regularly than nonserotonergic type 3 neurons. Comparison of electrophysiological and immunohistochemical characteristics suggested that the smallest type 3 serotonergic neurons had an increased risk of immunohistochemical \"misclassification\" due to failure to detect TrpH, possibly due to more complete dialysis of intracellular contents during lengthy recordings. This risk was minimal for type 1 or 2 serotonergic neurons. The three different types of spinally projecting serotonergic neurons also differed markedly in their responsiveness to the mu opioid receptor agonist D-Ala2, NMePhe4, Gly5-ol]enkephalin. These results provide important new electrophysiological and pharmacological evidence for a significant heterogeneity among spinally projecting serotonergic RVM neurons. They may also provide a basis for resolving the controversy concerning the role of serotonergic RVM neurons in opioid analgesia.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Henderson\n", - " givenName: Z\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Morris\n", - " givenName: N P\n", + " familyName: Zhang\n", + " givenName: Liang\n", " }\n", " {\n", " type: Person\n", - " familyName: Grimwood\n", - " givenName: P\n", + " familyName: Sykes\n", + " givenName: Kenneth T\n", " }\n", " {\n", " type: Person\n", - " familyName: Fiddler\n", - " givenName: G\n", + " familyName: Buhler\n", + " givenName: Amber V\n", " }\n", " {\n", " type: Person\n", - " familyName: Yang\n", - " givenName: H W\n", - " }\n", - " {\n", - " type: Person\n", - " familyName: Appenteng\n", - " givenName: K\n", + " familyName: Hammond\n", + " givenName: Donna L\n", " }\n", " ]\n", - " datePublished: 2001-2-12\n", + " datePublished: 2006-3\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 11169477\n", + " value: 16338998\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1002/1096-9861(20010212)430:3<410::aid-cne1040>3.0.co;2-i\n", + " value: 10.1152/jn.00883.2005\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0021-9967\n", - " name: The Journal of comparative neurology\n", + " issn: 0022-3077\n", + " name: Journal of neurophysiology\n", " }\n", - " name: article_91941\n", - " sameAs: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", - " title: Morphology of local axon collaterals of electrophysiologically characterised neurons in the rat medial septal/ diagonal band complex.\n", - " url: http%3A%2F%2Fonlinelibrary.wiley.com%2Fdoi%2F10.1002%2F1096-9861%2820010212%29430%3A3%3C410%3A%3AAID-CNE1040%3E3.0.CO%3B2-I%2Fabstract%3Bjsessionid%3D1C9F6EE828E8DCFCF6E6F5CEA0D5DE57.f01t03\n", + " name: article_34151\n", + " sameAs: http://jn.physiology.org/content/95/3/1853.long\n", + " title: Electrophysiological heterogeneity of spinally projecting serotonergic and nonserotonergic neurons in the rostral ventromedial medulla.\n", + " url: http://jn.physiology.org/content/95/3/1853.long\n", "}\n" ] } @@ -761,11 +750,6 @@ "## Query the NeuroMorpho WebService" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, { "cell_type": "code", "execution_count": 32, @@ -817,13 +801,42 @@ "metadata": {}, "outputs": [], "source": [ - "new_morphology = neuromorpho.map(nmo_resources[0], 'NeuronMorphology')" + "nmo_resources[2].bbpID = str(uuid.uuid4())" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, _links={'self': {'href': 'http://neuromorpho.org/api/neuron/id/100'}, 'measurements': {'href': 'http://neuromorpho.org/api/morphometry/id/100'}, 'persistence_vector': {'href': 'http://neuromorpho.org/api/pvec/id/100'}}, age_classification='young', age_scale='Month', archive='Turner', attributes='Diameter, 3D, Angles', bbpID='f947f690-a5ef-4ea4-aecf-89eb1145f400', brain_region=['hippocampus', 'CA1'], cell_type=['pyramidal', 'principal cell'], corrected_value=None, corrected_xy='133.0', corrected_z='400.0', deposition_date='2005-12-31', domain='Dendrites, Soma, No Axon', experiment_condition=['Control'], gender='Male/Female', magnification='100', max_age='8.0', max_weight='350.0', min_age='2.0', min_weight='200.0', neuron_id=100, neuron_name='n419', note='When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.3 (February 2016). The pre-6.3 version of the processed file is available for download here.', objective_type='oil', original_format='Neurolucida.swc', physical_Integrity='Dendrites Complete', png_url='http://neuromorpho.org/images/imageFiles/Turner/n419.png', protocol='in vivo', reconstruction_software='Neurolucida', reference_doi=['10.1002/(SICI)1096-9861(19980216)391:3<335::AID-CNE4>3.0.CO;2-2'], reference_pmid=['9492204'], reported_value=None, reported_xy='25.0', reported_z='75.0', scientific_name='rattus norvegicus', shrinkage_corrected='Corrected', shrinkage_reported='Reported', slicing_direction='coronal', slicing_thickness='80', soma_surface='897.846', species='rat', stain='biocytin', strain='Fischer 344', surface='23909.3', upload_date='2006-08-01', volume='4215.04')" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nmo_resources[2]" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "new_morphology = neuromorpho.map(nmo_resources[2], 'NeuronMorphology')" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -836,13 +849,12 @@ " \"Dataset\",\n", " \"NeuronMorphology\"\n", " ],\n", - " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/000000001\",\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/f947f690-a5ef-4ea4-aecf-89eb1145f400\",\n", " \"brainLocation\": {\n", - " \"@type\": \"Class\",\n", - " \"@id\": \"http://purl.obolibrary.org/obo/UBERON_0001950\",\n", - " \"label\": \"neocortex\",\n", - " \"subClassOf\": \"prov:Entity\",\n", - " \"isDefinedBy\": \"http://bbp.epfl.ch/neurosciencegraph/ontologies/mtypes\"\n", + " \"@type\": \"BrainLocation\",\n", + " \"brainRegion\": {\n", + " \"label\": \"hippocampus\"\n", + " }\n", " },\n", " \"contribution\": {\n", " \"@type\": \"Contribution\",\n", @@ -851,26 +863,31 @@ " \"label\": \"George Mason University\"\n", " }\n", " },\n", - " \"identifier\": 1,\n", - " \"archive\": \"Wearne_Hof\",\n", - " \"name\": \"cnic_001\",\n", + " \"identifier\": {\n", + " \"propertyID\": \"NeuroMorhoID\",\n", + " \"value\": 100\n", + " },\n", + " \"archive\": \"Turner\",\n", + " \"name\": \"n419\",\n", " \"generation\": {\n", " \"@type\": \"Generation\",\n", " \"activity\": {\n", - " \"@type\": \"nsg:NeuronMorphologyReconstruction\",\n", - " \"hadProtocol\": {}\n", + " \"@type\": \"nsg:NeuronMorphologyReconstruction\"\n", " }\n", " },\n", " \"subject\": {\n", " \"@type\": \"Subject\",\n", " \"species\": {\n", - " \"label\": \"monkey\"\n", - " },\n", - " \"strain\": {\n", " \"@type\": \"Class\",\n", - " \"@id\": \"http://purl.obolibrary.org/obo/NCBITaxon_9544\",\n", - " \"label\": \"Macaca mulatta\",\n", - " \"subClassOf\": \"obo:NCBITaxon_9443\"\n", + " \"@id\": \"http://purl.obolibrary.org/obo/NCBITaxon_10116\",\n", + " \"label\": [\n", + " \"Rattus norvegicus\",\n", + " \"Rattus Norvegicus\"\n", + " ],\n", + " \"subClassOf\": [\n", + " \"nsg:Species\",\n", + " \"obo:NCBITaxon_10114\"\n", + " ]\n", " }\n", " },\n", " \"license\": {\n", @@ -882,11 +899,14 @@ " \"@id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", " \"label\": \"Single Cell\"\n", " },\n", - " \"note\": \"When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.1 (May 2015). The pre-6.1 version of the processed file is available for download here.\",\n", - " \"dateCreated\": \"2005-12-31\",\n", - " \"dateUploaded\": \"2006-08-01\",\n", - " \"description\": \"Morphology of cnic_001 obtained from NeuroMorpho API.\",\n", - " \"stain\": \"lucifer yellow\"\n", + " \"note\": \"When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.3 (February 2016). The pre-6.3 version of the processed file is available for download here.\",\n", + " \"dateCreated\": {\n", + " \"@type\": \"xsd:dateTime\",\n", + " \"value\": \"2005-12-31\"\n", + " },\n", + " \"description\": \"Morphology of n419 obtained from NeuroMorpho API.\",\n", + " \"stain\": \"biocytin\",\n", + " \"version\": \"8.4.7\"\n", " }\n", "]\n" ] @@ -898,16 +918,16 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ - "format_file = neuromorpho.service.files_download['endpoint'] + \"/{}/Source-Version/{}.swc\"" + "format_file = neuromorpho.service.files_download['endpoint'] + \"/{}/CNG version/{}.CNG.swc\"" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -916,16 +936,65 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ - "file_path = f\"./downloaded/{example_url.split('/')[-1]}\"" + "clean_name = ''.join(example_url.split('.')[:-2])+'.swc'" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'http://neuromorphoorg/dableFiles/turner/CNG version/n419.swc'" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_name" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "file_path = f\"./downloaded/{clean_name.split('/')[-1]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'./downloaded/n419.swc'" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_path" + ] + }, + { + "cell_type": "code", + "execution_count": 46, "metadata": {}, "outputs": [ { @@ -943,16 +1012,16 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ - "neuromorpho.attach_file(new_morphology[0], file_path)" + "neuromorpho.attach_file(new_morphology[0], file_path, content_type='application/swc')" ] }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 48, "metadata": {}, "outputs": [ { @@ -960,91 +1029,90 @@ "output_type": "stream", "text": [ "{\n", - " \"@context\": \"https://bbp.neuroshapes.org\",\n", - " \"@type\": [\n", - " \"Dataset\",\n", - " \"NeuronMorphology\"\n", - " ],\n", - " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/000000001\",\n", - " \"brainLocation\": {\n", - " \"@type\": \"Class\",\n", - " \"@id\": \"http://purl.obolibrary.org/obo/UBERON_0001950\",\n", - " \"label\": \"neocortex\",\n", - " \"subClassOf\": \"prov:Entity\",\n", - " \"isDefinedBy\": \"http://bbp.epfl.ch/neurosciencegraph/ontologies/mtypes\"\n", - " },\n", - " \"contribution\": {\n", - " \"@type\": \"Contribution\",\n", - " \"agent\": {\n", - " \"@type\": \"Organization\",\n", - " \"label\": \"George Mason University\"\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/f947f690-a5ef-4ea4-aecf-89eb1145f400\n", + " type:\n", + " [\n", + " Dataset\n", + " NeuronMorphology\n", + " ]\n", + " archive: Turner\n", + " brainLocation:\n", + " {\n", + " type: BrainLocation\n", + " brainRegion:\n", + " {\n", + " label: hippocampus\n", " }\n", - " },\n", - " \"identifier\": 1,\n", - " \"archive\": \"Wearne_Hof\",\n", - " \"name\": \"cnic_001\",\n", - " \"generation\": {\n", - " \"@type\": \"Generation\",\n", - " \"activity\": {\n", - " \"@type\": \"nsg:NeuronMorphologyReconstruction\",\n", - " \"hadProtocol\": {}\n", + " }\n", + " contribution:\n", + " {\n", + " type: Contribution\n", + " agent:\n", + " {\n", + " type: Organization\n", + " label: George Mason University\n", " }\n", - " },\n", - " \"subject\": {\n", - " \"@type\": \"Subject\",\n", - " \"species\": {\n", - " \"label\": \"monkey\"\n", - " },\n", - " \"strain\": {\n", - " \"@type\": \"Class\",\n", - " \"@id\": \"http://purl.obolibrary.org/obo/NCBITaxon_9544\",\n", - " \"label\": \"Macaca mulatta\",\n", - " \"subClassOf\": \"obo:NCBITaxon_9443\"\n", + " }\n", + " dateCreated:\n", + " {\n", + " type: xsd:dateTime\n", + " value: 2005-12-31\n", + " }\n", + " description: Morphology of n419 obtained from NeuroMorpho API.\n", + " distribution: LazyAction(operation=Store.upload, args=['./downloaded/n419.swc', 'application/swc'])\n", + " generation:\n", + " {\n", + " type: Generation\n", + " activity:\n", + " {\n", + " type: nsg:NeuronMorphologyReconstruction\n", " }\n", - " },\n", - " \"license\": {\n", - " \"@type\": \"License\",\n", - " \"@id\": \"https://neuromorpho.org\"\n", - " },\n", - " \"objectOfStudy\": {\n", - " \"@type\": \"ObjectOfStudy\",\n", - " \"@id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", - " \"label\": \"Single Cell\"\n", - " },\n", - " \"note\": \"When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.1 (May 2015). The pre-6.1 version of the processed file is available for download here.\",\n", - " \"dateCreated\": \"2005-12-31\",\n", - " \"dateUploaded\": \"2006-08-01\",\n", - " \"description\": \"Morphology of cnic_001 obtained from NeuroMorpho API.\",\n", - " \"stain\": \"lucifer yellow\",\n", - " \"distribution\": {\n", - " \"@type\": \"DataDownload\",\n", - " \"contentSize\": {\n", - " \"unitCode\": \"bytes\",\n", - " \"value\": 65525\n", - " },\n", - " \"digest\": {\n", - " \"algorithm\": \"SHA-256\",\n", - " \"value\": \"d7b494c3b2bc244c445283594dc1d4061d60013fe6abb4e9e302dd9798324e03\"\n", - " },\n", - " \"encodingFormat\": \"application/octet-stream\",\n", - " \"name\": \"cnic_001.swc\",\n", - " \"contentUrl\": \"https://staging.nise.bbp.epfl.ch/nexus/v1/files/dke/kgforge/9ee023ae-d0d5-4e6a-83b9-94f954cd970e\",\n", - " \"atLocation\": {\n", - " \"@type\": \"Location\",\n", - " \"store\": {\n", - " \"@id\": \"https://bbp.epfl.ch/dke/kgforge/gpfsStore\",\n", - " \"@type\": \"RemoteDiskStorage\",\n", - " \"_rev\": 1\n", - " },\n", - " \"location\": \"file:///gpfs/bbp.cscs.ch/data/project/nexustest/nexus-staging/dke/kgforge/8/e/6/8/b/a/6/c/cnic_001.swc\"\n", + " }\n", + " identifier:\n", + " {\n", + " propertyID: NeuroMorhoID\n", + " value: 100\n", + " }\n", + " license:\n", + " {\n", + " id: https://neuromorpho.org\n", + " type: License\n", + " }\n", + " name: n419\n", + " note: When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.3 (February 2016). The pre-6.3 version of the processed file is available for download here.\n", + " objectOfStudy:\n", + " {\n", + " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", + " type: ObjectOfStudy\n", + " label: Single Cell\n", + " }\n", + " stain: biocytin\n", + " subject:\n", + " {\n", + " type: Subject\n", + " species:\n", + " {\n", + " id: http://purl.obolibrary.org/obo/NCBITaxon_10116\n", + " type: Class\n", + " label:\n", + " [\n", + " Rattus norvegicus\n", + " Rattus Norvegicus\n", + " ]\n", + " subClassOf:\n", + " [\n", + " nsg:Species\n", + " obo:NCBITaxon_10114\n", + " ]\n", " }\n", " }\n", + " version: 8.4.7\n", "}\n" ] } ], "source": [ - "print(json.dumps(forge.as_jsonld(new_morphology[0]), indent=4))" + "print(new_morphology[0])" ] }, { @@ -1058,8 +1126,7 @@ "text": [ " _validate_one\n", " False\n", - " ReportableRuntimeError: Evaluation path too deep!\n", - "->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>->->>\n" + " ValidationError: resource has lazy actions which need to be executed before\n" ] } ], @@ -1071,20 +1138,67 @@ "cell_type": "code", "execution_count": 50, "metadata": {}, + "outputs": [], + "source": [ + "# forge.register(new_morphology[0], schema_id=\"datashapes:neuronmorphology\")" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "# new_morphology[0].brainLocation.brainRegion" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "# new_morphology[0].brainLocation.brainRegion = forge.resolve('hippocampal formation', scope='ontology', strategy='BEST_MATCH')" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - " _register_one\n", - " False\n", - " RegistrationError: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/resources/dke/kgforge/datashapes%3Aneuronmorphology\n" + "ename": "ValueError", + "evalue": "can't convert, resource contains LazyActions", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_as_jsonld_one\u001b[0;34m(resource, form, store_metadata, model_context, metadata_context, context_resolver, **params)\u001b[0m\n\u001b[1;32m 223\u001b[0m data_graph, metadata_graph, encoded_resource, json_array = _as_graphs(\n\u001b[0;32m--> 224\u001b[0;31m \u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstore_metadata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 225\u001b[0m )\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_as_graphs\u001b[0;34m(resource, store_metadata, context, metadata_context)\u001b[0m\n\u001b[1;32m 316\u001b[0m )\n\u001b[0;32m--> 317\u001b[0;31m \u001b[0mconverted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_add_ld_keys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_context\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 318\u001b[0m \u001b[0mconverted\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"@context\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdocument\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"@context\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_add_ld_keys\u001b[0;34m(rsc, context, base)\u001b[0m\n\u001b[1;32m 449\u001b[0m raise ValueError(\n\u001b[0;32m--> 450\u001b[0;31m \u001b[0;34m\"can't convert, resource contains LazyActions\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 451\u001b[0m )\n", + "\u001b[0;31mValueError\u001b[0m: can't convert, resource contains LazyActions", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_6770/138230985.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_jsonld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_morphology\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mstack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_stack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/forge.py\u001b[0m in \u001b[0;36mas_jsonld\u001b[0;34m(self, data, form, store_metadata, **params)\u001b[0m\n\u001b[1;32m 837\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_store\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetadata_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 838\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresolve_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 839\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 840\u001b[0m )\n\u001b[1;32m 841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36mas_jsonld\u001b[0;34m(data, form, store_metadata, model_context, metadata_context, context_resolver, **params)\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0mmetadata_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0mcontext_resolver\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mdispatch\u001b[0;34m(data, fun_many, fun_one, *args, **params)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfun_many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mResource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfun_one\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 96\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"not a Resource nor a list of Resource\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_as_jsonld_one\u001b[0;34m(resource, form, store_metadata, model_context, metadata_context, context_resolver, **params)\u001b[0m\n\u001b[1;32m 225\u001b[0m )\n\u001b[1;32m 226\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstore_metadata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetadata_graph\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: can't convert, resource contains LazyActions" ] } ], "source": [ - "forge.register(new_morphology[0], schema_id=\"datashapes:neuronmorphology\")" + "print(json.dumps(forge.as_jsonld(new_morphology[0]), indent=4))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index 4b9e25a6..59e9c557 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -88,7 +88,7 @@ def map(self, resources : Union[List[Union[Resource, str]], Union[Resource, str] def types(self): # TODO: add other datatypes used, for instance, inside the mappings - return list(self.mappings().keys()) + return list(self._model.mappings(self._model.source, False).keys()) @abstractmethod def search(self, resolvers, *filters, **params) -> Resource: diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index 66db5478..a81e227a 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -98,7 +98,7 @@ def _mappings(self, source: str) -> Dict[str, List[str]]: for x in dirpath.glob("*/*.hjson"): mappings.setdefault(x.stem, []).append(x.parent.name) else: - raise ValueError("unrecognized source") + raise ValueError(f"unrecognized source {dirpath}") return mappings def mapping(self, entity: str, source: str, type: Callable) -> Mapping: @@ -107,7 +107,7 @@ def mapping(self, entity: str, source: str, type: Callable) -> Mapping: if filepath.is_file(): return type.load(filepath) else: - raise ValueError("unrecognized entity type or source file") + raise ValueError(f"unrecognized entity type or source file: {filepath}") # Templates. From 96c227c59f75a4e93caa3e7b703fd0305b258548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 7 Nov 2022 10:24:23 +0100 Subject: [PATCH 22/30] Added DemoDB folder. --- .../DemoDB/mappings/DictionaryMappings/Example.hjson | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson diff --git a/examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson b/examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson new file mode 100644 index 00000000..0706644e --- /dev/null +++ b/examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson @@ -0,0 +1,9 @@ +{ + id: forge.format('identifier', "examples", x.id.split('/')[-1]) + type: [ + Entity + ] + name: x.name + label: x.label + subject: forge.resolve(x.subject, scope='ontology') if forge.resolve(x.subject, scope='ontology') is not None else {"label": x.subject} +} \ No newline at end of file From ad6b5fde10fe22d83333660018f6e1c5fbd7d18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 7 Nov 2022 10:25:38 +0100 Subject: [PATCH 23/30] Make types in db_sources optional --- kgforge/core/forge.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index b1367aa6..9eb0b6c2 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -531,14 +531,22 @@ def db_sources(self, type_: Optional[List[str]] = None, if isinstance(type_, list): for type in type_: for db in self._db_sources: - types = self._db_sources[db].datatypes() - if type in types: - sources[db] = self._db_sources[db] + try: + types = self._db_sources[db].types() + if type in types: + sources[db] = self._db_sources[db] + except ValueError: + # skiping db without mappings + continue else: for db in self._db_sources: - types = self._db_sources[db].datatypes() - if type_ in types: - sources[db] = self._db_sources[db] + try: + types = self._db_sources[db].types() + if type_ in types: + sources[db] = self._db_sources[db] + except ValueError: + # skiping db without mappings + continue if not sources: print("No Database sources were found for the given type(s)") if pretty: From 539c75c629f0c3ee94d07a6545a88550b7e1da6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 8 Nov 2022 09:22:21 +0100 Subject: [PATCH 24/30] Some changes. --- .../mappings/DictionaryMapping/NeuronMorphology.hjson | 2 +- kgforge/specializations/databases/utils.py | 2 +- kgforge/specializations/stores/databases/web_service.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson index 55bec5b6..23b9ae2a 100644 --- a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson +++ b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -51,7 +51,7 @@ note: x.note dateCreated: { type: "xsd:dateTime" - value: x.deposition_date + @value: x.date_created } description: f"Morphology of {x.neuron_name} obtained from NeuroMorpho API." stain: x.stain diff --git a/kgforge/specializations/databases/utils.py b/kgforge/specializations/databases/utils.py index 1402b279..04a04158 100644 --- a/kgforge/specializations/databases/utils.py +++ b/kgforge/specializations/databases/utils.py @@ -52,7 +52,7 @@ def resources_from_results(results): for x in results ] -def request(url, headers, **params): +def resources_from_request(url, headers, **params): """Perform a HTTP request""" response_location = params.pop('response_loc', None) try: diff --git a/kgforge/specializations/stores/databases/web_service.py b/kgforge/specializations/stores/databases/web_service.py index 58c34e11..2a1fec57 100644 --- a/kgforge/specializations/stores/databases/web_service.py +++ b/kgforge/specializations/stores/databases/web_service.py @@ -36,7 +36,7 @@ recursive_resolve, ) from kgforge.core.wrappings.dict import DictWrapper, wrap_dict -from kgforge.specializations.databases.utils import request +from kgforge.specializations.databases.utils import resources_from_request from kgforge.specializations.stores.bluebrain_nexus import _error_message @@ -78,4 +78,4 @@ def search(self, resolvers, *filters, **params): if not isinstance(filter_params, dict): raise NotImplementedError('Currently only the use of a dictionary is implemented') query_params = {**filter_params, **params} - return request(self.endpoint, self.headers, **query_params) \ No newline at end of file + return resources_from_request(self.endpoint, self.headers, **query_params) \ No newline at end of file From 264ba9a1a7d9434704e13c842d84e7b751860b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 10 Nov 2022 09:08:06 +0100 Subject: [PATCH 25/30] Added initial unit tests for WebService Database. --- .../DictionaryMapping/NeuronMorphology.hjson | 18 +- .../17 - Database-sources.ipynb | 477 ++++++++---------- .../databases/webservice_database.py | 15 +- .../stores/databases/web_service.py | 29 +- .../databases/test_database.py | 143 ++++-- tests/specializations/stores/test_sparql.py | 55 +- 6 files changed, 341 insertions(+), 396 deletions(-) diff --git a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson index 23b9ae2a..7c328891 100644 --- a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson +++ b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -2,21 +2,28 @@ type: [ Dataset NeuronMorphology + Entity ] id: forge.format('identifier', 'neuronmorphologies/neuromorpho', f'{x.bbpID}') brainLocation: { type: BrainLocation - brainRegion: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') if forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') else {'label': x.brain_region[0]} + brainRegion: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') } contribution: { type: Contribution agent: { + id: https://ror.org/02jqj7156 type: Organization label: George Mason University } } + dateCreated: + { + type: "xsd:dateTime" + @value: f"{x.date_formatted}" + } identifier: { propertyID: NeuroMorhoID value: x.neuron_id @@ -34,8 +41,8 @@ subject: { type: Subject - species: forge.resolve(x.species, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.species, scope='ontology', strategy='EXACT_MATCH') else {'label': x.species} - strain: forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') if forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_MATCH') else None + species: forge.resolve(x.species, scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') + strain: forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') } license: { @@ -48,11 +55,6 @@ id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells label: Single Cell } - note: x.note - dateCreated: { - type: "xsd:dateTime" - @value: x.date_created - } description: f"Morphology of {x.neuron_name} obtained from NeuroMorpho API." stain: x.stain version: "8.4.7" diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb index 2c5a3853..15e10a49 100644 --- a/examples/notebooks/getting-started/17 - Database-sources.ipynb +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -24,6 +24,7 @@ "outputs": [], "source": [ "import os\n", + "import uuid\n", "import json\n", "\n", "from kgforge.core import KnowledgeGraphForge\n", @@ -47,8 +48,10 @@ "outputs": [], "source": [ "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", - "BUCKET = \"dke/kgforge\"\n", - "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\", endpoint=endpoint, bucket=BUCKET, debug=True)" + "BUCKET = \"bbp/atlas\"\n", + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\",\n", + " # endpoint=endpoint,\n", + " bucket=BUCKET, debug=True)" ] }, { @@ -330,7 +333,7 @@ " PREFIX void: \n", " PREFIX xml: \n", " PREFIX xsd: \n", - " PREFIX : \n", + " PREFIX : \n", " SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id rdf:type schema:ScholarlyArticle;\n", " ?_constrainedBy;\n", " ?_createdAt;\n", @@ -345,7 +348,7 @@ " ?_updatedAt;\n", " ?_updatedBy . \n", " Filter (?_deprecated = 'false'^^xsd:boolean)\n", - " Filter (?_project = )}} LIMIT 2\n", + " Filter (?_project = )}} LIMIT 2\n", "\n" ] } @@ -388,58 +391,69 @@ "text": [ "{\n", " context: https://bbp.neuroshapes.org\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/34151\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/35177\n", " type:\n", " [\n", " Entity\n", " ScholarlyArticle\n", " ]\n", - " abstract: This study examined the passive membrane and action potential properties of serotonergic and nonserotonergic neurons in the rostral ventromedial medulla (RVM) of the rat using whole cell patch-clamp recording techniques in the slice. Serotonergic neurons were identified by immunoreactivity for tryptophan hydroxylase (TrpH). Spinally projecting neurons were retrogradely labeled with 1'-dioactadecyl-3,3,3',3'-tetramethylindocarbodyanine perchlorate (DiI). Three types of neurons were identified within both spinally projecting serotonergic and nonserotonergic populations. Type 1 neurons exhibited irregular or sporadic spontaneous activity interspersed with periods of quiescence. Type 2 neurons were not spontaneously active and were additionally discriminated by a more negative resting membrane potential and a larger-amplitude action potential. Type 3 neurons fired repetitively without pause. Serotonergic neurons had a higher membrane resistance and greater action potential half-width than their nonserotonergic counterparts and rarely exhibited a fast afterhyperpolarization. Serotonergic type 3 neurons also fired more slowly and regularly than nonserotonergic type 3 neurons. Comparison of electrophysiological and immunohistochemical characteristics suggested that the smallest type 3 serotonergic neurons had an increased risk of immunohistochemical \"misclassification\" due to failure to detect TrpH, possibly due to more complete dialysis of intracellular contents during lengthy recordings. This risk was minimal for type 1 or 2 serotonergic neurons. The three different types of spinally projecting serotonergic neurons also differed markedly in their responsiveness to the mu opioid receptor agonist D-Ala2, NMePhe4, Gly5-ol]enkephalin. These results provide important new electrophysiological and pharmacological evidence for a significant heterogeneity among spinally projecting serotonergic RVM neurons. They may also provide a basis for resolving the controversy concerning the role of serotonergic RVM neurons in opioid analgesia.\n", + " abstract: On the one hand, neuronal activity can cause changes in pH; on the other hand, changes in pH can modulate neuronal activity. Consequently, the pH of the brain is regulated at various levels. Here we show that steady-state pH and acid extrusion were diminished in cultured hippocampal neurons of mice with a targeted disruption of the Na(+)-driven Cl(-)/HCO(3)(-) exchanger Slc4a8. Because Slc4a8 was found to predominantly localize to presynaptic nerve endings, we hypothesize that Slc4a8 is a key regulator of presynaptic pH. Supporting this hypothesis, spontaneous glutamate release in the CA1 pyramidal layer was reduced but could be rescued by increasing the intracellular pH. The reduced excitability in vitro correlated with an increased seizure threshold in vivo. Together with the altered kinetics of stimulated synaptic vesicle release, these data suggest that Slc4a8 modulates glutamate release in a pH-dependent manner.\n", " author:\n", " [\n", " {\n", " type: Person\n", - " familyName: Zhang\n", - " givenName: Liang\n", + " familyName: Sinning\n", + " givenName: Anne\n", " }\n", " {\n", " type: Person\n", - " familyName: Sykes\n", - " givenName: Kenneth T\n", + " familyName: Liebmann\n", + " givenName: Lutz\n", " }\n", " {\n", " type: Person\n", - " familyName: Buhler\n", - " givenName: Amber V\n", + " familyName: Kougioumtzes\n", + " givenName: Alexandra\n", " }\n", " {\n", " type: Person\n", - " familyName: Hammond\n", - " givenName: Donna L\n", + " familyName: Westermann\n", + " givenName: Martin\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Bruehl\n", + " givenName: Claus\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Hübner\n", + " givenName: Christian A\n", " }\n", " ]\n", - " datePublished: 2006-3\n", + " datePublished: 2011-5-18\n", " identifier:\n", " [\n", " {\n", " propertyID: PMID\n", - " value: 16338998\n", + " value: 21593314\n", " }\n", " {\n", " propertyID: doi\n", - " value: 10.1152/jn.00883.2005\n", + " value: 10.1523/JNEUROSCI.0269-11.2011\n", " }\n", " ]\n", " isPartOf:\n", " {\n", " type: Periodical\n", - " issn: 0022-3077\n", - " name: Journal of neurophysiology\n", + " issn: 0270-6474\n", + " name: The Journal of neuroscience : the official journal of the Society for Neuroscience\n", + " publisher: Society for Neuroscience\n", " }\n", - " name: article_34151\n", - " sameAs: http://jn.physiology.org/content/95/3/1853.long\n", - " title: Electrophysiological heterogeneity of spinally projecting serotonergic and nonserotonergic neurons in the rostral ventromedial medulla.\n", - " url: http://jn.physiology.org/content/95/3/1853.long\n", + " name: article_35177\n", + " sameAs: http://www.jneurosci.org/content/31/20/7300.long\n", + " title: Synaptic glutamate release is modulated by the Na+ -driven Cl-/HCO₃⁻ exchanger Slc4a8.\n", + " url: http://www.jneurosci.org/content/31/20/7300.long\n", "}\n" ] } @@ -770,7 +784,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 56, "metadata": {}, "outputs": [ { @@ -783,245 +797,178 @@ } ], "source": [ - "nmo_resources = forge.search(nmo_filters, db_source='NeuroMorpho')" + "nmo_resources = forge.search(nmo_filters, db_source='NeuroMorpho', size=3, searchendpoint='select_query', q=\"species:mouse\")" ] }, { "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "import uuid" - ] - }, - { - "cell_type": "code", - "execution_count": 36, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.obolibrary.org/obo/UBERON_0002301', type='Class', label='Neocortical Layer', _inner_sync=False, prefLabel='Neocortical Layer', subClassOf='bmo:BrainLayer')" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "nmo_resources[2].bbpID = str(uuid.uuid4())" + "forge.resolve(nmo_resources[0].brain_region[0], scope='ontology', strategy='BEST_MATCH')" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 51, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, _links={'self': {'href': 'http://neuromorpho.org/api/neuron/id/100'}, 'measurements': {'href': 'http://neuromorpho.org/api/morphometry/id/100'}, 'persistence_vector': {'href': 'http://neuromorpho.org/api/pvec/id/100'}}, age_classification='young', age_scale='Month', archive='Turner', attributes='Diameter, 3D, Angles', bbpID='f947f690-a5ef-4ea4-aecf-89eb1145f400', brain_region=['hippocampus', 'CA1'], cell_type=['pyramidal', 'principal cell'], corrected_value=None, corrected_xy='133.0', corrected_z='400.0', deposition_date='2005-12-31', domain='Dendrites, Soma, No Axon', experiment_condition=['Control'], gender='Male/Female', magnification='100', max_age='8.0', max_weight='350.0', min_age='2.0', min_weight='200.0', neuron_id=100, neuron_name='n419', note='When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.3 (February 2016). The pre-6.3 version of the processed file is available for download here.', objective_type='oil', original_format='Neurolucida.swc', physical_Integrity='Dendrites Complete', png_url='http://neuromorpho.org/images/imageFiles/Turner/n419.png', protocol='in vivo', reconstruction_software='Neurolucida', reference_doi=['10.1002/(SICI)1096-9861(19980216)391:3<335::AID-CNE4>3.0.CO;2-2'], reference_pmid=['9492204'], reported_value=None, reported_xy='25.0', reported_z='75.0', scientific_name='rattus norvegicus', shrinkage_corrected='Corrected', shrinkage_reported='Reported', slicing_direction='coronal', slicing_thickness='80', soma_surface='897.846', species='rat', stain='biocytin', strain='Fischer 344', surface='23909.3', upload_date='2006-08-01', volume='4215.04')" + "['neocortex', 'occipital', 'layer 6']" ] }, - "execution_count": 37, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "nmo_resources[2]" + "nmo_resources[0].brain_region" ] }, { - "cell_type": "code", - "execution_count": 38, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "new_morphology = neuromorpho.map(nmo_resources[2], 'NeuronMorphology')" + "### Format date" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 35, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\n", - " {\n", - " \"@context\": \"https://bbp.neuroshapes.org\",\n", - " \"@type\": [\n", - " \"Dataset\",\n", - " \"NeuronMorphology\"\n", - " ],\n", - " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/f947f690-a5ef-4ea4-aecf-89eb1145f400\",\n", - " \"brainLocation\": {\n", - " \"@type\": \"BrainLocation\",\n", - " \"brainRegion\": {\n", - " \"label\": \"hippocampus\"\n", - " }\n", - " },\n", - " \"contribution\": {\n", - " \"@type\": \"Contribution\",\n", - " \"agent\": {\n", - " \"@type\": \"Organization\",\n", - " \"label\": \"George Mason University\"\n", - " }\n", - " },\n", - " \"identifier\": {\n", - " \"propertyID\": \"NeuroMorhoID\",\n", - " \"value\": 100\n", - " },\n", - " \"archive\": \"Turner\",\n", - " \"name\": \"n419\",\n", - " \"generation\": {\n", - " \"@type\": \"Generation\",\n", - " \"activity\": {\n", - " \"@type\": \"nsg:NeuronMorphologyReconstruction\"\n", - " }\n", - " },\n", - " \"subject\": {\n", - " \"@type\": \"Subject\",\n", - " \"species\": {\n", - " \"@type\": \"Class\",\n", - " \"@id\": \"http://purl.obolibrary.org/obo/NCBITaxon_10116\",\n", - " \"label\": [\n", - " \"Rattus norvegicus\",\n", - " \"Rattus Norvegicus\"\n", - " ],\n", - " \"subClassOf\": [\n", - " \"nsg:Species\",\n", - " \"obo:NCBITaxon_10114\"\n", - " ]\n", - " }\n", - " },\n", - " \"license\": {\n", - " \"@type\": \"License\",\n", - " \"@id\": \"https://neuromorpho.org\"\n", - " },\n", - " \"objectOfStudy\": {\n", - " \"@type\": \"ObjectOfStudy\",\n", - " \"@id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", - " \"label\": \"Single Cell\"\n", - " },\n", - " \"note\": \"When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.3 (February 2016). The pre-6.3 version of the processed file is available for download here.\",\n", - " \"dateCreated\": {\n", - " \"@type\": \"xsd:dateTime\",\n", - " \"value\": \"2005-12-31\"\n", - " },\n", - " \"description\": \"Morphology of n419 obtained from NeuroMorpho API.\",\n", - " \"stain\": \"biocytin\",\n", - " \"version\": \"8.4.7\"\n", - " }\n", - "]\n" - ] - } - ], + "outputs": [], "source": [ - "print(json.dumps(forge.as_jsonld(new_morphology), indent=4))" + "from datetime import datetime" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ - "format_file = neuromorpho.service.files_download['endpoint'] + \"/{}/CNG version/{}.CNG.swc\"" + "for resource in nmo_resources:\n", + " resource.bbpID = str(uuid.uuid4())\n", + " date_ints = [int(p) for p in resource.deposition_date.split('-')]\n", + " date_str = datetime(*date_ints)\n", + " resource.date_formatted = date_str.strftime(\"%Y-%m-%dT%H:%M:%S\")" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ - "example_url = format_file.format(new_morphology[0].archive.lower(), new_morphology[0].name)" + "new_morphologies = neuromorpho.map(nmo_resources, 'NeuronMorphology')" ] }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ - "clean_name = ''.join(example_url.split('.')[:-2])+'.swc'" + "format_file = neuromorpho.service.files_download['endpoint'] + \"/{}/CNG version/{}.CNG.swc\"" ] }, { - "cell_type": "code", - "execution_count": 43, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'http://neuromorphoorg/dableFiles/turner/CNG version/n419.swc'" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "clean_name" + "## Attach file as distribution with morphologies" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ - "file_path = f\"./downloaded/{clean_name.split('/')[-1]}\"" + "import morphio" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 40, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "'./downloaded/n419.swc'" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n", + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n", + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n" + ] } ], "source": [ - "file_path" + "for morphology in new_morphologies:\n", + " url = format_file.format(morphology.archive.lower(), morphology.name)\n", + " base_name = ''.join(url.split('.')[:-2])\n", + " file_path = f\"./downloaded/{base_name.split('/')[-1]}\"\n", + " swc = file_path + '.swc'\n", + " neuromorpho.download(url, swc)\n", + " neuromorpho.attach_file(morphology, swc, content_type='application/swc')\n", + " # Generate other morphology files\n", + " m = morphio.mut.Morphology(swc)\n", + " for extension in ['h5', 'asc']:\n", + " path = f\"{file_path}.{extension}\"\n", + " m.write(path)\n", + " neuromorpho.attach_file(morphology, path, content_type=f'application/{extension}')\n", + " m.write(file_path+'.asc')" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 44, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", - " InsecureRequestWarning,\n" + " _register_one\n", + " False\n", + " RegistrationError: 400 Client Error: Bad Request for url: https://bbp.epfl.ch/nexus/v1/resources/bbp/atlas/datashapes%3Aneuronmorphology\n" ] } ], "source": [ - "neuromorpho._download_one(example_url, file_path)" + "forge.register(new_morphologies[0], schema_id=\"datashapes:neuronmorphology\")" ] }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ - "neuromorpho.attach_file(new_morphology[0], file_path, content_type='application/swc')" + "# forge.validate(new_morphologies[0], type_=\"NeuronMorphology\")" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 45, "metadata": {}, "outputs": [ { @@ -1029,26 +976,23 @@ "output_type": "stream", "text": [ "{\n", - " id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/f947f690-a5ef-4ea4-aecf-89eb1145f400\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/0b869f02-85b3-4c49-9cb3-5d6f12bbde28\n", " type:\n", " [\n", " Dataset\n", " NeuronMorphology\n", " ]\n", - " archive: Turner\n", + " archive: Scanziani\n", " brainLocation:\n", " {\n", " type: BrainLocation\n", - " brainRegion:\n", - " {\n", - " label: hippocampus\n", - " }\n", " }\n", " contribution:\n", " {\n", " type: Contribution\n", " agent:\n", " {\n", + " id: https://ror.org/02jqj7156\n", " type: Organization\n", " label: George Mason University\n", " }\n", @@ -1056,10 +1000,93 @@ " dateCreated:\n", " {\n", " type: xsd:dateTime\n", - " value: 2005-12-31\n", + " @value: 2012-07-01T00:00:00\n", " }\n", - " description: Morphology of n419 obtained from NeuroMorpho API.\n", - " distribution: LazyAction(operation=Store.upload, args=['./downloaded/n419.swc', 'application/swc'])\n", + " description: Morphology of TypeA-10 obtained from NeuroMorpho API.\n", + " distribution:\n", + " [\n", + " {\n", + " type: DataDownload\n", + " atLocation:\n", + " {\n", + " type: Location\n", + " location: file:///gpfs/bbp.cscs.ch/data/project/proj39/nexus/bbp/atlas/0/4/8/5/1/9/5/c/TypeA-10.swc\n", + " store:\n", + " {\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/2f120131-0bbc-4a7e-b84b-52f7f00eec9e\n", + " type: RemoteDiskStorage\n", + " _rev: 1\n", + " }\n", + " }\n", + " contentSize:\n", + " {\n", + " unitCode: bytes\n", + " value: 80865\n", + " }\n", + " contentUrl: https://bbp.epfl.ch/nexus/v1/files/bbp/atlas/85d5c8d9-788d-4777-b62f-825b2ff12a51\n", + " digest:\n", + " {\n", + " algorithm: SHA-256\n", + " value: d0e898900e129fe41173eb0fe36bd84bd87a2100acb303e0ee96bf584eccfae0\n", + " }\n", + " encodingFormat: application/swc\n", + " name: TypeA-10.swc\n", + " }\n", + " {\n", + " type: DataDownload\n", + " atLocation:\n", + " {\n", + " type: Location\n", + " location: file:///gpfs/bbp.cscs.ch/data/project/proj39/nexus/bbp/atlas/c/f/7/d/4/6/8/7/TypeA-10.h5\n", + " store:\n", + " {\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/2f120131-0bbc-4a7e-b84b-52f7f00eec9e\n", + " type: RemoteDiskStorage\n", + " _rev: 1\n", + " }\n", + " }\n", + " contentSize:\n", + " {\n", + " unitCode: bytes\n", + " value: 45952\n", + " }\n", + " contentUrl: https://bbp.epfl.ch/nexus/v1/files/bbp/atlas/03edd654-0c39-4069-9b24-c7e3d68cd3ad\n", + " digest:\n", + " {\n", + " algorithm: SHA-256\n", + " value: 6cd5a3786e5502ce9443616089924ca8d22718a765d8e8202a5619f34b4afc9e\n", + " }\n", + " encodingFormat: application/h5\n", + " name: TypeA-10.h5\n", + " }\n", + " {\n", + " type: DataDownload\n", + " atLocation:\n", + " {\n", + " type: Location\n", + " location: file:///gpfs/bbp.cscs.ch/data/project/proj39/nexus/bbp/atlas/a/0/4/e/6/9/d/9/TypeA-10.asc\n", + " store:\n", + " {\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/2f120131-0bbc-4a7e-b84b-52f7f00eec9e\n", + " type: RemoteDiskStorage\n", + " _rev: 1\n", + " }\n", + " }\n", + " contentSize:\n", + " {\n", + " unitCode: bytes\n", + " value: 147432\n", + " }\n", + " contentUrl: https://bbp.epfl.ch/nexus/v1/files/bbp/atlas/00a51e52-b9f5-47e6-b61f-c640820d01bf\n", + " digest:\n", + " {\n", + " algorithm: SHA-256\n", + " value: a2469aceb35eb4569f9b2826ff099fb362f25bd5501ba468b269e8d1a1a3d93c\n", + " }\n", + " encodingFormat: application/asc\n", + " name: TypeA-10.asc\n", + " }\n", + " ]\n", " generation:\n", " {\n", " type: Generation\n", @@ -1071,15 +1098,14 @@ " identifier:\n", " {\n", " propertyID: NeuroMorhoID\n", - " value: 100\n", + " value: 10047\n", " }\n", " license:\n", " {\n", " id: https://neuromorpho.org\n", " type: License\n", " }\n", - " name: n419\n", - " note: When originally released, this reconstruction had been incompletely processed, and this issue was fixed in release 6.3 (February 2016). The pre-6.3 version of the processed file is available for download here.\n", + " name: TypeA-10\n", " objectOfStudy:\n", " {\n", " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", @@ -1092,18 +1118,10 @@ " type: Subject\n", " species:\n", " {\n", - " id: http://purl.obolibrary.org/obo/NCBITaxon_10116\n", + " id: http://purl.obolibrary.org/obo/NCBITaxon_10088\n", " type: Class\n", - " label:\n", - " [\n", - " Rattus norvegicus\n", - " Rattus Norvegicus\n", - " ]\n", - " subClassOf:\n", - " [\n", - " nsg:Species\n", - " obo:NCBITaxon_10114\n", - " ]\n", + " label: Mouse\n", + " subClassOf: obo:NCBITaxon_9989\n", " }\n", " }\n", " version: 8.4.7\n", @@ -1112,93 +1130,8 @@ } ], "source": [ - "print(new_morphology[0])" + "print(new_morphologies[0])" ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " _validate_one\n", - " False\n", - " ValidationError: resource has lazy actions which need to be executed before\n" - ] - } - ], - "source": [ - "forge.validate(new_morphology[0], type_=\"NeuronMorphology\")" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "# forge.register(new_morphology[0], schema_id=\"datashapes:neuronmorphology\")" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "# new_morphology[0].brainLocation.brainRegion" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "# new_morphology[0].brainLocation.brainRegion = forge.resolve('hippocampal formation', scope='ontology', strategy='BEST_MATCH')" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "can't convert, resource contains LazyActions", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_as_jsonld_one\u001b[0;34m(resource, form, store_metadata, model_context, metadata_context, context_resolver, **params)\u001b[0m\n\u001b[1;32m 223\u001b[0m data_graph, metadata_graph, encoded_resource, json_array = _as_graphs(\n\u001b[0;32m--> 224\u001b[0;31m \u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstore_metadata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 225\u001b[0m )\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_as_graphs\u001b[0;34m(resource, store_metadata, context, metadata_context)\u001b[0m\n\u001b[1;32m 316\u001b[0m )\n\u001b[0;32m--> 317\u001b[0;31m \u001b[0mconverted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_add_ld_keys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_context\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 318\u001b[0m \u001b[0mconverted\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"@context\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdocument\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"@context\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_add_ld_keys\u001b[0;34m(rsc, context, base)\u001b[0m\n\u001b[1;32m 449\u001b[0m raise ValueError(\n\u001b[0;32m--> 450\u001b[0;31m \u001b[0;34m\"can't convert, resource contains LazyActions\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 451\u001b[0m )\n", - "\u001b[0;31mValueError\u001b[0m: can't convert, resource contains LazyActions", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_6770/138230985.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_jsonld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_morphology\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mstack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_stack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/forge.py\u001b[0m in \u001b[0;36mas_jsonld\u001b[0;34m(self, data, form, store_metadata, **params)\u001b[0m\n\u001b[1;32m 837\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_store\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetadata_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 838\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresolve_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 839\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 840\u001b[0m )\n\u001b[1;32m 841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36mas_jsonld\u001b[0;34m(data, form, store_metadata, model_context, metadata_context, context_resolver, **params)\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0mmetadata_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0mcontext_resolver\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mdispatch\u001b[0;34m(data, fun_many, fun_one, *args, **params)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfun_many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mResource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfun_one\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 96\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"not a Resource nor a list of Resource\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/conversions/rdf.py\u001b[0m in \u001b[0;36m_as_jsonld_one\u001b[0;34m(resource, form, store_metadata, model_context, metadata_context, context_resolver, **params)\u001b[0m\n\u001b[1;32m 225\u001b[0m )\n\u001b[1;32m 226\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstore_metadata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetadata_graph\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValueError\u001b[0m: can't convert, resource contains LazyActions" - ] - } - ], - "source": [ - "print(json.dumps(forge.as_jsonld(new_morphology[0]), indent=4))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/kgforge/specializations/databases/webservice_database.py b/kgforge/specializations/databases/webservice_database.py index 445d010a..1a908b98 100644 --- a/kgforge/specializations/databases/webservice_database.py +++ b/kgforge/specializations/databases/webservice_database.py @@ -16,7 +16,7 @@ import asyncio from aiohttp import ClientSession from requests.exceptions import SSLError -from typing import Callable, List, Optional, Any +from typing import Callable, List, Optional, Any, Union from kgforge.core import Resource from kgforge.core.archetypes import Database, Store @@ -109,10 +109,19 @@ def attach_file(self, resource, path: str, url: str = None, _set(resource, "distribution", action) @catch - def download(self, urls: str, paths: str, overwrite: bool = False) -> None: + def download(self, urls: Union[str, List[str]], + paths: Union[str, List[str]], overwrite: bool = False) -> None: # path: DirPath. """Download files """ - pass + if isinstance(urls, list): + # Check consistancy between urls and paths + if not isinstance(paths, list): + raise TypeError("Given multiple urls, paths should also be a list.") + if len(paths) != len(urls): + raise ValueError("Missmatch between urls and paths, they should be the same ammount.") + self._download_many(urls, paths) + else: + self._download_one(urls, paths) def _download_many(self, urls: List[str], paths: List[str]) -> None: diff --git a/kgforge/specializations/stores/databases/web_service.py b/kgforge/specializations/stores/databases/web_service.py index 2a1fec57..22992468 100644 --- a/kgforge/specializations/stores/databases/web_service.py +++ b/kgforge/specializations/stores/databases/web_service.py @@ -11,31 +11,13 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . - - -from logging import exception import copy -import json -from pathlib import Path -from collections import namedtuple -from copy import deepcopy -from enum import Enum from typing import Callable, Dict, List, Optional, Union, Tuple -from urllib.error import URLError -from urllib.parse import quote_plus, urlparse -import requests from numpy import nan -from requests import HTTPError -from kgforge.core import Resource from kgforge.core.commons.context import Context -from kgforge.core.commons.exceptions import DownloadingError -from kgforge.core.conversions.rdf import ( - _from_jsonld_one, - recursive_resolve, -) -from kgforge.core.wrappings.dict import DictWrapper, wrap_dict +from kgforge.core.commons.exceptions import ConfigurationError from kgforge.specializations.databases.utils import resources_from_request from kgforge.specializations.stores.bluebrain_nexus import _error_message @@ -59,6 +41,11 @@ def __init__( self.context = service_context self.max_connection = max_connection self.files_download = params.pop('files_download', None) + self.search_endpoints = params.pop('searchendpoints', None) + if self.search_endpoints: + for endpoint in self.search_endpoints: + if not 'endpoint' in self.search_endpoints[endpoint]: + raise ConfigurationError(f"Missing endpoint searchenpoints") self.params = copy.deepcopy(params) self.headers = {"Content-Type": content_type, "Accept": accept} @@ -77,5 +64,7 @@ def search(self, resolvers, *filters, **params): filter_params = filters[0] if not isinstance(filter_params, dict): raise NotImplementedError('Currently only the use of a dictionary is implemented') + searchendpoint = params.pop('searchendpoint', None) + endpoint = self.search_endpoints[searchendpoint]['endpoint'] if searchendpoint else self.endpoint query_params = {**filter_params, **params} - return resources_from_request(self.endpoint, self.headers, **query_params) \ No newline at end of file + return resources_from_request(endpoint, self.headers, **query_params) \ No newline at end of file diff --git a/tests/specializations/databases/test_database.py b/tests/specializations/databases/test_database.py index e0c36207..5bef506a 100644 --- a/tests/specializations/databases/test_database.py +++ b/tests/specializations/databases/test_database.py @@ -13,11 +13,13 @@ # along with Blue Brain Nexus Forge. If not, see . # Placeholder for the test suite for actions. +import copy import pytest from kgforge.core import Resource, KnowledgeGraphForge from kgforge.core.commons.context import Context from kgforge.core.wrappings.paths import Filter +from kgforge.specializations.databases import WebServiceDatabase from kgforge.specializations.databases.store_database import StoreDatabase from kgforge.specializations.databases.utils import type_from_filters from kgforge.core.commons.exceptions import ConfigurationError @@ -79,6 +81,18 @@ def context(dict_context): return Context(dict_context) +def forge(db_config, model_config): + config = { + "Model": model_config, + "Store": { + "name": "DemoStore", + }, + "Databases": { + 'dbpedia': db_config + } + } + return KnowledgeGraphForge(config) + @pytest.fixture def model_config(): return { @@ -88,7 +102,7 @@ def model_config(): } @pytest.fixture -def db_config(): +def storedb_config(): return { "origin": "store", "source": "SPARQLStore", @@ -96,7 +110,7 @@ def db_config(): "sparql":{ "endpoint": "http://dbpedia.org/sparql" }, - + }, "model": { "name": "DemoModel", @@ -107,44 +121,95 @@ def db_config(): } } } +class TestStoreDB: + + @pytest.fixture + def forge_with_store(storedb_config, model_config): + return forge(storedb_config, model_config) + + def test_database_config(forge_with_store, storedb_config, model_config): + with pytest.raises(ValueError): + StoreDatabase(storedb_config) # Missing forge + with pytest.raises(ValueError): + StoreDatabase(forge_with_store, **storedb_config) # Missing name + with pytest.raises(ValueError): + # Missing model + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/'}) + with pytest.raises(ConfigurationError): + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + def test_database_directory(forge_with_store, model_config): + with pytest.raises(Exception): + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'directory', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + def test_database_wrong_web_service(forge_with_store, model_config): + with pytest.raises(Exception): + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', + 'model': model_config}) + @pytest.fixture -def forge(db_config, model_config): - config = { - "Model": model_config, - "Store": { - "name": "DemoStore", - }, - "Databases": { - 'dbpedia': db_config - } +def ws_db_config(): + return { + "name": "uniprot", + "origin": "web_service", + "source": "https://rest.uniprot.org/uniprotkb/", + "model": { + "name": "DemoModel", + "origin": "directory", + "source": "tests/data/demo-model/", + "context":{ + "iri": "examples/database-sources/UniProt" + } + }, + "max_connection": 10, + "content_type": "application/json;charset=UTF-8", + "accept": "*/*" } - return KnowledgeGraphForge(config) - - -def test_database_config(forge, db_config, model_config): - with pytest.raises(ValueError): - StoreDatabase(db_config) # Missing forge - with pytest.raises(ValueError): - StoreDatabase(forge, **db_config) # Missing name - with pytest.raises(ValueError): - # Missing model - StoreDatabase(forge, **{'name': 'mydb', 'origin': 'store', - 'source':'tests/data/demo-store/'}) - with pytest.raises(ConfigurationError): - StoreDatabase(forge, **{'name': 'mydb', 'origin': 'store', - 'source':'tests/data/demo-store/', - 'model': model_config}) - -def test_database_directory(forge, model_config): - with pytest.raises(Exception): - StoreDatabase(forge, **{'name': 'mydb', 'origin': 'directory', - 'source':'tests/data/demo-store/', - 'model': model_config}) - -def test_database_web_serice(forge, model_config): - with pytest.raises(Exception): - StoreDatabase(forge, **{'name': 'mydb', 'origin': 'web_service', - 'source':'tests/data/demo-store/', - 'model': model_config}) +class TestWebServiceDB: + @pytest.fixture + def forge_with_wsdb(ws_db_config, model_config): + return forge(ws_db_config, model_config) + + def test_database_config(forge_with_wsdb, ws_db_config, model_config): + with pytest.raises(ValueError): + WebServiceDatabase(ws_db_config) # Missing forge + with pytest.raises(ValueError): + # Missing model + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', 'max_connection': 5, + 'content_type': 'application/json', 'accept': '*/*'}) + with pytest.raises(ValueError): + # Missing max_connection + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', + 'content_type': 'application/json', 'accept': '*/*'}) + with pytest.raises(ValueError): + # Missing content_type + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', 'max_connection': 5, + 'content_type': 'application/json'}) + + def test_database_directory(forge_with_wsdb, model_config): + with pytest.raises(Exception): + WebServiceDatabase(forge_with_wsdb, **{'name':'mydb', 'origin': 'directory', + 'source':'tests/data/demo-store/', + 'model': model_config}) + def test_database_wrong_store(forge_with_wsdb, model_config): + with pytest.raises(Exception): + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + def test_searchendpoint_config(forge_with_wsdb, ws_db_config): + tmp_config = copy.deepcopy(ws_db_config) + tmp_config['searchendpoints'] = {'some_endpoint': {'wrong': 'True'}} + with pytest.raises(ConfigurationError): + WebServiceDatabase(forge_with_wsdb, **tmp_config) \ No newline at end of file diff --git a/tests/specializations/stores/test_sparql.py b/tests/specializations/stores/test_sparql.py index 7b13a035..9b9a2eb3 100644 --- a/tests/specializations/stores/test_sparql.py +++ b/tests/specializations/stores/test_sparql.py @@ -88,57 +88,4 @@ def test_config(sparql_store, store_context): def test_search_params(sparql_store): with pytest.raises(ValueError): - sparql_store.search(filters=[None]) - - -class TestQuery: - - @pytest.mark.parametrize( - "filters,expected", - [ - pytest.param( - ([Filter(['type'], '__eq__','MusicalArtist')]), - ([Resource.from_json({'id': "http://dbpedia.org/resource/1969_NCAA_University_Division_Men's_Cross_Country_Championships"}), - Resource.from_json({'id': "http://dbpedia.org/resource/1970_NCAA_University_Division_Men's_Cross_Country_Championships"}), - Resource.from_json({'id': "http://dbpedia.org/resource/1971_NCAA_University_Division_Men's_Cross_Country_Championships"})]), - id="search_names", - ) - ] - ) - def test_sparql_search(self, sparql_store, filters, expected): - resources = sparql_store.search(None, *filters, limit=3, debug=True) - assert resources == expected - - @pytest.mark.parametrize( - "query,expected", - [ - pytest.param( - ("""SELECT ?name WHERE { ?person a dbo:MusicalArtist . - ?person foaf:name ?name . - FILTER (LANG(?name) = 'en') . - }"""), - ([Resource.from_json({'name': '20 Dead Flower Children'}), - Resource.from_json({'name': '21 Savage'}), - Resource.from_json({'name': '220 Kid'})]), - id="query_names", - ), - pytest.param( - ("""PREFIX : - SELECT ?name ?birth WHERE { - ?person a dbo:MusicalArtist . - ?person dbo:birthPlace dbr:Berlin . - ?person dbo:birthDate ?birth . - ?person foaf:name ?name . - ?person rdfs:comment ?description . - FILTER (LANG(?description) = 'en') . - }"""), - ([Resource.from_json({'name': 'Monika Kruse', 'birth': "1971-07-23"}), - Resource.from_json({'name': 'Walter Gronostay', 'birth': "1906-07-29"}), - Resource.from_json({'name': 'Wilhelm Guttmann', 'birth': "1886-01-01"})]), - id="query_berlin_birthdates", - ), - ] - ) - def test_sparql_query(self, sparql_store, query, expected): - resources = sparql_store.sparql(query, limit=3, debug=True) - assert resources == expected \ No newline at end of file + sparql_store.search(filters=[None]) \ No newline at end of file From dd7d22d478f88261b7fd18cebc2013d052f95b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 15 Nov 2022 11:18:32 +0100 Subject: [PATCH 26/30] Update tests for webservice databases. --- kgforge/specializations/databases/utils.py | 13 ++++++++++++- .../stores/databases/web_service.py | 17 +++++++++++++---- .../specializations/databases/test_database.py | 13 +++++++++++-- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/kgforge/specializations/databases/utils.py b/kgforge/specializations/databases/utils.py index 04a04158..43f86d70 100644 --- a/kgforge/specializations/databases/utils.py +++ b/kgforge/specializations/databases/utils.py @@ -41,6 +41,7 @@ def type_from_filters(*filters) -> Optional[str]: return resource_type def resources_from_results(results): + """Returns Resources from standard response bindings.""" return [ Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') @@ -53,7 +54,15 @@ def resources_from_results(results): ] def resources_from_request(url, headers, **params): - """Perform a HTTP request""" + """Perform a HTTP request + params: + ------- + response_loc : list[str] + The nested location of the relevat metadata in the + response. + Example: NeuroMorpho uses response["_embedded"]["neuronResources"] + which should be given as: response_loc = ["_embedded", "neuronResources"] + """ response_location = params.pop('response_loc', None) try: response = requests.get( @@ -65,6 +74,7 @@ def resources_from_request(url, headers, **params): else: data = response.json() if response_location: + # Get the resources directly from a location in the response if isinstance(response_location, str): results = data[response_location] elif isinstance(response_location, (list, tuple)): @@ -73,5 +83,6 @@ def resources_from_request(url, headers, **params): results = data return [Resource(**result) for result in results] else: + # Standard response format results = data["results"]["bindings"] return resources_from_results(results) \ No newline at end of file diff --git a/kgforge/specializations/stores/databases/web_service.py b/kgforge/specializations/stores/databases/web_service.py index 22992468..d9a3ef32 100644 --- a/kgforge/specializations/stores/databases/web_service.py +++ b/kgforge/specializations/stores/databases/web_service.py @@ -12,9 +12,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . import copy -from typing import Callable, Dict, List, Optional, Union, Tuple - -from numpy import nan +from typing import Dict, List, Optional, Union, Tuple from kgforge.core.commons.context import Context from kgforge.core.commons.exceptions import ConfigurationError @@ -65,6 +63,17 @@ def search(self, resolvers, *filters, **params): if not isinstance(filter_params, dict): raise NotImplementedError('Currently only the use of a dictionary is implemented') searchendpoint = params.pop('searchendpoint', None) - endpoint = self.search_endpoints[searchendpoint]['endpoint'] if searchendpoint else self.endpoint + if searchendpoint: + if self.search_endpoints is None: + raise ConfigurationError(f"No searchendpoints were given " \ + "in the initial configuration.") + try: + endpoint = self.search_endpoints[searchendpoint]['endpoint'] + except KeyError: + raise ConfigurationError(f"The {searchendpoint} searchpoint was not given "\ + "in the initial configuration.") + + else: + endpoint = self.endpoint query_params = {**filter_params, **params} return resources_from_request(endpoint, self.headers, **query_params) \ No newline at end of file diff --git a/tests/specializations/databases/test_database.py b/tests/specializations/databases/test_database.py index 5bef506a..480454bc 100644 --- a/tests/specializations/databases/test_database.py +++ b/tests/specializations/databases/test_database.py @@ -22,7 +22,7 @@ from kgforge.specializations.databases import WebServiceDatabase from kgforge.specializations.databases.store_database import StoreDatabase from kgforge.specializations.databases.utils import type_from_filters -from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.core.commons.exceptions import ConfigurationError, QueryingError from kgforge.core.wrappings.dict import DictWrapper, wrap_dict import json @@ -212,4 +212,13 @@ def test_searchendpoint_config(forge_with_wsdb, ws_db_config): tmp_config = copy.deepcopy(ws_db_config) tmp_config['searchendpoints'] = {'some_endpoint': {'wrong': 'True'}} with pytest.raises(ConfigurationError): - WebServiceDatabase(forge_with_wsdb, **tmp_config) \ No newline at end of file + WebServiceDatabase(forge_with_wsdb, **tmp_config) + + def test_search(forge_with_wsdb, ws_db_config): + tmp_config = copy.deepcopy(ws_db_config) + tmp_config['searchendpoints'] = {'some_endpoint': {'endpoint': "https://my_endpoint.com/API/"}} + db = WebServiceDatabase(forge_with_wsdb, **tmp_config) + with pytest.raises(ConfigurationError): + db.search([], {'type': 'Protein'}, searchendpoint='uniprot') + with pytest.raises(QueryingError): + db.search([], {'type': 'Protein'}) \ No newline at end of file From 5fd39bc4c7f50ab8a7d51437de08fe35af2125a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Mon, 18 Sep 2023 12:15:26 +0200 Subject: [PATCH 27/30] Resolve conflicts with old methods --- kgforge/core/commons/dictionaries.py | 7 +++++++ kgforge/specializations/stores/sparql.py | 14 +++++++------- tests/specializations/stores/test_sparql.py | 8 ++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/kgforge/core/commons/dictionaries.py b/kgforge/core/commons/dictionaries.py index c5667df6..0ddf3205 100644 --- a/kgforge/core/commons/dictionaries.py +++ b/kgforge/core/commons/dictionaries.py @@ -23,8 +23,15 @@ def with_defaults(original: Dict, other: Dict, original_key: str, other_key: str if original[original_key] == other[other_key]: use_values(original, other, keys) + def use_values(original: Dict, other: Dict, keys: List[str]) -> None: for x in keys: if x not in original and x in other: original[x] = other[x] + + +def update_dict(original: Dict, other: Dict) -> Dict: + original_copy = copy.deepcopy(original) + original_copy.update(other) + return original_copy \ No newline at end of file diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py index f3347739..58b7a3d4 100644 --- a/kgforge/specializations/stores/sparql.py +++ b/kgforge/specializations/stores/sparql.py @@ -31,11 +31,11 @@ from kgforge.core.commons.exceptions import QueryingError from kgforge.core.commons.execution import not_supported from kgforge.core.commons.parser import _parse_type +from kgforge.core.commons.sparql_query_builder import SPARQLQueryBuilder from kgforge.specializations.stores.bluebrain_nexus import ( - CategoryDataType, _create_select_query, - _box_value_as_full_iri, sparql_operator_map, - build_sparql_query_statements, - type_map, format_type) + CategoryDataType, + _create_select_query +) class SPARQLStore(Store): @@ -163,9 +163,9 @@ def search( "Field inclusion and exclusion are not supported when using SPARQL" ) - query_statements, query_filters = build_sparql_query_statements( - self.context, filters - ) + query_statements, query_filters = SPARQLQueryBuilder.build( + None, resolvers, self.model_context, *filters + ) statements = ";\n ".join(query_statements) _filters = ".\n".join(query_filters) + '\n' _vars = ["?id"] diff --git a/tests/specializations/stores/test_sparql.py b/tests/specializations/stores/test_sparql.py index 9b9a2eb3..9d393dba 100644 --- a/tests/specializations/stores/test_sparql.py +++ b/tests/specializations/stores/test_sparql.py @@ -14,13 +14,9 @@ import pytest -from kgforge.core import Resource -from kgforge.core.wrappings.paths import Filter from kgforge.core.commons.context import Context -from kgforge.specializations.stores.sparql import ( - SPARQLStore, - build_sparql_query_statements, -) +from kgforge.specializations.stores.sparql import SPARQLStore + SEARCH_ENDPOINT = {"sparql": {"endpoint": "http://dbpedia.org/sparql"}} From b26fefcb2f3daa8c45026359e064508e7e205f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 19 Oct 2023 18:34:28 +0200 Subject: [PATCH 28/30] Add mapper and mapping methods to SPARQL store --- kgforge/specializations/stores/sparql.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py index 58b7a3d4..d0ea2ac8 100644 --- a/kgforge/specializations/stores/sparql.py +++ b/kgforge/specializations/stores/sparql.py @@ -13,11 +13,8 @@ # along with Blue Brain Nexus Forge. If not, see . import json -from copy import deepcopy from pathlib import Path -from typing import Dict, List, Tuple, Optional, Union, Any -from uuid import uuid4 -from rdflib import Graph +from typing import Dict, List, Callable, Optional, Union, Any from rdflib.plugins.sparql.parser import Query from SPARQLWrapper import SPARQLWrapper, JSON @@ -30,7 +27,8 @@ from kgforge.core.wrappings.paths import create_filters_from_dict from kgforge.core.commons.exceptions import QueryingError from kgforge.core.commons.execution import not_supported -from kgforge.core.commons.parser import _parse_type +from kgforge.specializations.mappers import DictionaryMapper +from kgforge.specializations.mappings import DictionaryMapping from kgforge.core.commons.sparql_query_builder import SPARQLQueryBuilder from kgforge.specializations.stores.bluebrain_nexus import ( CategoryDataType, @@ -51,6 +49,13 @@ def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, # [C]RUD + @property + def mapping(self) -> Optional[Callable]: + return DictionaryMapping + + @property + def mapper(self) -> Optional[DictionaryMapper]: + return DictionaryMapper def register(self, data: Union[Resource, List[Resource]], schema_id: str = None ) -> None: From b443b1cffd30f285993c5715119a1abc7d051b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 26 Oct 2023 11:04:56 +0200 Subject: [PATCH 29/30] Small mixes --- kgforge/core/archetypes/database.py | 6 ++---- kgforge/core/archetypes/store.py | 3 ++- kgforge/core/forge.py | 8 +++----- kgforge/specializations/databases/webservice_database.py | 3 ++- setup.py | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py index 59e9c557..5f22f419 100644 --- a/kgforge/core/archetypes/database.py +++ b/kgforge/core/archetypes/database.py @@ -13,12 +13,10 @@ # along with Blue Brain Nexus Forge. If not, see . from abc import ABC, abstractmethod -import json from pathlib import Path -from typing import Any, Optional, Callable, Dict, List, Union +from typing import Any, Optional, Callable, Type, List, Union from kgforge.core import Resource -from kgforge.core.commons.context import Context from kgforge.core.commons.execution import not_supported from kgforge.core.archetypes import Mapping, Model from kgforge.core.commons.attributes import repr_class @@ -79,7 +77,7 @@ def map(self, resources : Union[List[Union[Resource, str]], Union[Resource, str] mapped_resources.append(self._forge.map(resource_dict, type_)) elif type_ in mappings: # type_ is the entity here - mapping_class : Mapping = import_class(mappings[type_][0], "mappings") + mapping_class : Type[Mapping] = import_class(mappings[type_][0], "mappings") mapping = self._model.mapping(type_, self._model.source, mapping_class) mapped_resources.append(self._forge.map(resource_dict, mapping)) else: diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index a5f7dc62..62cebe39 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -602,7 +602,8 @@ def replace(match: Match) -> str: return f"{pfx}\n{qr}" -def resources_from_construct_query(data, context): +def resources_from_construct_query(data, context) -> List[Resource]: + """Build resources from a CONSTRUCT query.""" subject_triples = {} for r in data["results"]["bindings"]: subject = r["subject"]["value"] diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 9eb0b6c2..b632c78c 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -1014,11 +1014,9 @@ def get_model_context(self): def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], store_config : Optional[Dict[str, Dict[str, str]]], model_context: Context - ) -> Union[Database, List[Database]]: - names = all_config.keys() + ) -> Dict[str, Database]: dbs = {} - for name in names: - config = all_config[name] + for name, config in all_config.items(): origin = config['origin'] source = config['source'] if origin == 'store': @@ -1027,7 +1025,7 @@ def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], store_copy = deepcopy(store_config) with_defaults(config, store_copy, "source", "name", - store_copy.keys()) + list(store_copy.keys())) config['model_context'] = model_context config['name'] = name dbs[name] = StoreDatabase(self, **config) diff --git a/kgforge/specializations/databases/webservice_database.py b/kgforge/specializations/databases/webservice_database.py index 1a908b98..d1e73570 100644 --- a/kgforge/specializations/databases/webservice_database.py +++ b/kgforge/specializations/databases/webservice_database.py @@ -118,7 +118,8 @@ def download(self, urls: Union[str, List[str]], if not isinstance(paths, list): raise TypeError("Given multiple urls, paths should also be a list.") if len(paths) != len(urls): - raise ValueError("Missmatch between urls and paths, they should be the same ammount.") + raise ValueError(f"Missmatch between urls ({len(urls)}) and paths ({len(paths)}), \ + they should be the same amount.") self._download_many(urls, paths) else: self._download_one(urls, paths) diff --git a/setup.py b/setup.py index 343e2bd4..7ff5a8f0 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ "pyparsing>=2.0.2", "owlrl>=5.2.3", "elasticsearch_dsl==7.4.0", - "SPARQLWrapper" + "SPARQLWrapper==2.0.0" ], extras_require={ "dev": [ From 3c6f952c8b42cf1001a6ae09bad47618dc870cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 26 Oct 2023 11:15:18 +0200 Subject: [PATCH 30/30] Add self to tests in classes --- .../databases/test_database.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/specializations/databases/test_database.py b/tests/specializations/databases/test_database.py index 480454bc..1524ee48 100644 --- a/tests/specializations/databases/test_database.py +++ b/tests/specializations/databases/test_database.py @@ -124,10 +124,10 @@ def storedb_config(): class TestStoreDB: @pytest.fixture - def forge_with_store(storedb_config, model_config): + def forge_with_store(self, storedb_config, model_config): return forge(storedb_config, model_config) - def test_database_config(forge_with_store, storedb_config, model_config): + def test_database_config(self, forge_with_store, storedb_config, model_config): with pytest.raises(ValueError): StoreDatabase(storedb_config) # Missing forge with pytest.raises(ValueError): @@ -141,13 +141,13 @@ def test_database_config(forge_with_store, storedb_config, model_config): 'source':'tests/data/demo-store/', 'model': model_config}) - def test_database_directory(forge_with_store, model_config): + def test_database_directory(self, forge_with_store, model_config): with pytest.raises(Exception): StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'directory', 'source':'tests/data/demo-store/', 'model': model_config}) - def test_database_wrong_web_service(forge_with_store, model_config): + def test_database_wrong_web_service(self, forge_with_store, model_config): with pytest.raises(Exception): StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'web_service', 'source':'tests/data/demo-store/', @@ -175,10 +175,10 @@ def ws_db_config(): class TestWebServiceDB: @pytest.fixture - def forge_with_wsdb(ws_db_config, model_config): + def forge_with_wsdb(self, ws_db_config, model_config): return forge(ws_db_config, model_config) - def test_database_config(forge_with_wsdb, ws_db_config, model_config): + def test_database_config(self, forge_with_wsdb, ws_db_config, model_config): with pytest.raises(ValueError): WebServiceDatabase(ws_db_config) # Missing forge with pytest.raises(ValueError): @@ -197,24 +197,24 @@ def test_database_config(forge_with_wsdb, ws_db_config, model_config): 'source':'tests/data/demo-store/', 'max_connection': 5, 'content_type': 'application/json'}) - def test_database_directory(forge_with_wsdb, model_config): + def test_database_directory(self, forge_with_wsdb, model_config): with pytest.raises(Exception): WebServiceDatabase(forge_with_wsdb, **{'name':'mydb', 'origin': 'directory', 'source':'tests/data/demo-store/', 'model': model_config}) - def test_database_wrong_store(forge_with_wsdb, model_config): + def test_database_wrong_store(self, forge_with_wsdb, model_config): with pytest.raises(Exception): WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'store', 'source':'tests/data/demo-store/', 'model': model_config}) - def test_searchendpoint_config(forge_with_wsdb, ws_db_config): + def test_searchendpoint_config(self, forge_with_wsdb, ws_db_config): tmp_config = copy.deepcopy(ws_db_config) tmp_config['searchendpoints'] = {'some_endpoint': {'wrong': 'True'}} with pytest.raises(ConfigurationError): WebServiceDatabase(forge_with_wsdb, **tmp_config) - def test_search(forge_with_wsdb, ws_db_config): + def test_search(self, forge_with_wsdb, ws_db_config): tmp_config = copy.deepcopy(ws_db_config) tmp_config['searchendpoints'] = {'some_endpoint': {'endpoint': "https://my_endpoint.com/API/"}} db = WebServiceDatabase(forge_with_wsdb, **tmp_config)