Allow skipping specific validator (#153)

Makefile

@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.37.0"
+version="0.38.0"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.37.0"
+__version__ = "0.38.0"

cognite/neat/rules/exporter/rules2pydantic_models.py

@@ -43,7 +43,7 @@ def default_model_configuration():
 
 
 def default_model_methods():
-    return [from_graph, to_asset, to_relationship, to_node, to_edge, to_graph]
+    return [from_graph, to_asset, to_relationship, to_node, to_edge, to_graph, get_field_description, get_field_name]
 
 
 def default_model_property_attributes():
@@ -130,14 +130,16 @@ def _properties_to_pydantic_fields(
 
     fields = {"external_id": (str, Field(..., alias="external_id"))}
 
-    for name, property_ in properties.items():
+    for property_id, property_ in properties.items():
         field_type = _define_field_type(property_)
-
         field_definition: dict = {
-            "alias": name,
+            "alias": property_.property_id,
+            "description": property_.description if property_.description else None,
             # keys below will be available under json_schema_extra
             "property_type": field_type.__name__ if field_type in [EdgeOneToOne, EdgeOneToMany] else "NodeAttribute",
             "property_value_type": property_.expected_value_type,
+            "property_name": property_.property_name,
+            "property_id": property_.property_id,
         }
 
         if field_type.__name__ in [EdgeOneToMany.__name__, list.__name__]:
@@ -151,7 +153,7 @@ def _properties_to_pydantic_fields(
 
         # making sure that field names are python compliant
         # their original names are stored as aliases
-        fields[re.sub(r"[^_a-zA-Z0-9/_]", "_", name)] = (
+        fields[re.sub(r"[^_a-zA-Z0-9/_]", "_", property_id)] = (
             field_type,
             Field(**field_definition),  # type: ignore[pydantic-field]
         )
@@ -543,6 +545,24 @@ def to_graph(self, transformation_rules: Rules, graph: Graph):
     ...
 
 
+@classmethod  # type: ignore
+def get_field_description(cls, field_id: str) -> str | None:
+    """Returns description of the field if one exists"""
+    if field_id in cls.model_fields:
+        return cls.model_fields[field_id].description
+    else:
+        return None
+
+
+@classmethod  # type: ignore
+def get_field_name(cls, field_id: str) -> str | None:
+    """Returns name of the field if one exists"""
+    if field_id in cls.model_fields and cls.model_fields[field_id].json_schema_extra:
+        if "property_name" in cls.model_fields[field_id].json_schema_extra:
+            return cls.model_fields[field_id].json_schema_extra["property_name"]
+    return None
+
+
 def add_class_prefix_to_xid(class_name: str, external_id: str) -> str:
     """Adds class name as prefix to the external_id"""
     return f"{class_name}_{external_id}"

cognite/neat/rules/importer/_base.py

@@ -30,26 +30,28 @@ def to_raw_rules(self) -> RawRules:
         return RawRules.from_tables(tables=tables, importer_type=self.__class__.__name__)
 
     def to_rules(
-        self, return_report: bool = False, skip_validation: bool = False
+        self, return_report: bool = False, skip_validation: bool = False, validators_to_skip: list[str] | None = None
     ) -> tuple[Rules | None, list[ErrorDetails] | None, list | None] | Rules:
         """
         Creates `Rules` object from the data.
 
         Args:
             return_report: To return validation report. Defaults to False.
             skip_validation: Bypasses Rules validation. Defaults to False.
+            validators_to_skip: List of validators to skip. Defaults to None.
 
         Returns:
-            Instance of `Rules`, which can be validated or not validated based on
-            `skip_validation` flag, and optional list of errors and warnings if
+            Instance of `Rules`, which can be validated, not validated based on
+            `skip_validation` flag, or partially validated if `validators_to_skip` is set,
+            and optional list of errors and warnings if
             `return_report` is set to True.
 
         !!! Note
             `skip_validation` flag should be only used for purpose when `Rules` object
             is exported to an Excel file. Do not use this flag for any other purpose!
         """
         raw_rules = self.to_raw_rules()
-        return raw_rules.to_rules(return_report, skip_validation)
+        return raw_rules.to_rules(return_report, skip_validation, validators_to_skip)
 
     def _default_metadata(self):
         return {

cognite/neat/rules/importer/dms2rules.py

@@ -79,14 +79,14 @@ def to_tables(self) -> dict[str, pd.DataFrame]:
         classes: list[dict[str, str | float]] = []
         properties: list[dict[str, str | float]] = []
         for view in self.views:
-            class_name = view.external_id
+            class_id = view.external_id
             classes.append(
                 {
-                    "Class": class_name,
+                    "Class": class_id,
                     "Description": view.description or float("nan"),
                 }
             )
-            for prop_name, prop in view.properties.items():
+            for prop_id, prop in view.properties.items():
                 if isinstance(prop, MappedProperty):
                     # Edge 1-1
                     if isinstance(prop.type, DirectRelation):
@@ -110,14 +110,15 @@ def to_tables(self) -> dict[str, pd.DataFrame]:
 
                 properties.append(
                     {
-                        "Class": class_name,
-                        "Property": prop_name,
+                        "Class": class_id,
+                        "Property": prop_id,
+                        "Name": prop.name if prop.name else prop_id,
                         "Description": prop.description or float("nan"),
                         "Type": type_,
                         "Min Count": "1",
                         "Max Count": max_count,
                         "Rule Type": "rdfpath",
-                        "Rule": f"cim:{class_name}(cim:{prop_name}.name)",
+                        "Rule": f"cim:{class_id}(cim:{prop_id}.name)",
                     }
                 )
 

cognite/neat/rules/models/raw_rules.py

@@ -151,12 +151,14 @@ def to_rules(
         self,
         return_report: bool = False,
         skip_validation: bool = False,
+        validators_to_skip: list[str] | None = None,
     ) -> tuple[Rules | None, list[ErrorDetails] | None, list | None] | Rules:
         """Validates RawRules instances and returns Rules instance.
 
         Args:
             return_report: To return validation report. Defaults to False.
             skip_validation: Bypasses Rules validation. Defaults to False.
+            validators_to_skip: List of validators to skip. Defaults to None.
 
         Returns:
             Instance of `Rules`, which can be validated or not validated based on
@@ -171,6 +173,8 @@ def to_rules(
         rules_dict = _raw_tables_to_rules_dict(self)
         if skip_validation:
             return _to_invalidated_rules(rules_dict)
+        elif validators_to_skip:
+            return _to_partially_validated_rules(rules_dict, validators_to_skip)
         else:
             return _to_validated_rules(rules_dict, return_report)
 
@@ -214,6 +218,10 @@ def _to_validated_rules(
             raise e
 
 
+def _to_partially_validated_rules(rules_dict: dict, validators_to_skip: list[str]) -> Rules:
+    return Rules(validators_to_skip=validators_to_skip, **rules_dict)
+
+
 def _to_invalidated_rules(rules_dict: dict) -> Rules:
     args = {
         "metadata": Metadata.model_construct(**rules_dict["metadata"]),

cognite/neat/rules/models/rules.py

@@ -4,14 +4,16 @@
 
 from __future__ import annotations
 
+import inspect
 import math
 import re
 import sys
 import warnings
 from collections.abc import ItemsView, Iterator, KeysView, ValuesView
 from datetime import datetime
 from pathlib import Path
-from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
+from types import FrameType
+from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
 
 import pandas as pd
 from pydantic import (
@@ -850,14 +852,18 @@ class Rules(RuleModel):
                     to transform data from source to target representation
         prefixes: Prefixes used in the data model. Defaults to PREFIXES
         instances: Instances defined in the data model. Defaults to None
+        validators_to_skip: List of validators to skip. Defaults to []
 
     !!! note "Importers"
         Neat supports importing data from different sources. See the importers section for more details.
 
     !!! note "Exporters"
         Neat supports exporting data to different sources. See the exporters section for more details.
+
+    !!! note "validators_to_skip" use this only if you are sure what you are doing
     """
 
+    validators_to_skip: list[str] = Field(default_factory=list, exclude=True)
     metadata: Metadata
     classes: Classes
     properties: Properties
@@ -897,6 +903,10 @@ def dict_to_properties_obj(cls, value: dict | Properties) -> Properties:
     @model_validator(mode="after")
     def properties_refer_existing_classes(self) -> Self:
         errors = []
+
+        if cast(FrameType, inspect.currentframe()).f_code.co_name in self.validators_to_skip:
+            return self
+
         for property_ in self.properties.values():
             if property_.class_id not in self.classes:
                 errors.append(
@@ -915,7 +925,10 @@ def properties_refer_existing_classes(self) -> Self:
         return self
 
     @validator("properties")
-    def is_type_defined_as_object(cls, value):
+    def is_type_defined_as_object(cls, value, values):
+        if inspect.currentframe().f_code.co_name in values["validators_to_skip"]:
+            return value
+
         defined_objects = {property_.class_id for property_ in value.values()}
 
         if undefined_objects := [

docs/CHANGELOG.md

@@ -15,6 +15,14 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.38.0] - 31-10-23
+## Added
+ - Ability to partially validate Rules
+ - Description and name of fields added to rules generated pydantic models
+
+## Improved
+ - Improved naming of internal variables in `cognite/neat/rules/exporter/rules2pydantic_models.py`
+
 ## [0.37.0] - 31-10-23
 
 ## Added

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.37.0"
+version = "0.38.0"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [