[NEAT-228] Rename columns 𐇦 (#436)

* tests: Added failing tests * refactor: added DMS Node Entitiy * refactor: setup shell for new wrapped entities * feat: Implemented wrapped filter * refactor: Switched to wrapped entity * refactor: introduce DMSFilder base class * refactor: moved out creation of filter method * refactor: moved logic * refactor: reorg * refactor: support setting explicit NodeType and HasData Filter * build: changelog * refactor: introduce data model type * refactor; implemented smart default * tests: updated tests * style: better words * build: bug * tests: updated tests * refactor; extend column * tests: Added olav test * fix: parent filter when no properties * fix: Dump dataModelType * refactor: fix comparison * tests: update tests * refactor: moved modelType up * refactor: robust against duplicated node types * build: changelog * build; changelog * build; updated docs * docs; documented new smart defaults * refactor; DMSImporter infer data model type * tests: update * tests: updated test to return direct listable * refactor: removed warning and create correct container * refactor: reimplement the view property conversion * refactor: pre-commits * tests: update tests * refactor: update examples * tests: updated tests * refactor: fix * tests: updated tests * refactor: update ref rules * docs; docs section * build: bump + changelog * refactor: added validation of value type for relation * tests: added DMSImporter test for tutorial examples * refactor: ensure deterministic output when dumping DMSRules * refactor: moved out DMS Rules serialization * refactor: fixed minor bugs * refactor: complete reimplementation of DMS importer * refactor: implementation * fix: a few minor bugs in the DMSImporter * fix; added safety * refactor: Added support for exporting/importing ref model * refactor: fix extended * fix: typo in Olav rules * fix: bug in importer * refactor: Olav passing tests * refactor: support solo inwards edges * refactor: Robustify DMRRules read methods part 1 * refactor; validate format of string as well * refactor: Robustify DMRRules read methods part 2 * refactor; pass context from dictionary and zip file * fix: introduced bug * refactor; introduced bug * fix: Finidhed up the last issues * fix: immutable classes cannot be used for exceptions * refactor: removed properties for SheetEntity * refactor: remove default-view-version * refactor: added new warning * refactor: added warning if view is not matching data model * build: changelog * refactor: update tutorial examples * refactor: fix inconsitent warning naming * refactor: setup shell for new warnings * refactor: added warning on deploying to space with other data model * refactor: Warning if solution on top of solution * refactor; tiny optimization * refactor: improved implementation * refactor: Rename columns * refactor: rename relation to connection * refactor: Tutorial examples renaming * refactor: forgotten * refactor: update DMSRulesWrite * refactor: More renaming * tests: upgrade test data * refactor: fix bug * tests: optimized test * refactro: ref rules * tests: update tests data * build: changelog * docs; update docs * refactor: bad merge * Update cognite/neat/rules/issues/dms.py Co-authored-by: Nikola Vasiljevic <[email protected]> * Linting and static code checks --------- Co-authored-by: Nikola Vasiljevic <[email protected]> Co-authored-by: doctrino <[email protected]>
cognitedata · May 6, 2024 · e5ca195 · e5ca195
1 parent 56bc57e
commit e5ca195
Show file tree

Hide file tree

Showing 21 changed files with 151 additions and 146 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.75.9"
+version="0.76.0"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

diff --git a/cognite/neat/_version.py b/cognite/neat/_version.py
@@ -1 +1 @@
-__version__ = "0.75.9"
+__version__ = "0.76.0"
diff --git a/cognite/neat/rules/importers/_dms2rules.py b/cognite/neat/rules/importers/_dms2rules.py
@@ -300,7 +300,7 @@ def _create_dms_property(
             property_=prop_id,
             description=prop.description,
             name=prop.name,
-            relation=self._get_relation_type(prop),
+            connection=self._get_relation_type(prop),
             value_type=value_type,
             is_list=self._get_is_list(prop),
             nullable=self._get_nullable(prop),

diff --git a/cognite/neat/rules/importers/_spreadsheet2rules.py b/cognite/neat/rules/importers/_spreadsheet2rules.py
@@ -22,7 +22,15 @@
 from ._base import BaseImporter, Rules, _handle_issues
 
 SOURCE_SHEET__TARGET_FIELD__HEADERS = [
-    ("Properties", "Properties", "Class"),
+    (
+        "Properties",
+        "Properties",
+        {
+            RoleTypes.domain_expert: "Property",
+            RoleTypes.information_architect: "Property",
+            RoleTypes.dms_architect: "View Property",
+        },
+    ),
     ("Classes", "Classes", "Class"),
     ("Containers", "Containers", "Container"),
     ("Views", "Views", "View"),
@@ -131,11 +139,15 @@ def _read_sheets(
             )
             return None, read_info_by_sheet
 
-        for source_sheet_name, target_sheet_name, headers in SOURCE_SHEET__TARGET_FIELD__HEADERS:
+        for source_sheet_name, target_sheet_name, headers_input in SOURCE_SHEET__TARGET_FIELD__HEADERS:
             source_sheet_name = self.to_reference_sheet(source_sheet_name) if self._is_reference else source_sheet_name
 
             if source_sheet_name not in excel_file.sheet_names:
                 continue
+            if isinstance(headers_input, dict):
+                headers = headers_input[metadata.role]
+            else:
+                headers = headers_input
 
             try:
                 sheets[target_sheet_name], read_info_by_sheet[source_sheet_name] = read_individual_sheet(

diff --git a/cognite/neat/rules/issues/dms.py b/cognite/neat/rules/issues/dms.py
@@ -22,7 +22,7 @@
     "DuplicatedViewInDataModelError",
     "ContainerPropertyUsedMultipleTimesError",
     "EmptyContainerWarning",
-    "UnsupportedRelationWarning",
+    "UnsupportedConnectionWarning",
     "MultipleReferenceWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
     "ReverseRelationMissingOtherSideWarning",
@@ -365,25 +365,25 @@ def dump(self) -> dict[str, Any]:
 
 
 @dataclass(frozen=True)
-class UnsupportedRelationWarning(DMSSchemaWarning):
-    description = "The relatio type is not supported by neat"
-    fix = "Change the relation to a supported type"
-    error_name: ClassVar[str] = "UnsupportedRelationWarning"
+class UnsupportedConnectionWarning(DMSSchemaWarning):
+    description = "The connection type is not supported by neat"
+    fix = "Change the connection to a supported type"
+    error_name: ClassVar[str] = "UnsupportedConnectionWarning"
     view_id: dm.ViewId
     property: str
-    relation: str
+    connection: str
 
     def message(self) -> str:
         return (
-            f"The relation {self.relation} in {self.view_id}.{self.property} is not supported."
+            f"The connection {self.connection} in {self.view_id}.{self.property} is not supported."
             "This property will be ignored."
         )
 
     def dump(self) -> dict[str, Any]:
         output = super().dump()
         output["view_id"] = self.view_id.dump()
         output["property"] = self.property
-        output["relation"] = self.relation
+        output["connection"] = self.connection
         return output
 
 

diff --git a/cognite/neat/rules/models/rules/_dms_architect_rules.py b/cognite/neat/rules/models/rules/_dms_architect_rules.py
@@ -188,41 +188,41 @@ def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
 
 class DMSProperty(SheetEntity):
     view: ViewEntity = Field(alias="View")
-    view_property: str = Field(alias="ViewProperty")
+    view_property: str = Field(alias="View Property")
     name: str | None = Field(alias="Name", default=None)
     description: str | None = Field(alias="Description", default=None)
-    relation: Literal["direct", "edge", "reverse"] | None = Field(None, alias="Relation")
+    connection: Literal["direct", "edge", "reverse"] | None = Field(None, alias="Connection")
     value_type: DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity = Field(alias="Value Type")
     nullable: bool | None = Field(default=None, alias="Nullable")
-    is_list: bool | None = Field(default=None, alias="IsList")
+    is_list: bool | None = Field(default=None, alias="Is List")
     default: str | int | dict | None = Field(None, alias="Default")
     reference: URLEntity | ReferenceEntity | None = Field(default=None, alias="Reference", union_mode="left_to_right")
     container: ContainerEntity | None = Field(None, alias="Container")
-    container_property: str | None = Field(None, alias="ContainerProperty")
+    container_property: str | None = Field(None, alias="Container Property")
     index: StrListType | None = Field(None, alias="Index")
     constraint: StrListType | None = Field(None, alias="Constraint")
-    class_: ClassEntity = Field(alias="Class")
-    property_: PropertyType = Field(alias="Property")
+    class_: ClassEntity = Field(alias="Class (linage)")
+    property_: PropertyType = Field(alias="Property (linage)")
 
     @field_validator("nullable")
     def direct_relation_must_be_nullable(cls, value: Any, info: ValidationInfo) -> None:
-        if info.data.get("relation") == "direct" and value is False:
+        if info.data.get("connection") == "direct" and value is False:
             raise ValueError("Direct relation must be nullable")
         return value
 
     @field_validator("value_type", mode="after")
-    def relations_value_type(
+    def connections_value_type(
         cls, value: ViewPropertyEntity | ViewEntity | DMSUnknownEntity, info: ValidationInfo
     ) -> DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity:
-        if (relation := info.data.get("relation")) is None:
+        if (connection := info.data.get("connection")) is None:
             return value
-        if relation == "direct" and not isinstance(value, ViewEntity | DMSUnknownEntity):
+        if connection == "direct" and not isinstance(value, ViewEntity | DMSUnknownEntity):
             raise ValueError(f"Direct relation must have a value type that points to a view, got {value}")
-        elif relation == "edge" and not isinstance(value, ViewEntity):
-            raise ValueError(f"Edge relation must have a value type that points to a view, got {value}")
-        elif relation == "reverse" and not isinstance(value, ViewPropertyEntity | ViewEntity):
+        elif connection == "edge" and not isinstance(value, ViewEntity):
+            raise ValueError(f"Edge connection must have a value type that points to a view, got {value}")
+        elif connection == "reverse" and not isinstance(value, ViewPropertyEntity | ViewEntity):
             raise ValueError(
-                f"Reverse relation must have a value type that points to a view or view property, got {value}"
+                f"Reverse connection must have a value type that points to a view or view property, got {value}"
             )
         return value
 
@@ -241,7 +241,7 @@ class DMSContainer(SheetEntity):
     description: str | None = Field(alias="Description", default=None)
     reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
     constraint: ContainerEntityList | None = Field(None, alias="Constraint")
-    class_: ClassEntity = Field(alias="Class")
+    class_: ClassEntity = Field(alias="Class (linage)")
 
     def as_container(self) -> dm.ContainerApply:
         container_id = self.container.as_id()
@@ -283,8 +283,8 @@ class DMSView(SheetEntity):
     implements: ViewEntityList | None = Field(None, alias="Implements")
     reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
     filter_: HasDataFilter | NodeTypeFilter | None = Field(None, alias="Filter")
-    in_model: bool = Field(True, alias="InModel")
-    class_: ClassEntity = Field(alias="Class")
+    in_model: bool = Field(True, alias="In Model")
+    class_: ClassEntity = Field(alias="Class (linage)")
 
     def as_view(self) -> dm.ViewApply:
         view_id = self.view.as_id()
@@ -917,7 +917,7 @@ def _create_view_property(
         if prop.container and prop.container_property:
             container_prop_identifier = prop.container_property
             extra_args: dict[str, Any] = {}
-            if prop.relation == "direct":
+            if prop.connection == "direct":
                 if isinstance(prop.value_type, ViewEntity):
                     extra_args["source"] = prop.value_type.as_id()
                 elif isinstance(prop.value_type, DMSUnknownEntity):
@@ -928,11 +928,11 @@ def _create_view_property(
                         "If this error occurs it is a bug in NEAT, please report"
                         f"Debug Info, Invalid valueType direct: {prop.model_dump_json()}"
                     )
-            elif prop.relation is not None:
+            elif prop.connection is not None:
                 # Should have been validated.
                 raise ValueError(
                     "If this error occurs it is a bug in NEAT, please report"
-                    f"Debug Info, Invalid relation: {prop.model_dump_json()}"
+                    f"Debug Info, Invalid connection: {prop.model_dump_json()}"
                 )
             return dm.MappedPropertyApply(
                 container=prop.container.as_id(),
@@ -941,7 +941,7 @@ def _create_view_property(
                 description=prop.description,
                 **extra_args,
             )
-        elif prop.relation == "edge":
+        elif prop.connection == "edge":
             if isinstance(prop.value_type, ViewEntity):
                 source_view_id = prop.value_type.as_id()
             else:
@@ -962,7 +962,7 @@ def _create_view_property(
                 name=prop.name,
                 description=prop.description,
             )
-        elif prop.relation == "reverse":
+        elif prop.connection == "reverse":
             reverse_prop_id: str | None = None
             if isinstance(prop.value_type, ViewPropertyEntity):
                 source_view_id = prop.value_type.as_view_id()
@@ -973,7 +973,7 @@ def _create_view_property(
                 # Should have been validated.
                 raise ValueError(
                     "If this error occurs it is a bug in NEAT, please report"
-                    f"Debug Info, Invalid valueType reverse relation: {prop.model_dump_json()}"
+                    f"Debug Info, Invalid valueType reverse connection: {prop.model_dump_json()}"
                 )
             reverse_prop: DMSProperty | None = None
             if reverse_prop_id is not None:
@@ -992,7 +992,7 @@ def _create_view_property(
                     stacklevel=2,
                 )
 
-            if reverse_prop is None or reverse_prop.relation == "edge":
+            if reverse_prop is None or reverse_prop.connection == "edge":
                 inwards_edge_cls = (
                     dm.MultiEdgeConnectionApply if prop.is_list in [True, None] else SingleEdgeConnectionApply
                 )
@@ -1003,7 +1003,7 @@ def _create_view_property(
                     description=prop.description,
                     direction="inwards",
                 )
-            elif reverse_prop_id and reverse_prop and reverse_prop.relation == "direct":
+            elif reverse_prop_id and reverse_prop and reverse_prop.connection == "direct":
                 reverse_direct_cls = (
                     dm.MultiReverseDirectRelationApply if prop.is_list is True else SingleReverseDirectRelationApply
                 )
@@ -1016,9 +1016,9 @@ def _create_view_property(
             else:
                 return None
 
-        elif prop.view and prop.view_property and prop.relation:
+        elif prop.view and prop.view_property and prop.connection:
             warnings.warn(
-                issues.dms.UnsupportedRelationWarning(prop.view.as_id(), prop.view_property, prop.relation or ""),
+                issues.dms.UnsupportedConnectionWarning(prop.view.as_id(), prop.view_property, prop.connection or ""),
                 stacklevel=2,
             )
         return None

diff --git a/cognite/neat/rules/models/rules/_dms_rules_write.py b/cognite/neat/rules/models/rules/_dms_rules_write.py
@@ -77,7 +77,7 @@ class DMSPropertyWrite:
     class_: str | None = None
     name: str | None = None
     description: str | None = None
-    relation: Literal["direct", "edge", "reverse"] | None = None
+    connection: Literal["direct", "edge", "reverse"] | None = None
     nullable: bool | None = None
     is_list: bool | None = None
     default: str | int | dict | None = None
@@ -118,7 +118,7 @@ def load(
             class_=data.get("class_"),
             name=data.get("name"),
             description=data.get("description"),
-            relation=data.get("relation"),
+            connection=data.get("connection"),
             nullable=data.get("nullable"),
             is_list=data.get("is_list"),
             default=data.get("default"),
@@ -143,23 +143,23 @@ def dump(self, default_space: str, default_version: str) -> dict[str, Any]:
 
         return {
             "View": ViewEntity.load(self.view, space=default_space, version=default_version),
-            "ViewProperty": self.view_property,
+            "View Property": self.view_property,
             "Value Type": value_type,
-            "Property": self.property_ or self.view_property,
-            "Class": ClassEntity.load(self.class_, prefix=default_space, version=default_version)
+            "Property (linage)": self.property_ or self.view_property,
+            "Class (linage)": ClassEntity.load(self.class_, prefix=default_space, version=default_version)
             if self.class_
             else None,
             "Name": self.name,
             "Description": self.description,
-            "Relation": self.relation,
+            "Connection": self.connection,
             "Nullable": self.nullable,
-            "IsList": self.is_list,
+            "Is List": self.is_list,
             "Default": self.default,
             "Reference": self.reference,
             "Container": ContainerEntity.load(self.container, space=default_space, version=default_version)
             if self.container
             else None,
-            "ContainerProperty": self.container_property,
+            "Container Property": self.container_property,
             "Index": self.index,
             "Constraint": self.constraint,
         }
@@ -208,19 +208,21 @@ def load(
 
     def dump(self, default_space: str) -> dict[str, Any]:
         container = ContainerEntity.load(self.container, space=default_space)
-        return dict(
-            Container=container,
-            Class=ClassEntity.load(self.class_, prefix=default_space) if self.class_ else container.as_class(),
-            Name=self.name,
-            Description=self.description,
-            Reference=self.reference,
-            Constraint=[
+        return {
+            "Container": container,
+            "Class (linage)": ClassEntity.load(self.class_, prefix=default_space)
+            if self.class_
+            else container.as_class(),
+            "Name": self.name,
+            "Description": self.description,
+            "Reference": self.reference,
+            "Constraint": [
                 ContainerEntity.load(constraint.strip(), space=default_space)
                 for constraint in self.constraint.split(",")
             ]
             if self.constraint
             else None,
-        )
+        }
 
 
 @dataclass
@@ -268,23 +270,23 @@ def load(cls, data: dict[str, Any] | list[dict[str, Any]] | None) -> "DMSViewWri
 
     def dump(self, default_space: str, default_version: str) -> dict[str, Any]:
         view = ViewEntity.load(self.view, space=default_space, version=default_version)
-        return dict(
-            View=view,
-            Class=ClassEntity.load(self.class_, prefix=default_space, version=default_version)
+        return {
+            "View": view,
+            "Class (linage)": ClassEntity.load(self.class_, prefix=default_space, version=default_version)
             if self.class_
             else view.as_class(),
-            Name=self.name,
-            Description=self.description,
-            Implements=[
+            "Name": self.name,
+            "Description": self.description,
+            "Implements": [
                 ViewEntity.load(implement, space=default_space, version=default_version)
                 for implement in self.implements.split(",")
             ]
             if self.implements
             else None,
-            Reference=self.reference,
-            Filter=self.filter_,
-            InModel=self.in_model,
-        )
+            "Reference": self.reference,
+            "Filter": self.filter_,
+            "In Model": self.in_model,
+        }
 
 
 @dataclass

diff --git a/cognite/neat/rules/models/rules/_information_rules.py b/cognite/neat/rules/models/rules/_information_rules.py
@@ -438,7 +438,7 @@ def as_dms_architect_rules(self, created: datetime | None = None, updated: datet
         for class_ in self.information.classes:
             properties: list[DMSProperty] = properties_by_class.get(class_.class_.versioned_id, [])
             if not properties or all(
-                isinstance(prop.value_type, ViewPropertyEntity) and prop.relation != "direct" for prop in properties
+                isinstance(prop.value_type, ViewPropertyEntity) and prop.connection != "direct" for prop in properties
             ):
                 classes_without_properties.add(class_.class_.versioned_id)
 
@@ -511,7 +511,7 @@ def _as_dms_property(cls, prop: InformationProperty, default_space: str, default
             value_type=value_type,
             nullable=nullable,
             is_list=is_list,
-            relation=relation,
+            connection=relation,
             default=prop.default,
             reference=prop.reference,
             container=container,