Merge/merging release 2.26.X into develop ver3 (#1997)

.gitignore

@@ -55,8 +55,6 @@ build.log
 #  syntax: regexp
 #  ^\.pc/
 
-build.log
-
 .cache*
 dependency-reduced-pom.xml
 

data-model/src/main/scala/za/co/absa/enceladus/model/ConformedSchema.scala

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.model
+
+import org.apache.spark.sql.types.StructField
+import za.co.absa.enceladus.model.conformanceRule._
+
+case class ConformedSchema(schema: List[StructField], dataset: Dataset) {
+  def hasField(field: String): Boolean = {
+    if (schema.exists(_.name == field)) true else {
+      val ss = dataset.conformance.find {
+        case MappingConformanceRule(_, _, _, _, _, _, outputColumn, additionalColumns, _, _, _) =>
+          outputColumn == field || additionalColumns.getOrElse(Map()).contains(field)
+        case SingleColumnConformanceRule(_, _, outputColumn, _, inputColumnAlias) =>
+          outputColumn == field || field == outputColumn + "." + inputColumnAlias
+        case DropConformanceRule(_, _, _) => false
+        case c: ConformanceRule => c.outputColumn == field
+      }
+
+      ss match {
+        case None => false
+        case Some(matchedRule: ConformanceRule) =>
+          val maybeRule = dataset.conformance.find {
+            case DropConformanceRule(_, _, outputCol) => outputCol == matchedRule.outputColumn
+            case _ => false
+          }
+          maybeRule.isEmpty
+        }
+    }
+  }
+}
+
+

data-model/src/main/scala/za/co/absa/enceladus/model/conformanceRule/package.scala

@@ -88,9 +88,11 @@ package object conformanceRule {
                                    ) extends ConformanceRule {
 
     def allOutputColumns(): Map[String, String] = {
-      additionalColumns.getOrElse(Map()) + (outputColumn -> targetAttribute)
+      definedAdditionalColumns() + (outputColumn -> targetAttribute)
     }
 
+    def definedAdditionalColumns(): Map[String, String] = additionalColumns.getOrElse(Map())
+
     override def withUpdatedOrder(newOrder: Int): MappingConformanceRule = copy(order = newOrder)
 
     override def connectedEntities: Seq[ConnectedEntity] = Seq(

data-model/src/test/scala/za/co/absa/enceladus/model/ConformedSchemaTest.scala

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.model
+
+import org.apache.spark.sql.types.{StringType, StructField}
+import org.scalatest.funsuite.AnyFunSuite
+import za.co.absa.enceladus.model.conformanceRule.{DropConformanceRule, LiteralConformanceRule, MappingConformanceRule, SingleColumnConformanceRule}
+
+class ConformedSchemaTest extends AnyFunSuite{
+  private val conformanceRule1 = LiteralConformanceRule(
+    order = 0,
+    controlCheckpoint = true,
+    outputColumn = "something",
+    value = "1.01"
+  )
+
+  private val conformanceRule1d = LiteralConformanceRule(
+    order = 0,
+    controlCheckpoint = true,
+    outputColumn = "fieldToDelete",
+    value = "1.01"
+  )
+
+  private val conformanceRule2 = DropConformanceRule(order = 0,
+    controlCheckpoint = true,
+    outputColumn = "fieldToDelete")
+
+  private val conformanceRule3 = MappingConformanceRule(order = 0,
+    controlCheckpoint = true,
+    outputColumn = "something3",additionalColumns = Some(Map("newCol" -> "mappedCol")),
+    mappingTable = "",mappingTableVersion = 1,
+    attributeMappings = Map(),targetAttribute = "col")
+
+  private val conformanceRule4 = SingleColumnConformanceRule(
+    order = 0,
+    outputColumn = "singleCol",inputColumn = "as",
+    inputColumnAlias = "subCol", controlCheckpoint = false)
+
+  private val dataset = Dataset(name = "Test DS",
+    version = 1,
+    hdfsPath = "newPath",
+    hdfsPublishPath = "newPublishPath",
+    schemaName = "newSchema",
+    schemaVersion = 1,
+    conformance = List(conformanceRule1, conformanceRule1d, conformanceRule2, conformanceRule3, conformanceRule4),
+    properties = Some(Map(
+      "property1" -> "value1",
+      "property2.sub" -> "value2"
+    )
+    ))
+
+  val schemaFields = List(StructField("stdField",StringType))
+
+  test("conformed schema") {
+    val conformedSchema = ConformedSchema(schemaFields, dataset)
+    assertResult(conformedSchema.hasField("stdField"))(true)
+    assertResult(conformedSchema.hasField("fieldToDelete"))(false)
+    assertResult(conformedSchema.hasField("something"))(true)
+    assertResult(conformedSchema.hasField("newCol"))(true)
+    assertResult(conformedSchema.hasField("newCol1"))(false)
+    assertResult(conformedSchema.hasField("mappedColCol1"))(false)
+    assertResult(conformedSchema.hasField("something3"))(true)
+    assertResult(conformedSchema.hasField("col"))(false)
+    assertResult(conformedSchema.hasField("singleCol"))(true)
+    assertResult(conformedSchema.hasField("singleCol.subCol"))(true)
+  }
+}

menas/ui/components/dataset/conformanceRule/ConformanceRuleDialog.js

@@ -109,10 +109,10 @@ class ConformanceRuleDialog {
     this.beforeSubmitChanges(newRule);
     this.resetRuleValidation();
 
-    newRule.hasValidFilter = true; // default for non-MappingConformanceRules
+    newRule.filterValidations = {empty: true, valid: true}; // default for non-MappingConformanceRules
     if (newRule._t === "MappingConformanceRule") {
-      const validFilter = this.filterEdit.validateFilterData();
-      newRule.hasValidFilter = validFilter;
+      const filterValidations = this.filterEdit.validateFilterData();
+      newRule.filterValidations = filterValidations;
     }
 
     if (this.ruleForms.byType(newRule._t).isValid(newRule, this.controller._transitiveSchemas, currentDataset.conformance)) {

menas/ui/components/dataset/conformanceRule/ConformanceRuleForm.js

@@ -369,18 +369,20 @@ class MappingConformanceRuleForm extends ConformanceRuleForm {
   isCorrectlyConfigured(rule) {
     return this.hasValidInputColumn(rule.targetAttribute)
       & this.hasValidOutputColumns(rule)
-      & this.hasValidJoinConditions(rule.newJoinConditions)
-      & rule.hasValidFilter;
+      & this.hasValidJoinConditions(rule.newJoinConditions, rule.filterValidations.empty)
+      & rule.filterValidations.valid;
   }
 
-  hasValidJoinConditions(fieldValue = []) {
-    let isValid = fieldValue.length >= 1;
+  hasValidJoinConditions(fieldValue = [], filtersEmpty) {
+    const nonEmptyJoinConditions = fieldValue.length >= 1;
+    const validJoinConditions = (nonEmptyJoinConditions || !filtersEmpty);
 
-    if (!isValid) {
-      sap.m.MessageToast.show("At least 1 join condition is required.");
+
+    if (!validJoinConditions) {
+      sap.m.MessageToast.show("Either provide a join condition or a filter!");
     }
 
-    return isValid
+    return validJoinConditions;
   }
 
   hasValidOutputColumns(rule = []) {

menas/ui/components/dataset/conformanceRule/MappingConformanceRule/add.fragment.xml

@@ -14,6 +14,7 @@
   -->
 
 <core:FragmentDefinition xmlns="sap.m" xmlns:core="sap.ui.core">
+    <core:Fragment type="XML" fragmentName="components.dataset.conformanceRule.add.checkpointBox"/>
     <core:Fragment type="XML" fragmentName="components.mappingTableSelector"/>
     <core:Fragment type="XML" fragmentName="components.dataset.conformanceRule.MappingConformanceRule.joinConditions"/>
     <core:Fragment type="XML" fragmentName="components.dataset.conformanceRule.MappingConformanceRule.outputColumns"/>

menas/ui/components/dataset/conformanceRule/MappingConformanceRule/display.fragment.xml

@@ -18,6 +18,10 @@
     <CustomListItem>
         <form:SimpleForm adjustLabelSpan="true" editable="false">
             <form:content>
+                <!-- normally, commonRuleFields.fragment.xml would be used here, but for MRC we need special content bc. of multiple outputs -->
+                <Title text="{=${order} + 1}. {_t}" level="H2"/>
+                <core:Fragment type="XML" fragmentName="components.dataset.conformanceRule.add.checkpointBox"/>
+
                 <Label text="Mapping Table"/>
                 <Link text="{mappingTable} (v{mappingTableVersion})" press="toMappingTable"
                       cust:name="{mappingTable}" cust:version="{mappingTableVersion}"/>

menas/ui/components/dataset/conformanceRule/MappingConformanceRule/joinConditions.fragment.xml

@@ -21,7 +21,7 @@
     <Label text="Join Conditions"/>
     <List id="joinConditions"
           items="{/newRule/newJoinConditions}"
-          noDataText="No join conditions (1 minimum)"
+          noDataText="No join conditions"
           mode="Delete"
           delete=".onDeleteJoinCondition">
         <items>

menas/ui/components/dataset/conformanceRule/MappingConformanceRule/outputColumns.fragment.xml

@@ -16,7 +16,6 @@
 <core:FragmentDefinition xmlns="sap.m" xmlns:core="sap.ui.core"
                          xmlns:cust="http://schemas.sap.com/sapui5/extension/sap.ui.core.CustomData/1">
 
-    <CheckBox selected="{/newRule/isNullSafe}"/>
     <Label text="Output Columns"/>
     <List id="outputColumns"
           items="{/newRule/newOutputColumns}"

menas/ui/components/mappingTable/filterEdit/FilterEdit.js

@@ -198,72 +198,78 @@ class FilterEdit {
   }
 
   /**
-   * Validates data in the filter TreeTable, sets their valueState|valueStateText (error+error descs)
-   * @returns {boolean} returns true if the filter content is valid, false when invalid
+   * Validates data and emptiness in the filter TreeTable, sets their valueState|valueStateText (error+error descs)
+   * @returns {empty: boolean, valid: boolean}
    */
   validateFilterData() {
     const treeTable = this.#getById("filterTreeEdit");
     const treeTableModel = treeTable.getBinding().getModel();
     const filterData = treeTableModel.getProperty("/editingFilters");
 
-    let hasValidFilter = true;
     // filter data can be [filter], [null] or null
     if (!filterData || filterData.filter(x => x).length == 0) {
-      hasValidFilter = true;
-    } else {
-      // validate filter tree
-      const validateInUiFn = function (filterNode) {
-        switch (filterNode._t) {
-          case "AndJoinedFilters":
-          case "OrJoinedFilters":
-            if (filterNode.filterItems.filter(x => x).length == 0) { // empty deleted ([null]) is not valid
-              filterNode.filter_valueState = "Error";
-              filterNode.filter_valueStateText = "Container filter must contain child filters!";
-              hasValidFilter = false;
-            }
-            break;
-
-          case "NotFilter":
-            if (!filterNode.inputFilter) {
-              filterNode.filter_valueState = "Error";
-              filterNode.filter_valueStateText = "Container filter must contain a child filter!";
-              hasValidFilter = false;
-            }
-            break;
-
-          case "EqualsFilter":
-          case "DiffersFilter":
-            if (filterNode.columnName.length == 0) {
-              filterNode.columnName_valueState = "Error";
-              filterNode.columnName_valueStateText = "Select the column.";
-              hasValidFilter = false;
-            }
-
-            if (filterNode.value.length == 0) {
-              filterNode.value_valueState = "Error";
-              filterNode.value_valueStateText = "Fill in the value.";
-              hasValidFilter = false;
-            }
-            break;
-
-          case "IsNullFilter":
-            if (filterNode.columnName.length == 0) {
-              filterNode.columnName_valueState = "Error";
-              filterNode.columnName_valueStateText = "Fill in column name.";
-              hasValidFilter = false;
-            }
-            break;
-
-          default:
-        }
+      return {
+        empty: true,
+        valid: true
       };
-
-      const validatedFilter = FilterTreeUtils.applyToFilterDataImmutably(filterData[0], validateInUiFn);
-      treeTableModel.setProperty("/editingFilters", [validatedFilter]);
-      treeTableModel.refresh();
     }
 
-    return hasValidFilter;
+    // nonempty filter: validate filter tree
+    let filterValid = true;
+    const validateInUiFn = function (filterNode) {
+      switch (filterNode._t) {
+        case "AndJoinedFilters":
+        case "OrJoinedFilters":
+          if (filterNode.filterItems.filter(x => x).length == 0) { // empty deleted ([null]) is not valid
+            filterNode.filter_valueState = "Error";
+            filterNode.filter_valueStateText = "Container filter must contain child filters!";
+            filterValid = false;
+          }
+          break;
+
+        case "NotFilter":
+          if (!filterNode.inputFilter) {
+            filterNode.filter_valueState = "Error";
+            filterNode.filter_valueStateText = "Container filter must contain a child filter!";
+            filterValid = false;
+          }
+          break;
+
+        case "EqualsFilter":
+        case "DiffersFilter":
+          if (filterNode.columnName.length == 0) {
+            filterNode.columnName_valueState = "Error";
+            filterNode.columnName_valueStateText = "Select the column.";
+            filterValid = false;
+          }
+
+          if (filterNode.value.length == 0) {
+            filterNode.value_valueState = "Error";
+            filterNode.value_valueStateText = "Fill in the value.";
+            filterValid = false;
+          }
+          break;
+
+        case "IsNullFilter":
+          if (filterNode.columnName.length == 0) {
+            filterNode.columnName_valueState = "Error";
+            filterNode.columnName_valueStateText = "Fill in column name.";
+            filterValid = false;
+          }
+          break;
+
+        default:
+      }
+    };
+
+    const validatedFilter = FilterTreeUtils.applyToFilterDataImmutably(filterData[0], validateInUiFn);
+    treeTableModel.setProperty("/editingFilters", [validatedFilter]);
+    treeTableModel.refresh();
+
+    return {
+      empty: false,
+      valid: filterValid
+    };
   }
 
   /**

menas/ui/components/mappingTable/filterEdit/filterTreeEdit.fragment.xml

@@ -28,9 +28,9 @@
                         <Menu>
                             <MenuItem id="addAndBtn" text="AND" icon="sap-icon://combine" />
                             <MenuItem id="addOrBtn" text="OR" icon="sap-icon://split" />
-                            <MenuItem id="addNotBtn" text="NOT" icon="sap-icon://filter" />
-                            <MenuItem id="addEqualsBtn" text="Equals" icon="sap-icon://clear-filter" />
-                            <MenuItem id="addDiffersBtn" text="Differs" icon="sap-icon://SAP-icons-TNT/solution-not-licensed" />
+                            <MenuItem id="addNotBtn" text="NOT" icon="sap-icon://SAP-icons-TNT/solution-not-licensed" />
+                            <MenuItem id="addEqualsBtn" text="Equals" icon="sap-icon://filter" />
+                            <MenuItem id="addDiffersBtn" text="Differs" icon="sap-icon://clear-filter" />
                             <MenuItem id="addIsNullBtn" text="IsNull" icon="sap-icon://SAP-icons-TNT/marquee" />
                         </Menu>
                     </menu>

menas/ui/service/EntityDialog.js

@@ -453,7 +453,7 @@ class MappingTableDialog extends EntityDialog {
       this.oController.byId("newMappingTableName"));
     let hasValidSchema = EntityValidationService.hasValidSchema(oMT, "Mapping Table",
       this.oController.byId("schemaVersionSelect"));
-    let hasValidFilter = this.filterEdit.validateFilterData();
+    let hasValidFilter = this.filterEdit.validateFilterData().valid; // validity flag suffices, emptiness irrelevant
 
     if (oMT.hdfsBrowserEnabled) {
       let hasValidHDFSPath = EntityValidationService.hasValidHDFSPath(oMT.hdfsPath,