Merge/release 2.24 into develop ver 3 (#1928)

* merge 2.24 into develop-ver-3

.github/CODEOWNERS

@@ -1 +1 @@
-* @lokm01 @benedeki @DzMakatun @HuvarVer @dk1844 @AdrianOlosutean
+* @benedeki @dk1844 @AdrianOlosutean

.github/ISSUE_TEMPLATE/bug_report.md

@@ -15,7 +15,7 @@ Steps to reproduce the behavior OR commands run:
 3. Enter value '...'
 4. See error
 
-## Expected behaviour
+## Expected behavior
 A clear and concise description of what you expected to happen.
 
 ## Screenshots

.github/ISSUE_TEMPLATE/feature_request.md

@@ -15,7 +15,7 @@ A description of the requested feature.
 A simple example if applicable.
 
 ## Proposed Solution [Optional]
-Solution Ideas 
+Solution Ideas:
 1. 
 2. 
 3. 

.github/ISSUE_TEMPLATE/poc.md

@@ -0,0 +1,18 @@
+---
+name: POC
+about: Proof of Concept, usually a middle-sized effort to test some idea
+labels: 'poc, under discussion, priority: undecided'
+
+---
+
+## Background
+A clear and concise intro into the situation.
+
+## Goal
+The goal that the _Proof of Concept_ wants to test
+
+## Proposed Approach [Optional]
+Approach Ideas:
+1.
+2.
+3.

.github/workflows/license_check.yml

@@ -28,4 +28,4 @@ jobs:
       - uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      - run: mvn -Plicense-check apache-rat:check
+      - run: mvn --no-transfer-progress -Plicense-check apache-rat:check

README.md

@@ -13,13 +13,20 @@
 
 # Enceladus
 
+### <a name="latest_release"/>Latest Release
+[![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa.enceladus/parent/badge.png)](https://maven-badges.herokuapp.com/maven-central/za.co.absa.enceladus/parent/)
+
 ### <a name="build_status"/>Build Status
 | master | develop |
 | ------------- | ------------- |
 | [![Build Status](https://opensource.bigusdatus.com/jenkins/buildStatus/icon?job=Absa-OSS-Projects%2Fenceladus%2Fmaster)](https://opensource.bigusdatus.com/jenkins/job/Absa-OSS-Projects/job/enceladus/job/master/) | [![Build Status](https://opensource.bigusdatus.com/jenkins/buildStatus/icon?job=Absa-OSS-Projects%2Fenceladus%2Fdevelop)](https://opensource.bigusdatus.com/jenkins/job/Absa-OSS-Projects/job/enceladus/job/develop/) |
 ### <a name="code_quality_status"/>Code Quality Status
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=AbsaOSS_enceladus&metric=alert_status)](https://sonarcloud.io/dashboard?id=AbsaOSS_enceladus)
 
+### <a name="documentation"/>Documentation
+[![Read the Docs](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://absaoss.github.io/enceladus/)
+[![Read the Docs](https://img.shields.io/badge/docs-release%20notes-yellow.svg)](https://absaoss.github.io/enceladus/blog/)
+[![Read the Docs](https://img.shields.io/badge/docs-release--1.x-red.svg)](https://absaoss.github.io/enceladus/docs/1.0.0/components)
 ___
 
 <!-- toc -->
@@ -33,7 +40,6 @@ ___
 - [Plugins](#plugins)
     - [Built-in Plugins](#built-in-plugins)
 - [How to contribute](#how-to-contribute)
-- [Documentation](#documentation)
 <!-- tocstop -->
 
 ## What is Enceladus?
@@ -349,6 +355,3 @@ A module containing [examples](examples/README.md) of the project usage.
 
 ## How to contribute
 Please see our [**Contribution Guidelines**](CONTRIBUTING.md).
-
-## Documentation
-Please see the [documentation pages](https://absaoss.github.io/enceladus/).

dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala

@@ -15,7 +15,12 @@
 
 package za.co.absa.enceladus.dao.rest
 
+import java.time.ZonedDateTime
+
 import org.scalactic.{AbstractStringUniformity, Uniformity}
+import za.co.absa.enceladus.model.conformanceRule.{CastingConformanceRule, LiteralConformanceRule, MappingConformanceRule}
+import za.co.absa.enceladus.model.dataFrameFilter._
+import za.co.absa.enceladus.model.menas.MenasReference
 import za.co.absa.enceladus.model.test.VersionedModelMatchers
 import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, RunFactory, SchemaFactory}
 import za.co.absa.enceladus.model.{Dataset, MappingTable, Run, Schema}
@@ -82,7 +87,7 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
       """{
         |  "name": "Test",
         |  "version": 5,
-        |  "description": "",
+        |  "description": "some description here",
         |  "hdfsPath": "/bigdata/test",
         |  "hdfsPublishPath": "/bigdata/test2",
         |  "schemaName": "Cobol1",
@@ -114,11 +119,61 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
         |      },
         |      "targetAttribute": "CCC",
         |      "outputColumn": "ConformedCCC",
-        |      "isNullSafe": true
+        |      "additionalColumns": null,
+        |      "isNullSafe": true,
+        |      "mappingTableFilter": {
+        |        "_t": "AndJoinedFilters",
+        |        "filterItems": [
+        |          {
+        |            "_t": "OrJoinedFilters",
+        |            "filterItems": [
+        |              {
+        |                "_t": "EqualsFilter",
+        |                "columnName": "column1",
+        |                "value": "soughtAfterValue",
+        |                "valueType": "string"
+        |              },
+        |              {
+        |                "_t": "EqualsFilter",
+        |                "columnName": "column1",
+        |                "value": "alternativeSoughtAfterValue",
+        |                "valueType": "string"
+        |              }
+        |            ]
+        |          },
+        |          {
+        |            "_t": "DiffersFilter",
+        |            "columnName": "column2",
+        |            "value": "anotherValue",
+        |            "valueType": "string"
+        |          },
+        |          {
+        |            "_t": "NotFilter",
+        |            "inputFilter": {
+        |              "_t": "IsNullFilter",
+        |              "columnName": "col3"
+        |            }
+        |          }
+        |        ]
+        |      },
+        |      "overrideMappingTableOwnFilter": true
+        |    },
+        |    {
+        |      "_t": "MappingConformanceRule",
+        |      "order": 2,"controlCheckpoint": true,
+        |      "mappingTable": "CurrencyMappingTable2",
+        |      "mappingTableVersion": 10,
+        |      "attributeMappings": {},
+        |      "targetAttribute": "CCC",
+        |      "outputColumn": "ConformedCCC",
+        |      "additionalColumns": null,
+        |      "isNullSafe": false,
+        |      "mappingTableFilter": null,
+        |      "overrideMappingTableOwnFilter": false
         |    },
         |    {
         |      "_t": "LiteralConformanceRule",
-        |      "order": 2,
+        |      "order": 3,
         |      "outputColumn": "ConformedLiteral",
         |      "controlCheckpoint": false,
         |      "value": "AAA"
@@ -130,6 +185,8 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
         |    "version": 4
         |  },
         |  "schedule": null,
+        |  "properties": null,
+        |  "propertiesValidation": null,
         |  "createdMessage": {
         |    "menasRef": {
         |      "collection": null,
@@ -143,14 +200,76 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
         |        "field": "",
         |        "oldValue": null,
         |        "newValue": null,
-        |        "message": "Test"
+        |        "message": "Dataset Test created."
         |      }
         |    ]
         |  }
         |}""".stripMargin
 
-      "deserializing should not throw" in {
-        JsonSerializer.fromJson[Dataset](datasetJson)
+      val dataset: Dataset = DatasetFactory.getDummyDataset(
+        name = "Test",
+        version = 5,
+        description = Some("some description here"),
+        hdfsPath = "/bigdata/test",
+        hdfsPublishPath = "/bigdata/test2",
+        schemaName = "Cobol1",
+        schemaVersion = 3,
+        dateCreated = ZonedDateTime.parse("2019-07-22T08:05:57.47Z"),
+        userCreated = "system",
+        lastUpdated = ZonedDateTime.parse("2020-04-02T15:53:02.947Z"),
+        userUpdated = "system",
+
+        conformance = List(
+          CastingConformanceRule(0,
+            outputColumn = "ConformedInt",
+            controlCheckpoint = false,
+            inputColumn = "STRING_VAL",
+            outputDataType = "integer"
+          ),
+          MappingConformanceRule(1,
+            controlCheckpoint = true,
+            mappingTable = "CurrencyMappingTable",
+            mappingTableVersion = 9, //scalastyle:ignore magic.number
+            attributeMappings = Map("InputValue" -> "STRING_VAL"),
+            targetAttribute = "CCC",
+            outputColumn = "ConformedCCC",
+            isNullSafe = true,
+            mappingTableFilter = Some(
+              AndJoinedFilters(Set(
+                OrJoinedFilters(Set(
+                  EqualsFilter("column1", "soughtAfterValue"),
+                  EqualsFilter("column1", "alternativeSoughtAfterValue")
+                )),
+                DiffersFilter("column2", "anotherValue"),
+                NotFilter(IsNullFilter("col3"))
+              ))
+            ),
+            overrideMappingTableOwnFilter = Some(true)
+          ),
+          MappingConformanceRule(2,
+            controlCheckpoint = true,
+            mappingTable = "CurrencyMappingTable2",
+            mappingTableVersion = 10, //scalastyle:ignore magic.number
+            attributeMappings = Map(),
+            targetAttribute = "CCC",
+            outputColumn = "ConformedCCC"
+          ),
+          LiteralConformanceRule(3,
+            outputColumn = "ConformedLiteral",
+            controlCheckpoint = false,
+            value = "AAA"
+          )
+        ),
+        parent = Some(MenasReference(Some("dataset"),"Test", 4)) // scalastyle:off magic.number
+      )
+
+      "serializing" in {
+        val result = JsonSerializer.toJson(dataset)
+        result should equal(datasetJson)(after being whiteSpaceNormalised)
+      }
+      "deserializing" in {
+        val result = JsonSerializer.fromJson[Dataset](datasetJson)
+        result should matchTo(dataset)
       }
     }
 
@@ -227,8 +346,7 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
 
     "handle MappingTables" when {
       val mappingTableJson =
-        """
-          |{
+        """{
           |  "name": "dummyName",
           |  "version": 1,
           |  "description": null,
@@ -277,6 +395,106 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
       }
     }
 
+    "handle MappingTables with filters" when {
+      val mappingTableJson =
+        """
+          |{
+          |  "name": "dummyName",
+          |  "version": 1,
+          |  "description": null,
+          |  "hdfsPath": "/dummy/path",
+          |  "schemaName": "dummySchema",
+          |  "schemaVersion": 1,
+          |  "defaultMappingValue": [],
+          |  "dateCreated": "2017-12-04T16:19:17Z",
+          |  "userCreated": "dummyUser",
+          |  "lastUpdated": "2017-12-04T16:19:17Z",
+          |  "userUpdated": "dummyUser",
+          |  "disabled": false,
+          |  "dateDisabled": null,
+          |  "userDisabled": null,
+          |  "parent": null,
+          |  "filter": {
+          |    "_t": "AndJoinedFilters",
+          |    "filterItems": [
+          |      {
+          |        "_t": "OrJoinedFilters",
+          |        "filterItems": [
+          |          {
+          |            "_t": "EqualsFilter",
+          |            "columnName": "column1",
+          |            "value": "soughtAfterValue",
+          |            "valueType": "string"
+          |          },
+          |          {
+          |            "_t": "EqualsFilter",
+          |            "columnName": "column1",
+          |            "value": "alternativeSoughtAfterValue",
+          |            "valueType": "string"
+          |          }
+          |        ]
+          |      },
+          |      {
+          |        "_t": "DiffersFilter",
+          |        "columnName": "column2",
+          |        "value": "anotherValue",
+          |        "valueType": "string"
+          |      },
+          |      {
+          |        "_t": "NotFilter",
+          |        "inputFilter": {
+          |          "_t": "IsNullFilter",
+          |          "columnName": "col3"
+          |        }
+          |      }
+          |    ]
+          |  },
+          |  "createdMessage": {
+          |    "menasRef": {
+          |      "collection": null,
+          |      "name": "dummyName",
+          |      "version": 1
+          |    },
+          |    "updatedBy": "dummyUser",
+          |    "updated": "2017-12-04T16:19:17Z",
+          |    "changes": [
+          |      {
+          |        "field": "",
+          |        "oldValue": null,
+          |        "newValue": null,
+          |        "message": "Mapping Table dummyName created."
+          |      }
+          |    ]
+          |  },
+          |  "defaultMappingValues": {}
+          |}
+          |""".stripMargin
+
+      val mappingTable = MappingTableFactory.getDummyMappingTable(
+        filter = Some(
+          AndJoinedFilters(Set(
+            OrJoinedFilters(Set(
+              EqualsFilter("column1", "soughtAfterValue"),
+              EqualsFilter("column1", "alternativeSoughtAfterValue")
+            )),
+            DiffersFilter("column2", "anotherValue"),
+            NotFilter(
+              IsNullFilter("col3")
+            )
+          ))
+        )
+      )
+
+      "serializing" in {
+        val result = JsonSerializer.toJson(mappingTable)
+        result should equal(mappingTableJson)(after being whiteSpaceNormalised)
+      }
+      "deserializing" in {
+        val result = JsonSerializer.fromJson[MappingTable](mappingTableJson)
+        result should matchTo(mappingTable)
+      }
+    }
+
     "handle Schemas" when {
       val schemaJson =
         """

data-model/src/main/scala/za/co/absa/enceladus/model/SchemaField.scala

@@ -41,4 +41,9 @@ case class SchemaField
   def getAllChildren: Seq[String] = {
     children.flatMap(child => child.getAllChildren :+ child.getAbsolutePath)
   }
+
+  @JsonIgnore
+  def getAllChildrenBasePath: Seq[String] = {
+    children.flatMap(child => child.getAllChildrenBasePath :+ child.path)
+  }
 }