diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bc0344893..a2f1254c6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @lokm01 @benedeki @DzMakatun @HuvarVer @dk1844 @AdrianOlosutean +* @benedeki @dk1844 @AdrianOlosutean diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e4884938c..51f2a7680 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -15,7 +15,7 @@ Steps to reproduce the behavior OR commands run: 3. Enter value '...' 4. See error -## Expected behaviour +## Expected behavior A clear and concise description of what you expected to happen. ## Screenshots diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 64bed30d2..84b87ffa6 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -15,7 +15,7 @@ A description of the requested feature. A simple example if applicable. ## Proposed Solution [Optional] -Solution Ideas +Solution Ideas: 1. 2. 3. diff --git a/.github/ISSUE_TEMPLATE/poc.md b/.github/ISSUE_TEMPLATE/poc.md new file mode 100644 index 000000000..cf9dfe4fc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/poc.md @@ -0,0 +1,18 @@ +--- +name: POC +about: Proof of Concept, usually a middle-sized effort to test some idea +labels: 'poc, under discussion, priority: undecided' + +--- + +## Background +A clear and concise intro into the situation. + +## Goal +The goal that the _Proof of Concept_ wants to test + +## Proposed Approach [Optional] +Approach Ideas: +1. +2. +3. diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index f8c7697bb..916047963 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -28,4 +28,4 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 1.8 - - run: mvn -Plicense-check apache-rat:check + - run: mvn --no-transfer-progress -Plicense-check apache-rat:check diff --git a/README.md b/README.md index a9f5a999a..81e8e756b 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ # Enceladus +### Latest Release +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa.enceladus/parent/badge.png)](https://maven-badges.herokuapp.com/maven-central/za.co.absa.enceladus/parent/) + ### Build Status | master | develop | | ------------- | ------------- | @@ -20,6 +23,10 @@ ### Code Quality Status [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=AbsaOSS_enceladus&metric=alert_status)](https://sonarcloud.io/dashboard?id=AbsaOSS_enceladus) +### Documentation +[![Read the Docs](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://absaoss.github.io/enceladus/) +[![Read the Docs](https://img.shields.io/badge/docs-release%20notes-yellow.svg)](https://absaoss.github.io/enceladus/blog/) +[![Read the Docs](https://img.shields.io/badge/docs-release--1.x-red.svg)](https://absaoss.github.io/enceladus/docs/1.0.0/components) ___ @@ -33,7 +40,6 @@ ___ - [Plugins](#plugins) - [Built-in Plugins](#built-in-plugins) - [How to contribute](#how-to-contribute) -- [Documentation](#documentation) ## What is Enceladus? @@ -349,6 +355,3 @@ A module containing [examples](examples/README.md) of the project usage. ## How to contribute Please see our [**Contribution Guidelines**](CONTRIBUTING.md). - -## Documentation -Please see the [documentation pages](https://absaoss.github.io/enceladus/). diff --git a/dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala b/dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala index 443df246c..d4bb8d4fe 100644 --- a/dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala +++ b/dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala @@ -15,7 +15,12 @@ package za.co.absa.enceladus.dao.rest +import java.time.ZonedDateTime + import org.scalactic.{AbstractStringUniformity, Uniformity} +import za.co.absa.enceladus.model.conformanceRule.{CastingConformanceRule, LiteralConformanceRule, MappingConformanceRule} +import za.co.absa.enceladus.model.dataFrameFilter._ +import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.test.VersionedModelMatchers import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, RunFactory, SchemaFactory} import za.co.absa.enceladus.model.{Dataset, MappingTable, Run, Schema} @@ -82,7 +87,7 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers { """{ | "name": "Test", | "version": 5, - | "description": "", + | "description": "some description here", | "hdfsPath": "/bigdata/test", | "hdfsPublishPath": "/bigdata/test2", | "schemaName": "Cobol1", @@ -114,11 +119,61 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers { | }, | "targetAttribute": "CCC", | "outputColumn": "ConformedCCC", - | "isNullSafe": true + | "additionalColumns": null, + | "isNullSafe": true, + | "mappingTableFilter": { + | "_t": "AndJoinedFilters", + | "filterItems": [ + | { + | "_t": "OrJoinedFilters", + | "filterItems": [ + | { + | "_t": "EqualsFilter", + | "columnName": "column1", + | "value": "soughtAfterValue", + | "valueType": "string" + | }, + | { + | "_t": "EqualsFilter", + | "columnName": "column1", + | "value": "alternativeSoughtAfterValue", + | "valueType": "string" + | } + | ] + | }, + | { + | "_t": "DiffersFilter", + | "columnName": "column2", + | "value": "anotherValue", + | "valueType": "string" + | }, + | { + | "_t": "NotFilter", + | "inputFilter": { + | "_t": "IsNullFilter", + | "columnName": "col3" + | } + | } + | ] + | }, + | "overrideMappingTableOwnFilter": true + | }, + | { + | "_t": "MappingConformanceRule", + | "order": 2,"controlCheckpoint": true, + | "mappingTable": "CurrencyMappingTable2", + | "mappingTableVersion": 10, + | "attributeMappings": {}, + | "targetAttribute": "CCC", + | "outputColumn": "ConformedCCC", + | "additionalColumns": null, + | "isNullSafe": false, + | "mappingTableFilter": null, + | "overrideMappingTableOwnFilter": false | }, | { | "_t": "LiteralConformanceRule", - | "order": 2, + | "order": 3, | "outputColumn": "ConformedLiteral", | "controlCheckpoint": false, | "value": "AAA" @@ -130,6 +185,8 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers { | "version": 4 | }, | "schedule": null, + | "properties": null, + | "propertiesValidation": null, | "createdMessage": { | "menasRef": { | "collection": null, @@ -143,14 +200,76 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers { | "field": "", | "oldValue": null, | "newValue": null, - | "message": "Test" + | "message": "Dataset Test created." | } | ] | } |}""".stripMargin - "deserializing should not throw" in { - JsonSerializer.fromJson[Dataset](datasetJson) + val dataset: Dataset = DatasetFactory.getDummyDataset( + name = "Test", + version = 5, + description = Some("some description here"), + hdfsPath = "/bigdata/test", + hdfsPublishPath = "/bigdata/test2", + schemaName = "Cobol1", + schemaVersion = 3, + dateCreated = ZonedDateTime.parse("2019-07-22T08:05:57.47Z"), + userCreated = "system", + lastUpdated = ZonedDateTime.parse("2020-04-02T15:53:02.947Z"), + userUpdated = "system", + + conformance = List( + CastingConformanceRule(0, + outputColumn = "ConformedInt", + controlCheckpoint = false, + inputColumn = "STRING_VAL", + outputDataType = "integer" + ), + MappingConformanceRule(1, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable", + mappingTableVersion = 9, //scalastyle:ignore magic.number + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ), + MappingConformanceRule(2, + controlCheckpoint = true, + mappingTable = "CurrencyMappingTable2", + mappingTableVersion = 10, //scalastyle:ignore magic.number + attributeMappings = Map(), + targetAttribute = "CCC", + outputColumn = "ConformedCCC" + ), + LiteralConformanceRule(3, + outputColumn = "ConformedLiteral", + controlCheckpoint = false, + value = "AAA" + ) + ), + parent = Some(MenasReference(Some("dataset"),"Test", 4)) // scalastyle:off magic.number + ) + + "serializing" in { + val result = JsonSerializer.toJson(dataset) + result should equal(datasetJson)(after being whiteSpaceNormalised) + } + "deserializing" in { + val result = JsonSerializer.fromJson[Dataset](datasetJson) + result should matchTo(dataset) } } @@ -227,8 +346,7 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers { "handle MappingTables" when { val mappingTableJson = - """ - |{ + """{ | "name": "dummyName", | "version": 1, | "description": null, @@ -277,6 +395,106 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers { } } + "handle MappingTables with filters" when { + val mappingTableJson = + """ + |{ + | "name": "dummyName", + | "version": 1, + | "description": null, + | "hdfsPath": "/dummy/path", + | "schemaName": "dummySchema", + | "schemaVersion": 1, + | "defaultMappingValue": [], + | "dateCreated": "2017-12-04T16:19:17Z", + | "userCreated": "dummyUser", + | "lastUpdated": "2017-12-04T16:19:17Z", + | "userUpdated": "dummyUser", + | "disabled": false, + | "dateDisabled": null, + | "userDisabled": null, + | "parent": null, + | "filter": { + | "_t": "AndJoinedFilters", + | "filterItems": [ + | { + | "_t": "OrJoinedFilters", + | "filterItems": [ + | { + | "_t": "EqualsFilter", + | "columnName": "column1", + | "value": "soughtAfterValue", + | "valueType": "string" + | }, + | { + | "_t": "EqualsFilter", + | "columnName": "column1", + | "value": "alternativeSoughtAfterValue", + | "valueType": "string" + | } + | ] + | }, + | { + | "_t": "DiffersFilter", + | "columnName": "column2", + | "value": "anotherValue", + | "valueType": "string" + | }, + | { + | "_t": "NotFilter", + | "inputFilter": { + | "_t": "IsNullFilter", + | "columnName": "col3" + | } + | } + | ] + | }, + | "createdMessage": { + | "menasRef": { + | "collection": null, + | "name": "dummyName", + | "version": 1 + | }, + | "updatedBy": "dummyUser", + | "updated": "2017-12-04T16:19:17Z", + | "changes": [ + | { + | "field": "", + | "oldValue": null, + | "newValue": null, + | "message": "Mapping Table dummyName created." + | } + | ] + | }, + | "defaultMappingValues": {} + |} + |""".stripMargin + + val mappingTable = MappingTableFactory.getDummyMappingTable( + filter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter( + IsNullFilter("col3") + ) + )) + ) + ) + + "serializing" in { + val result = JsonSerializer.toJson(mappingTable) + result should equal(mappingTableJson)(after being whiteSpaceNormalised) + } + "deserializing" in { + val result = JsonSerializer.fromJson[MappingTable](mappingTableJson) + result should matchTo(mappingTable) + } + } + "handle Schemas" when { val schemaJson = """ diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/SchemaField.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/SchemaField.scala index 02c06063c..a878adae2 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/SchemaField.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/SchemaField.scala @@ -41,4 +41,9 @@ case class SchemaField def getAllChildren: Seq[String] = { children.flatMap(child => child.getAllChildren :+ child.getAbsolutePath) } + + @JsonIgnore + def getAllChildrenBasePath: Seq[String] = { + children.flatMap(child => child.getAllChildrenBasePath :+ child.path) + } } diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala new file mode 100644 index 000000000..fa933a5de --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.properties + +import za.co.absa.enceladus.model.properties.essentiality.Essentiality + +case class PropertyDefinitionStats(name: String, + version: Int = 1, + essentiality: Essentiality = Essentiality.Optional, + missingInDatasetsCount: Int = 0) + +object PropertyDefinitionStats { + def apply(propertyDefinition: PropertyDefinition, missingCounts: Int): PropertyDefinitionStats = { + PropertyDefinitionStats(propertyDefinition.name, propertyDefinition.version, + propertyDefinition.essentiality, missingCounts) + } +} diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/MappingTableFactory.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/MappingTableFactory.scala index d36bc1b9d..984afd357 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/MappingTableFactory.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/MappingTableFactory.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.model.test.factories import java.time.ZonedDateTime +import za.co.absa.enceladus.model.dataFrameFilter.DataFrameFilter import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.{DefaultValue, MappingTable, Schema} @@ -38,7 +39,8 @@ object MappingTableFactory extends EntityFactory[Schema] { disabled: Boolean = false, dateDisabled: Option[ZonedDateTime] = None, userDisabled: Option[String] = None, - parent: Option[MenasReference] = None): MappingTable = { + parent: Option[MenasReference] = None, + filter: Option[DataFrameFilter] = None): MappingTable = { MappingTable(name, version, @@ -54,7 +56,9 @@ object MappingTableFactory extends EntityFactory[Schema] { disabled, dateDisabled, userDisabled, - parent) + parent, + filter + ) } def getDummyDefaultValue(columnName: String = "dummyColumnName", diff --git a/data-model/src/test/scala/za/co/absa/enceladus/model/SchemaFieldTest.scala b/data-model/src/test/scala/za/co/absa/enceladus/model/SchemaFieldTest.scala index 400df960b..d67263976 100644 --- a/data-model/src/test/scala/za/co/absa/enceladus/model/SchemaFieldTest.scala +++ b/data-model/src/test/scala/za/co/absa/enceladus/model/SchemaFieldTest.scala @@ -21,7 +21,7 @@ class SchemaFieldTest extends AnyFunSuite { private val schemaFieldChildSecondLevel = SchemaField( name = "String", `type` = "string", - path = "", + path = "AnyStruct.AnyStruct2.String", elementType = None, containsNull = None, nullable = false, @@ -32,7 +32,7 @@ class SchemaFieldTest extends AnyFunSuite { private val schemaFieldChildOne = SchemaField( name = "AnyStruct2", `type` = "struct", - path = "", + path = "AnyStruct.AnyStruct2", elementType = None, containsNull = None, nullable = true, @@ -43,7 +43,7 @@ class SchemaFieldTest extends AnyFunSuite { private val schemaFieldChildTwo = SchemaField( name = "Number", `type` = "ling", - path = "AnyStruct", + path = "AnyStruct.Number", elementType = None, containsNull = None, nullable = true, @@ -54,7 +54,7 @@ class SchemaFieldTest extends AnyFunSuite { private val schemaFieldRoot = SchemaField( name = "AnyStruct", `type` = "struct", - path = "", + path = "AnyStruct", elementType = None, containsNull = None, nullable = true, @@ -63,8 +63,13 @@ class SchemaFieldTest extends AnyFunSuite { ) test("testGetAllChildren") { - val expectedAllChildren = List("String", "AnyStruct2", "AnyStruct.Number") + val expectedAllChildren = List("AnyStruct.AnyStruct2.String.String", "AnyStruct.AnyStruct2.AnyStruct2", "AnyStruct.Number.Number") assert(schemaFieldRoot.getAllChildren == expectedAllChildren) } + test("testGetAllChildrenBasePath") { + val expectedAllChildren = List("AnyStruct.AnyStruct2.String", "AnyStruct.AnyStruct2", "AnyStruct.Number") + assert(schemaFieldRoot.getAllChildrenBasePath == expectedAllChildren) + } + } diff --git a/examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample4.scala b/examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample4.scala index 2dd99b3f9..99de25fdd 100644 --- a/examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample4.scala +++ b/examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample4.scala @@ -57,7 +57,7 @@ object CustomRuleSample4 extends CustomRuleSampleFs { opt[String]("row-tag").optional.action((value, config) => config.copy(rowTag = Some(value))).text("use the specific row tag instead of 'ROW' for XML format") .validate(_ => - if (inputFormat.isDefined && inputFormat.get.equalsIgnoreCase("xml")) { + if (inputFormat.isDefined && inputFormat.get =="xml") { success } else { failure("The --row-tag option is supported only for XML raw data format") diff --git a/menas/pom.xml b/menas/pom.xml index c03691353..a83472bfc 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -34,8 +34,8 @@ ${dockerfile.menas.repository} ${dockerfile.tag} - ${dockerfile.base.artifact} - ${dockerfile.base.tag} + ${dockerfile.menas.repository} + ${dockerfile.tag} diff --git a/menas/ui/components/Component.js b/menas/ui/components/Component.js index a6f3278f2..c5ea51927 100644 --- a/menas/ui/components/Component.js +++ b/menas/ui/components/Component.js @@ -70,7 +70,12 @@ sap.ui.define([ name: "mappingTables", pattern: "mapping/:id:/:version:", target: "mappingTable" - } + }, + { + name: "properties", + pattern: "properties/:id:", + target: "property" + }, ], targets: { login: { @@ -102,6 +107,11 @@ sap.ui.define([ viewName: "components.mappingTable.mappingTableDetail", viewLevel: 1, viewId: "mappingTableDetailView" + }, + property: { + viewName: "components.property.datasetPropertyDetail", + viewLevel: 1, + viewId: "datasetPropertyDetailView" } } } diff --git a/menas/ui/components/app.controller.js b/menas/ui/components/app.controller.js index 67c4f7ec9..4f7949ec4 100644 --- a/menas/ui/components/app.controller.js +++ b/menas/ui/components/app.controller.js @@ -129,6 +129,11 @@ sap.ui.define([ this._app.toMaster(this.createId("mappingTablesPage")); }, + onPropertiesPress: function (oEv) { + this._eventBus.publish("properties", "list"); + this._app.toMaster(this.createId("propertiesPage")); + }, + onEntityCreated: function (sTopic, sEvent, oData) { this._router.navTo(sTopic, { id: oData.name, diff --git a/menas/ui/components/app.view.xml b/menas/ui/components/app.view.xml index 46419866c..fc779d087 100644 --- a/menas/ui/components/app.view.xml +++ b/menas/ui/components/app.view.xml @@ -25,6 +25,7 @@ +