From d9a72fc137c21dc6d3f1818a13dd3c20dd4f6987 Mon Sep 17 00:00:00 2001 From: Adrian Olosutean Date: Mon, 10 Jan 2022 12:28:51 +0000 Subject: [PATCH] Merge/merging release 2.26.X into develop ver3 (#1997) --- .gitignore | 2 - .../enceladus/model/ConformedSchema.scala | 46 +++++++ .../model/conformanceRule/package.scala | 4 +- .../enceladus/model/ConformedSchemaTest.scala | 80 ++++++++++++ .../conformanceRule/ConformanceRuleDialog.js | 6 +- .../conformanceRule/ConformanceRuleForm.js | 16 ++- .../MappingConformanceRule/add.fragment.xml | 1 + .../display.fragment.xml | 4 + .../joinConditions.fragment.xml | 2 +- .../outputColumns.fragment.xml | 1 - .../mappingTable/filterEdit/FilterEdit.js | 116 +++++++++-------- .../filterEdit/filterTreeEdit.fragment.xml | 6 +- menas/ui/service/EntityDialog.js | 2 +- menas/ui/service/FilterTreeUtils.js | 8 +- pom.xml | 1 + rest-api/pom.xml | 11 ++ .../rest_api/services/DatasetService.scala | 96 +++++++++----- .../services/VersionedModelService.scala | 4 +- .../services/DatasetServiceTest.scala | 122 +++++++++++++++++- scripts/bash/run_enceladus.sh | 16 +++ spark-jobs/pom.xml | 4 +- .../enceladus/common/CommonJobExecution.scala | 41 ++++-- .../conformance/ConformanceExecution.scala | 12 +- .../conformance/HyperConformance.scala | 8 +- .../HyperConformanceAttributes.scala | 2 +- .../conformance/config/FilterFromConfig.scala | 54 -------- .../interpreter/DynamicInterpreter.scala | 16 ++- .../CommonMappingRuleInterpreter.scala | 9 +- .../MappingRuleInterpreterBroadcast.scala | 2 +- .../MappingRuleInterpreterGroupExplode.scala | 2 +- .../streaming/InfoVersionFactory.scala | 19 ++- .../StandardizationExecution.scala | 8 +- .../StandardizationAndConformanceJob.scala | 1 + .../src/test/resources/application.conf | 6 - .../src/test/resources/data/empty/_SUCCESS | 0 .../common/CommonExecutionSuite.scala | 17 ++- .../config/FilterFromConfigSuite.scala | 61 --------- .../fixtures/StreamingFixture.scala | 11 +- .../interpreter/rules/RulesSuite.scala | 8 ++ .../SimpleTestCaseFactory.scala | 4 +- .../HyperConformanceIntegrationSuite.scala | 22 +++- .../conformed_literal_conf_info_col.json | 20 +++ .../utils/broadcast/LocalMappingTable.scala | 3 - .../utils/broadcast/BroadcastUtilsSuite.scala | 6 - .../broadcast/LocalMappingTableSuite.scala | 13 +- 45 files changed, 581 insertions(+), 312 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/ConformedSchema.scala create mode 100644 data-model/src/test/scala/za/co/absa/enceladus/model/ConformedSchemaTest.scala delete mode 100644 spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/config/FilterFromConfig.scala create mode 100644 spark-jobs/src/test/resources/data/empty/_SUCCESS delete mode 100644 spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/config/FilterFromConfigSuite.scala create mode 100644 spark-jobs/src/test/testData/nestedStructs/conformed_literal_conf_info_col.json diff --git a/.gitignore b/.gitignore index b217b745f..94ae1cf36 100644 --- a/.gitignore +++ b/.gitignore @@ -55,8 +55,6 @@ build.log # syntax: regexp # ^\.pc/ -build.log - .cache* dependency-reduced-pom.xml diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/ConformedSchema.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/ConformedSchema.scala new file mode 100644 index 000000000..f9fdd76a3 --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/ConformedSchema.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model + +import org.apache.spark.sql.types.StructField +import za.co.absa.enceladus.model.conformanceRule._ + +case class ConformedSchema(schema: List[StructField], dataset: Dataset) { + def hasField(field: String): Boolean = { + if (schema.exists(_.name == field)) true else { + val ss = dataset.conformance.find { + case MappingConformanceRule(_, _, _, _, _, _, outputColumn, additionalColumns, _, _, _) => + outputColumn == field || additionalColumns.getOrElse(Map()).contains(field) + case SingleColumnConformanceRule(_, _, outputColumn, _, inputColumnAlias) => + outputColumn == field || field == outputColumn + "." + inputColumnAlias + case DropConformanceRule(_, _, _) => false + case c: ConformanceRule => c.outputColumn == field + } + + ss match { + case None => false + case Some(matchedRule: ConformanceRule) => + val maybeRule = dataset.conformance.find { + case DropConformanceRule(_, _, outputCol) => outputCol == matchedRule.outputColumn + case _ => false + } + maybeRule.isEmpty + } + } + } +} + + diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/conformanceRule/package.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/conformanceRule/package.scala index 4415765d1..96185ebb1 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/conformanceRule/package.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/conformanceRule/package.scala @@ -88,9 +88,11 @@ package object conformanceRule { ) extends ConformanceRule { def allOutputColumns(): Map[String, String] = { - additionalColumns.getOrElse(Map()) + (outputColumn -> targetAttribute) + definedAdditionalColumns() + (outputColumn -> targetAttribute) } + def definedAdditionalColumns(): Map[String, String] = additionalColumns.getOrElse(Map()) + override def withUpdatedOrder(newOrder: Int): MappingConformanceRule = copy(order = newOrder) override def connectedEntities: Seq[ConnectedEntity] = Seq( diff --git a/data-model/src/test/scala/za/co/absa/enceladus/model/ConformedSchemaTest.scala b/data-model/src/test/scala/za/co/absa/enceladus/model/ConformedSchemaTest.scala new file mode 100644 index 000000000..b87a99664 --- /dev/null +++ b/data-model/src/test/scala/za/co/absa/enceladus/model/ConformedSchemaTest.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model + +import org.apache.spark.sql.types.{StringType, StructField} +import org.scalatest.funsuite.AnyFunSuite +import za.co.absa.enceladus.model.conformanceRule.{DropConformanceRule, LiteralConformanceRule, MappingConformanceRule, SingleColumnConformanceRule} + +class ConformedSchemaTest extends AnyFunSuite{ + private val conformanceRule1 = LiteralConformanceRule( + order = 0, + controlCheckpoint = true, + outputColumn = "something", + value = "1.01" + ) + + private val conformanceRule1d = LiteralConformanceRule( + order = 0, + controlCheckpoint = true, + outputColumn = "fieldToDelete", + value = "1.01" + ) + + private val conformanceRule2 = DropConformanceRule(order = 0, + controlCheckpoint = true, + outputColumn = "fieldToDelete") + + private val conformanceRule3 = MappingConformanceRule(order = 0, + controlCheckpoint = true, + outputColumn = "something3",additionalColumns = Some(Map("newCol" -> "mappedCol")), + mappingTable = "",mappingTableVersion = 1, + attributeMappings = Map(),targetAttribute = "col") + + private val conformanceRule4 = SingleColumnConformanceRule( + order = 0, + outputColumn = "singleCol",inputColumn = "as", + inputColumnAlias = "subCol", controlCheckpoint = false) + + private val dataset = Dataset(name = "Test DS", + version = 1, + hdfsPath = "newPath", + hdfsPublishPath = "newPublishPath", + schemaName = "newSchema", + schemaVersion = 1, + conformance = List(conformanceRule1, conformanceRule1d, conformanceRule2, conformanceRule3, conformanceRule4), + properties = Some(Map( + "property1" -> "value1", + "property2.sub" -> "value2" + ) + )) + + val schemaFields = List(StructField("stdField",StringType)) + + test("conformed schema") { + val conformedSchema = ConformedSchema(schemaFields, dataset) + assertResult(conformedSchema.hasField("stdField"))(true) + assertResult(conformedSchema.hasField("fieldToDelete"))(false) + assertResult(conformedSchema.hasField("something"))(true) + assertResult(conformedSchema.hasField("newCol"))(true) + assertResult(conformedSchema.hasField("newCol1"))(false) + assertResult(conformedSchema.hasField("mappedColCol1"))(false) + assertResult(conformedSchema.hasField("something3"))(true) + assertResult(conformedSchema.hasField("col"))(false) + assertResult(conformedSchema.hasField("singleCol"))(true) + assertResult(conformedSchema.hasField("singleCol.subCol"))(true) + } +} diff --git a/menas/ui/components/dataset/conformanceRule/ConformanceRuleDialog.js b/menas/ui/components/dataset/conformanceRule/ConformanceRuleDialog.js index 9d640688a..470a4ec42 100644 --- a/menas/ui/components/dataset/conformanceRule/ConformanceRuleDialog.js +++ b/menas/ui/components/dataset/conformanceRule/ConformanceRuleDialog.js @@ -109,10 +109,10 @@ class ConformanceRuleDialog { this.beforeSubmitChanges(newRule); this.resetRuleValidation(); - newRule.hasValidFilter = true; // default for non-MappingConformanceRules + newRule.filterValidations = {empty: true, valid: true}; // default for non-MappingConformanceRules if (newRule._t === "MappingConformanceRule") { - const validFilter = this.filterEdit.validateFilterData(); - newRule.hasValidFilter = validFilter; + const filterValidations = this.filterEdit.validateFilterData(); + newRule.filterValidations = filterValidations; } if (this.ruleForms.byType(newRule._t).isValid(newRule, this.controller._transitiveSchemas, currentDataset.conformance)) { diff --git a/menas/ui/components/dataset/conformanceRule/ConformanceRuleForm.js b/menas/ui/components/dataset/conformanceRule/ConformanceRuleForm.js index b27121f4a..560b462bb 100644 --- a/menas/ui/components/dataset/conformanceRule/ConformanceRuleForm.js +++ b/menas/ui/components/dataset/conformanceRule/ConformanceRuleForm.js @@ -369,18 +369,20 @@ class MappingConformanceRuleForm extends ConformanceRuleForm { isCorrectlyConfigured(rule) { return this.hasValidInputColumn(rule.targetAttribute) & this.hasValidOutputColumns(rule) - & this.hasValidJoinConditions(rule.newJoinConditions) - & rule.hasValidFilter; + & this.hasValidJoinConditions(rule.newJoinConditions, rule.filterValidations.empty) + & rule.filterValidations.valid; } - hasValidJoinConditions(fieldValue = []) { - let isValid = fieldValue.length >= 1; + hasValidJoinConditions(fieldValue = [], filtersEmpty) { + const nonEmptyJoinConditions = fieldValue.length >= 1; + const validJoinConditions = (nonEmptyJoinConditions || !filtersEmpty); - if (!isValid) { - sap.m.MessageToast.show("At least 1 join condition is required."); + + if (!validJoinConditions) { + sap.m.MessageToast.show("Either provide a join condition or a filter!"); } - return isValid + return validJoinConditions; } hasValidOutputColumns(rule = []) { diff --git a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/add.fragment.xml b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/add.fragment.xml index 8ce064040..86b059ae8 100644 --- a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/add.fragment.xml +++ b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/add.fragment.xml @@ -14,6 +14,7 @@ --> + diff --git a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/display.fragment.xml b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/display.fragment.xml index a7c9a9675..1d479d53b 100644 --- a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/display.fragment.xml +++ b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/display.fragment.xml @@ -18,6 +18,10 @@ + + + <core:Fragment type="XML" fragmentName="components.dataset.conformanceRule.add.checkpointBox"/> + <Label text="Mapping Table"/> <Link text="{mappingTable} (v{mappingTableVersion})" press="toMappingTable" cust:name="{mappingTable}" cust:version="{mappingTableVersion}"/> diff --git a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/joinConditions.fragment.xml b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/joinConditions.fragment.xml index 1b834a3eb..67bb9c002 100644 --- a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/joinConditions.fragment.xml +++ b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/joinConditions.fragment.xml @@ -21,7 +21,7 @@ <Label text="Join Conditions"/> <List id="joinConditions" items="{/newRule/newJoinConditions}" - noDataText="No join conditions (1 minimum)" + noDataText="No join conditions" mode="Delete" delete=".onDeleteJoinCondition"> <items> diff --git a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/outputColumns.fragment.xml b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/outputColumns.fragment.xml index aa1f62c0b..f7b6897b1 100644 --- a/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/outputColumns.fragment.xml +++ b/menas/ui/components/dataset/conformanceRule/MappingConformanceRule/outputColumns.fragment.xml @@ -16,7 +16,6 @@ <core:FragmentDefinition xmlns="sap.m" xmlns:core="sap.ui.core" xmlns:cust="http://schemas.sap.com/sapui5/extension/sap.ui.core.CustomData/1"> - <CheckBox selected="{/newRule/isNullSafe}"/> <Label text="Output Columns"/> <List id="outputColumns" items="{/newRule/newOutputColumns}" diff --git a/menas/ui/components/mappingTable/filterEdit/FilterEdit.js b/menas/ui/components/mappingTable/filterEdit/FilterEdit.js index 0599f693c..1eff3147c 100644 --- a/menas/ui/components/mappingTable/filterEdit/FilterEdit.js +++ b/menas/ui/components/mappingTable/filterEdit/FilterEdit.js @@ -198,72 +198,78 @@ class FilterEdit { } /** - * Validates data in the filter TreeTable, sets their valueState|valueStateText (error+error descs) - * @returns {boolean} returns true if the filter content is valid, false when invalid + * Validates data and emptiness in the filter TreeTable, sets their valueState|valueStateText (error+error descs) + * @returns {empty: boolean, valid: boolean} */ validateFilterData() { const treeTable = this.#getById("filterTreeEdit"); const treeTableModel = treeTable.getBinding().getModel(); const filterData = treeTableModel.getProperty("/editingFilters"); - let hasValidFilter = true; // filter data can be [filter], [null] or null if (!filterData || filterData.filter(x => x).length == 0) { - hasValidFilter = true; - } else { - // validate filter tree - const validateInUiFn = function (filterNode) { - switch (filterNode._t) { - case "AndJoinedFilters": - case "OrJoinedFilters": - if (filterNode.filterItems.filter(x => x).length == 0) { // empty deleted ([null]) is not valid - filterNode.filter_valueState = "Error"; - filterNode.filter_valueStateText = "Container filter must contain child filters!"; - hasValidFilter = false; - } - break; - - case "NotFilter": - if (!filterNode.inputFilter) { - filterNode.filter_valueState = "Error"; - filterNode.filter_valueStateText = "Container filter must contain a child filter!"; - hasValidFilter = false; - } - break; - - case "EqualsFilter": - case "DiffersFilter": - if (filterNode.columnName.length == 0) { - filterNode.columnName_valueState = "Error"; - filterNode.columnName_valueStateText = "Select the column."; - hasValidFilter = false; - } - - if (filterNode.value.length == 0) { - filterNode.value_valueState = "Error"; - filterNode.value_valueStateText = "Fill in the value."; - hasValidFilter = false; - } - break; - - case "IsNullFilter": - if (filterNode.columnName.length == 0) { - filterNode.columnName_valueState = "Error"; - filterNode.columnName_valueStateText = "Fill in column name."; - hasValidFilter = false; - } - break; - - default: - } + return { + empty: true, + valid: true }; - - const validatedFilter = FilterTreeUtils.applyToFilterDataImmutably(filterData[0], validateInUiFn); - treeTableModel.setProperty("/editingFilters", [validatedFilter]); - treeTableModel.refresh(); } - return hasValidFilter; + // nonempty filter: validate filter tree + let filterValid = true; + const validateInUiFn = function (filterNode) { + switch (filterNode._t) { + case "AndJoinedFilters": + case "OrJoinedFilters": + if (filterNode.filterItems.filter(x => x).length == 0) { // empty deleted ([null]) is not valid + filterNode.filter_valueState = "Error"; + filterNode.filter_valueStateText = "Container filter must contain child filters!"; + filterValid = false; + } + break; + + case "NotFilter": + if (!filterNode.inputFilter) { + filterNode.filter_valueState = "Error"; + filterNode.filter_valueStateText = "Container filter must contain a child filter!"; + filterValid = false; + } + break; + + case "EqualsFilter": + case "DiffersFilter": + if (filterNode.columnName.length == 0) { + filterNode.columnName_valueState = "Error"; + filterNode.columnName_valueStateText = "Select the column."; + filterValid = false; + } + + if (filterNode.value.length == 0) { + filterNode.value_valueState = "Error"; + filterNode.value_valueStateText = "Fill in the value."; + filterValid = false; + } + break; + + case "IsNullFilter": + if (filterNode.columnName.length == 0) { + filterNode.columnName_valueState = "Error"; + filterNode.columnName_valueStateText = "Fill in column name."; + filterValid = false; + } + break; + + default: + } + }; + + const validatedFilter = FilterTreeUtils.applyToFilterDataImmutably(filterData[0], validateInUiFn); + treeTableModel.setProperty("/editingFilters", [validatedFilter]); + treeTableModel.refresh(); + + return { + empty: false, + valid: filterValid + }; } /** diff --git a/menas/ui/components/mappingTable/filterEdit/filterTreeEdit.fragment.xml b/menas/ui/components/mappingTable/filterEdit/filterTreeEdit.fragment.xml index 107b2e7ca..8928bc3c6 100644 --- a/menas/ui/components/mappingTable/filterEdit/filterTreeEdit.fragment.xml +++ b/menas/ui/components/mappingTable/filterEdit/filterTreeEdit.fragment.xml @@ -28,9 +28,9 @@ <Menu> <MenuItem id="addAndBtn" text="AND" icon="sap-icon://combine" /> <MenuItem id="addOrBtn" text="OR" icon="sap-icon://split" /> - <MenuItem id="addNotBtn" text="NOT" icon="sap-icon://filter" /> - <MenuItem id="addEqualsBtn" text="Equals" icon="sap-icon://clear-filter" /> - <MenuItem id="addDiffersBtn" text="Differs" icon="sap-icon://SAP-icons-TNT/solution-not-licensed" /> + <MenuItem id="addNotBtn" text="NOT" icon="sap-icon://SAP-icons-TNT/solution-not-licensed" /> + <MenuItem id="addEqualsBtn" text="Equals" icon="sap-icon://filter" /> + <MenuItem id="addDiffersBtn" text="Differs" icon="sap-icon://clear-filter" /> <MenuItem id="addIsNullBtn" text="IsNull" icon="sap-icon://SAP-icons-TNT/marquee" /> </Menu> </menu> diff --git a/menas/ui/service/EntityDialog.js b/menas/ui/service/EntityDialog.js index 53a6fd698..352805f33 100644 --- a/menas/ui/service/EntityDialog.js +++ b/menas/ui/service/EntityDialog.js @@ -453,7 +453,7 @@ class MappingTableDialog extends EntityDialog { this.oController.byId("newMappingTableName")); let hasValidSchema = EntityValidationService.hasValidSchema(oMT, "Mapping Table", this.oController.byId("schemaVersionSelect")); - let hasValidFilter = this.filterEdit.validateFilterData(); + let hasValidFilter = this.filterEdit.validateFilterData().valid; // validity flag suffices, emptiness irrelevant if (oMT.hdfsBrowserEnabled) { let hasValidHDFSPath = EntityValidationService.hasValidHDFSPath(oMT.hdfsPath, diff --git a/menas/ui/service/FilterTreeUtils.js b/menas/ui/service/FilterTreeUtils.js index 371c824ab..92a2e80c9 100644 --- a/menas/ui/service/FilterTreeUtils.js +++ b/menas/ui/service/FilterTreeUtils.js @@ -61,6 +61,10 @@ class FilterTreeUtils { filterNode.text = "OR"; filterNode.icon = "sap-icon://split"; break; + case "NotFilter": + filterNode.text = "NOT"; + filterNode.icon = "sap-icon://SAP-icons-TNT/solution-not-licensed"; + break; case "EqualsFilter": filterNode.text = `Value of "${filterNode.columnName}" equals to "${filterNode.value}" (of type ${filterNode.valueType})`; filterNode.icon = "sap-icon://filter"; @@ -69,10 +73,6 @@ class FilterTreeUtils { filterNode.text = `Value of "${filterNode.columnName}" differs from "${filterNode.value}" (of type ${filterNode.valueType})`; filterNode.icon = "sap-icon://clear-filter"; break; - case "NotFilter": - filterNode.text = "NOT"; - filterNode.icon = "sap-icon://SAP-icons-TNT/solution-not-licensed"; - break; case "IsNullFilter": filterNode.text = `Value of "${filterNode.columnName}" is null`; filterNode.icon = "sap-icon://SAP-icons-TNT/marquee"; diff --git a/pom.xml b/pom.xml index 893fd445b..ca56bcd31 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,7 @@ <junit.version>4.11</junit.version> <kafka.spark.version>0-10</kafka.spark.version> <lodash.version>4.17.10</lodash.version> + <log4j.version>2.15.0</log4j.version> <mockito.core.version>3.5.2</mockito.core.version> <mockito.scala.version>1.16.42</mockito.scala.version> <momentjs.version>2.22.2</momentjs.version> diff --git a/rest-api/pom.xml b/rest-api/pom.xml index de5f84567..1f1a22c53 100644 --- a/rest-api/pom.xml +++ b/rest-api/pom.xml @@ -123,6 +123,17 @@ <version>${bson.codec.jsr310.version}</version> </dependency> <!-- SPRING --> + <!-- ensuring that log4j used us at least 2.15.0--> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${log4j.version}</version> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-to-slf4j</artifactId> + <version>${log4j.version}</version> + </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index ee70baedb..368ffe6bb 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -272,29 +272,29 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository val ruleValidationsAndFields = conformanceRules.foldLeft(accumulator) { case (validationsAndFields, conformanceRule) => conformanceRule match { case cr: CastingConformanceRule => - validationsAndFields.update(validateInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateInAndOut(validationsAndFields.fields, cr)) case cr: NegationConformanceRule => - validationsAndFields.update(validateInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateInAndOut(validationsAndFields.fields, cr)) case cr: UppercaseConformanceRule => - validationsAndFields.update(validateInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateInAndOut(validationsAndFields.fields, cr)) case cr: SingleColumnConformanceRule => - validationsAndFields.update(validateInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateInAndOut(validationsAndFields.fields, cr)) case cr: FillNullsConformanceRule => - validationsAndFields.update(validateInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateInAndOut(validationsAndFields.fields, cr)) case cr: ConcatenationConformanceRule => - validationsAndFields.update(validateMultipleInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateMultipleInAndOut(validationsAndFields.fields, cr)) case cr: CoalesceConformanceRule => - validationsAndFields.update(validateMultipleInAndOut(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateMultipleInAndOut(validationsAndFields.fields, cr)) case cr: LiteralConformanceRule => - validationsAndFields.update(validateOutputColumn(validationsAndFields.fields, cr.outputColumn)) + validationsAndFields.updateWithFieldsReplace(validateOutputColumn(validationsAndFields.fields, cr.outputColumn)) case cr: SparkSessionConfConformanceRule => - validationsAndFields.update(validateOutputColumn(validationsAndFields.fields, cr.outputColumn)) + validationsAndFields.updateWithFieldsReplace(validateOutputColumn(validationsAndFields.fields, cr.outputColumn)) case cr: DropConformanceRule => - validationsAndFields.update(validateDrop(validationsAndFields.fields, cr.outputColumn)) + validationsAndFields.updateWithFieldsReplace(validateDrop(validationsAndFields.fields, cr.outputColumn)) case cr: MappingConformanceRule => - validationsAndFields.update(validateMappingTable(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(validateMappingTable(validationsAndFields.fields, cr)) case cr => - validationsAndFields.update(unknownRule(validationsAndFields.fields, cr)) + validationsAndFields.updateWithFieldsReplace(unknownRule(validationsAndFields.fields, cr)) } } @@ -304,7 +304,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository private def validateDrop(currentColumns: Future[Set[String]], output: String): RuleValidationsAndFields = { validateInputColumn(currentColumns, output) - .update(currentColumns.map(f => f - output)) + .updateFields(currentColumns.map(f => f - output)) } private type WithInAndOut = {def inputColumn: String; def outputColumn: String} @@ -314,19 +314,40 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository cr: C): RuleValidationsAndFields = { val withOutputValidated = validateOutputColumn(fields, cr.outputColumn) val validationInputFields = validateInputColumn(fields, cr.inputColumn) - validationInputFields.update(withOutputValidated) + validationInputFields.updateWithFieldsReplace(withOutputValidated) } def validateMappingTable(fields: Future[Set[String]], - mt: MappingConformanceRule): RuleValidationsAndFields = { - val inputValidation = mt.attributeMappings.values.map { input => - validateInputColumn(fields, input) - } - val outputValidation = validateOutputColumn(fields, mt.outputColumn) + mtRule: MappingConformanceRule): RuleValidationsAndFields = { + val inputValidation = mtRule.attributeMappings.values.map(validateInputColumn(fields, _)) + val allOutput = mtRule.allOutputColumns() + val outputColumns = mtRule.allOutputColumns().keySet + + val mtFields = for { + someMappingTable <- datasetMongoRepository.getConnectedMappingTable(mtRule.mappingTable, mtRule.mappingTableVersion) + mtSchema: Option[Schema] <- someMappingTable.map(mt => datasetMongoRepository.getConnectedSchema(mt.schemaName, mt.schemaVersion)).get + result = mtSchema.map(_. + fields.flatMap(f => f.getAllChildrenBasePath :+ f.path).toSet + ).getOrElse(Set.empty) + } yield result + + val inputsValidated = inputValidation + .foldLeft(RuleValidationsAndFields(Seq.empty, fields))((acc, instance) => acc.updateWithFieldsReplace(instance)) + + val validatedOutputCols = outputColumns.foldLeft(inputsValidated)((acc, outputCol: String) => { + val updated: RuleValidationsAndFields = validateOutputColumn(acc.fields, outputCol) + acc.updateWithFieldsReplace(updated) + }) + + val outputColsFlat: Future[Set[String]] = for { + fieldsFromMT <- mtFields + oldFields <- validatedOutputCols.fields + newFields = allOutput.flatMap { case (out, in) => + DatasetService.replacePrefixIfFound(fieldsFromMT, out, in) + } + } yield oldFields ++ newFields - inputValidation - .foldLeft(RuleValidationsAndFields(Seq.empty, fields))((acc, instance) => acc.update(instance)) - .update(outputValidation) + validatedOutputCols.updateFields(outputColsFlat) } private def validateMultipleInAndOut[C <: WithMultipleInAndOut](fields: Future[Set[String]], @@ -337,18 +358,16 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository val outputValidation = validateOutputColumn(fields, cr.outputColumn) inputValidation - .foldLeft(RuleValidationsAndFields(Seq.empty, fields))((acc, instance) => acc.update(instance)) - .update(outputValidation) + .foldLeft(RuleValidationsAndFields(Seq.empty, fields))((acc, instance) => acc.updateWithFieldsReplace(instance)) + .updateWithFieldsReplace(outputValidation) } private def validateInputColumn(fields: Future[Set[String]], input: String): RuleValidationsAndFields = { - val validation = Validation() - val newValidation = for { f <- fields } yield { - validation.withErrorIf( + Validation().withErrorIf( !f.contains(input), "item.conformanceRules", s"Input column $input for conformance rule cannot be found" @@ -359,12 +378,10 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository private def validateOutputColumn(fields: Future[Set[String]], output: String): RuleValidationsAndFields = { - val validation = Validation() - val newValidation = for { f <- fields } yield { - validation.withErrorIf( + Validation().withErrorIf( f.contains(output), "item.conformanceRules", s"Output column $output already exists" @@ -392,12 +409,13 @@ object DatasetService { // Local class for the representation of validation of conformance rules. final case class RuleValidationsAndFields(validations: Seq[Future[Validation]], fields: Future[Set[String]]) { - def update(ruleValidationsAndFields: RuleValidationsAndFields): RuleValidationsAndFields = copy( + def updateWithFieldsReplace(ruleValidationsAndFields: RuleValidationsAndFields): RuleValidationsAndFields = copy( validations = validations ++ ruleValidationsAndFields.validations, fields = ruleValidationsAndFields.fields ) - def update(fields: Future[Set[String]]): RuleValidationsAndFields = copy(fields = fields) + def updateFields(fields: Future[Set[String]]): RuleValidationsAndFields = copy(fields = fields) + def appendValidations(v: Seq[Future[Validation]]): RuleValidationsAndFields = copy(validations = validations ++ v) def mergeValidations(): Future[Validation] = Future.fold(validations)(Validation())((v1, v2) => v1.merge(v2)) } @@ -413,4 +431,18 @@ object DatasetService { _.filter { case (_, propValue) => propValue.nonEmpty } } } + + private[services] def replacePrefixIfFound(fieldName: String, replacement: String, lookFor: String): Option[String] = { + fieldName match { + case `lookFor` => Some(replacement) // exact match + case field if field.startsWith(s"$lookFor.") => + val strippedField = field.stripPrefix(s"$lookFor.") + Some(s"$replacement.$strippedField") + case _ => None + } + } + + private[services] def replacePrefixIfFound(fieldNames: Iterable[String], replacement: String, lookFor: String): Iterable[String] = { + fieldNames.flatMap(replacePrefixIfFound(_, replacement, lookFor)) // Nones discarded, Some's lifted + } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index 8827694b0..b11fae9d0 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -127,11 +127,9 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit private[services] def validateSchema(schemaName: String, schemaVersion: Int, maybeSchema: Future[Option[Schema]]): Future[Validation] = { - val validation = Validation() - for { schema <- maybeSchema - } yield validation.withErrorIf( + } yield Validation().withErrorIf( schema.isEmpty, "item.schema", s"schema $schemaName v$schemaVersion defined for the dataset could not be found" diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala index 5d163f20a..48914328b 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/services/DatasetServiceTest.scala @@ -21,11 +21,11 @@ import org.mongodb.scala.bson.BsonDocument import org.scalatest.matchers.should.Matchers import za.co.absa.enceladus.rest_api.exceptions.ValidationException import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} -import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.model.{Dataset, Schema, SchemaField, Validation} import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} -import za.co.absa.enceladus.model.test.factories.DatasetFactory +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, SchemaFactory} import za.co.absa.enceladus.utils.validation.ValidationLevel import scala.concurrent.Future @@ -59,6 +59,8 @@ class DatasetServiceTest extends VersionedModelServiceTest[Dataset] with Matcher Mockito.when(modelRepository.getLatestVersionValue("dataset")).thenReturn(Future.successful(Some(1))) Mockito.when(modelRepository.isUniqueName("dataset")).thenReturn(Future.successful(true)) Mockito.when(modelRepository.update(eqTo("user"), any[Dataset])).thenReturn(Future.failed(writeException)) + Mockito.when(modelRepository.getConnectedMappingTable("dummyMappingTable", 1)).thenReturn(Future.successful(Some(MappingTableFactory.getDummyMappingTable()))) + Mockito.when(modelRepository.getConnectedSchema("dummySchema", 1)).thenReturn(Future.successful(Some(SchemaFactory.getDummySchema()))) val result = intercept[ValidationException] { await(service.update("user", dataset)) @@ -145,6 +147,101 @@ class DatasetServiceTest extends VersionedModelServiceTest[Dataset] with Matcher validationResult shouldBe expectedValidationResultForSetup } + test("Validate mapping table with valid fields") { + val initialSet = Future.successful(Set("dummyValue")) + val validationResultFut = service.validateMappingTable(initialSet, DatasetFactory.getDummyMappingRule()) + + val validationResult: Validation = await(validationResultFut.mergeValidations()) + + assertResult(Validation(Map(), Map()))(validationResult) + assertResult(Set("dummyValue", "dummyOutputCol"))(await(validationResultFut.fields)) + } + + test("Validate mapping table with valid fields - additional fields") { + val initialSet = Future.successful(Set("dummyValue")) + val validationResultFut = service.validateMappingTable(initialSet, + DatasetFactory.getDummyMappingRule(additionalOutputs = Some(Map("a"->"abc", "b"->"cc")))) + + val validationsResult: Validation = await(validationResultFut.mergeValidations()) + + assertResult(Validation(Map(), Map()))(validationsResult) + assertResult(Set("dummyValue", "dummyOutputCol", "a", "b"))(await(validationResultFut.fields)) + } + + test("Validate mapping table with valid fields - structs") { + val initialSet = Future.successful(Set( + "Byte", + "SomeBox", + "SomeBox.boxedVal", + "RegularField1" + )) + + Mockito.when(modelRepository.getConnectedMappingTable("SourceSystemMappingTable", 1)).thenReturn(Future.successful( + Some(MappingTableFactory.getDummyMappingTable(name = "SourceSystemMappingTable", schemaName = "SourceSystemMappingSchema")) + )) + Mockito.when(modelRepository.getConnectedSchema("SourceSystemMappingSchema", 1)) + .thenReturn(Future.successful(Some( + Schema("SourceSystemMappingSchema", description = None, userCreated = "user", userUpdated = "user", fields = List( + SchemaField("RawFeedName", "string", "RawFeedName", nullable = true, metadata = Map(), children = List()), + SchemaField("SourceSystem", "struct", "SourceSystem", nullable = true, metadata = Map(), children = List( + SchemaField("Description", "string", "SourceSystem.Description", nullable = true, metadata = Map(), children = List()), + SchemaField("Details", "struct", "SourceSystem.Details", nullable = true, metadata = Map(), children = List( + SchemaField("Stable", "boolean", "SourceSystem.Details.Stable", nullable = true, metadata = Map(), children = List()), + SchemaField("Version", "integer", "SourceSystem.Details.Version", nullable = true, metadata = Map(), children = List()) + )) + )) + )) + ))) + + val mCr = DatasetFactory.getDummyMappingRule( + mappingTable = "SourceSystemMappingTable", + attributeMappings = Map("RawFeedName" -> "Byte"), // "Byte" must exists in input, because it is joined on + targetAttribute = "SourceSystem", + outputColumn = "Alfa" + ) + + val validationResultFut = service.validateMappingTable(initialSet, mCr) + + val validationsResult: Validation = await(validationResultFut.mergeValidations()) + assertResult(Validation(Map(), Map()))(validationsResult) // no errors, no warnings + + val expectedFields = Set( + "Alfa", // these fields are included from MT schema with root renamed as per mCr definition + "Alfa.Description", + "Alfa.Details", + "Alfa.Details.Stable", + "Alfa.Details.Version", + "Byte", + "SomeBox", + "SomeBox.boxedVal", + "RegularField1" + ) + + assertResult(expectedFields)(await(validationResultFut.fields)) + } + + test("Validate mapping table with invalid input field") { + val existingIncompleteSet = Future.successful(Set("first", "second")) + + val validationResult = service.validateMappingTable(existingIncompleteSet, + DatasetFactory.getDummyMappingRule(additionalOutputs = Some(Map("a"->"abc", "b"->"cc")))) + + assertResult(Validation(Map("item.conformanceRules" -> + List("Input column dummyValue for conformance rule cannot be found")), Map()) + )(await(validationResult.mergeValidations())) + assertResult(Set("a", "b", "first", "second", "dummyOutputCol"))(await(validationResult.fields)) + } + + test("Validate mapping table with invalid output field") { + val existingCompleteSet = Future.successful(Set("dummyValue", "first", "second")) + val validationResult4 = service.validateMappingTable(existingCompleteSet, + DatasetFactory.getDummyMappingRule(additionalOutputs = Some(Map("a"->"abc", "b"->"cc", "first"->"there")))) + assertResult(Validation(Map("item.conformanceRules" -> List("Output column first already exists")), Map()) + )(await(validationResult4.mergeValidations())) + assertResult(Set("a", "b", "first", "second", "dummyValue", "dummyOutputCol") + )(await(validationResult4.fields)) + } + val dataset = DatasetFactory.getDummyDataset(name = "dataset", version = 1, properties = Some(datasetProperties)) Seq( ("validation for run", ValidationLevel.ForRun, Some(dataset), Some(dataset.copy(propertiesValidation = Some(expectedValidationResultForRun)))), @@ -211,4 +308,25 @@ class DatasetServiceTest extends VersionedModelServiceTest[Dataset] with Matcher DatasetService.removeBlankProperties(dataset.properties) shouldBe Some(Map("propKey1" -> "someValue")) } + test("DatasetService.replacePrefixIfFound replaces field prefixes") { + DatasetService.replacePrefixIfFound("Alfa", "Beta", "Alfa") shouldBe Some("Beta") + DatasetService.replacePrefixIfFound("Omega", "Beta", "Alfa") shouldBe None + + DatasetService.replacePrefixIfFound("Alfa.abc.def", "Beta", "Alfa") shouldBe Some("Beta.abc.def") + // not a .-separated prefix: + DatasetService.replacePrefixIfFound("Alfaville.there", "Beta", "Alfa") shouldBe None + // not a prefix + DatasetService.replacePrefixIfFound("some.Alfa.other", "Beta", "Alfa") shouldBe None + + // all at once in an iterable: + DatasetService.replacePrefixIfFound(Seq( + "Alfa", "Omega", + "Alfa.abc.def", "Alfaville", "Alfaville.there" + ), "Beta", "Alfa") shouldBe Seq( + "Beta", + "Beta.abc.def" + ) + + } + } diff --git a/scripts/bash/run_enceladus.sh b/scripts/bash/run_enceladus.sh index c7c0df1ba..cbe6364ea 100644 --- a/scripts/bash/run_enceladus.sh +++ b/scripts/bash/run_enceladus.sh @@ -412,6 +412,12 @@ get_temp_log_file() { mktemp -p "$LOG_DIR" -t "$TEMPLATE" } +add_keytab_to_files() { + MENAS_AUTH_KEYTAB_NAME=`echo "${MENAS_AUTH_KEYTAB}" | grep -o '[^/]*$'` + FILES="${FILES},${MENAS_AUTH_KEYTAB}#${MENAS_AUTH_KEYTAB_NAME}" + MENAS_AUTH_KEYTAB="${MENAS_AUTH_KEYTAB_NAME}" +} + CMD_LINE="$SPARK_SUBMIT" # Constructing the grand command line @@ -480,6 +486,16 @@ if [ "$HELP_CALL" == "1" ]; then exit "$?" fi +if [[ "${MENAS_AUTH_KEYTAB}" =~ "^(s|S)3://.*" ]]; then + echo "Using Keytab from S3" + add_keytab_to_files +elif [[ -f "${MENAS_AUTH_KEYTAB}" ]]; then + echo "Using Keytab from local FS" + add_keytab_to_files +else + echo "Using Keytab from HDFS" +fi + # Adding command line parameters that go BEFORE the jar file add_to_cmd_line "--master" "${MASTER}" add_to_cmd_line "--deploy-mode" "${DEPLOY_MODE}" diff --git a/spark-jobs/pom.xml b/spark-jobs/pom.xml index 62d1ca0ed..00d22f50b 100644 --- a/spark-jobs/pom.xml +++ b/spark-jobs/pom.xml @@ -26,7 +26,7 @@ <properties> <scalastyle.configLocation>${project.parent.basedir}/scalastyle-config.xml</scalastyle.configLocation> - <hyperdrive.version>4.0.0</hyperdrive.version> + <hyperdrive.version>4.5.2</hyperdrive.version> <spark.sql.kafka.version>2.4.5</spark.sql.kafka.version> </properties> @@ -75,7 +75,7 @@ <!-- Hyperdrive --> <dependency> <groupId>za.co.absa.hyperdrive</groupId> - <artifactId>api</artifactId> + <artifactId>api_${scala.compat.version}</artifactId> <version>${hyperdrive.version}</version> <scope>provided</scope> </dependency> diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala index db0179da2..4246aff1f 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala @@ -21,11 +21,15 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.functions.{lit, to_date} +import org.apache.spark.sql.{DataFrame, SparkSession} import org.slf4j.{Logger, LoggerFactory} import za.co.absa.atum.AtumImplicits._ import za.co.absa.atum.core.{Atum, ControlType} import za.co.absa.enceladus.common.Constants.{InfoDateColumn, InfoVersionColumn} import za.co.absa.enceladus.common.config.{CommonConfConstants, JobConfigParser, PathConfig} +import za.co.absa.enceladus.common.Constants.{InfoDateColumn, InfoDateColumnString, InfoVersionColumn, ReportDateFormat} +import za.co.absa.enceladus.common.config.{CommonConfConstants, JobConfigParser, PathConfig} import za.co.absa.enceladus.common.plugin.PostProcessingService import za.co.absa.enceladus.common.plugin.menas.{MenasPlugin, MenasRunUrl} import za.co.absa.enceladus.common.version.SparkVersionGuard @@ -195,19 +199,26 @@ trait CommonJobExecution extends ProjectMetadata { outputDf } - val catalystPlan = df.queryExecution.logical - val sizeInBytes = spark.sessionState.executePlan(catalystPlan).optimizedPlan.stats.sizeInBytes + val currentPartionCount = df.rdd.getNumPartitions - val currentBlockSize = sizeInBytes / df.rdd.getNumPartitions + if (currentPartionCount > 0) { + val catalystPlan = df.queryExecution.logical + val sizeInBytes = spark.sessionState.executePlan(catalystPlan).optimizedPlan.stats.sizeInBytes - (minBlockSize, maxBlockSize) match { - case (Some(min), None) if currentBlockSize < min => - changePartitionCount(computeBlockCount(min, sizeInBytes, addRemainder = false), df.coalesce) - case (None, Some(max)) if currentBlockSize > max => - changePartitionCount(computeBlockCount(max, sizeInBytes, addRemainder = true), df.repartition) - case (Some(min), Some(max)) if currentBlockSize < min || currentBlockSize > max => - changePartitionCount(computeBlockCount(max, sizeInBytes, addRemainder = true), df.repartition) - case _ => df + val currentBlockSize = sizeInBytes / df.rdd.getNumPartitions + + (minBlockSize, maxBlockSize) match { + case (Some(min), None) if currentBlockSize < min => + changePartitionCount(computeBlockCount(min, sizeInBytes, addRemainder = false), df.coalesce) + case (None, Some(max)) if currentBlockSize > max => + changePartitionCount(computeBlockCount(max, sizeInBytes, addRemainder = true), df.repartition) + case (Some(min), Some(max)) if currentBlockSize < min || currentBlockSize > max => + changePartitionCount(computeBlockCount(max, sizeInBytes, addRemainder = true), df.repartition) + case _ => df + } + } else { + // empty dataframe + df } } @@ -314,6 +325,14 @@ trait CommonJobExecution extends ProjectMetadata { } } + protected def addInfoColumns(intoDf: DataFrame, reportDate: String, reportVersion: Int): DataFrame = { + import za.co.absa.enceladus.utils.implicits.DataFrameImplicits.DataFrameEnhancements + intoDf + .withColumnIfDoesNotExist(InfoDateColumn, to_date(lit(reportDate), ReportDateFormat)) + .withColumnIfDoesNotExist(InfoDateColumnString, lit(reportDate)) + .withColumnIfDoesNotExist(InfoVersionColumn, lit(reportVersion)) + } + private def getReportVersion[T](jobConfig: JobConfigParser[T], dataset: Dataset)(implicit hadoopConf: Configuration): Int = { jobConfig.reportVersion match { case Some(version) => version diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala index 8c4b42914..cc69b72a7 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala @@ -19,11 +19,9 @@ import java.io.{PrintWriter, StringWriter} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem -import org.apache.spark.sql.functions.{lit, to_date} import org.apache.spark.sql.{DataFrame, SparkSession} import za.co.absa.atum.AtumImplicits._ import za.co.absa.atum.core.Atum -import za.co.absa.enceladus.common.Constants.{InfoDateColumn, InfoDateColumnString, InfoVersionColumn, ReportDateFormat} import za.co.absa.enceladus.common.RecordIdGeneration._ import za.co.absa.enceladus.common.config.{CommonConfConstants, JobConfigParser, PathConfig} import za.co.absa.enceladus.common.plugin.menas.MenasPlugin @@ -37,7 +35,6 @@ import za.co.absa.enceladus.model.Dataset import za.co.absa.enceladus.standardization_conformance.config.StandardizationConformanceConfig import za.co.absa.enceladus.utils.config.{ConfigReader, PathWithFs} import za.co.absa.enceladus.utils.fs.HadoopFsUtils -import za.co.absa.enceladus.utils.implicits.DataFrameImplicits.DataFrameEnhancements import za.co.absa.enceladus.utils.modules.SourcePhase import za.co.absa.enceladus.common.performance.PerformanceMetricTools import za.co.absa.enceladus.utils.schema.SchemaUtils @@ -147,10 +144,7 @@ trait ConformanceExecution extends CommonJobExecution { menasCredentials.username, cmdLineArgs ) - val withPartCols = result - .withColumnIfDoesNotExist(InfoDateColumn, to_date(lit(cmd.reportDate), ReportDateFormat)) - .withColumnIfDoesNotExist(InfoDateColumnString, lit(cmd.reportDate)) - .withColumnIfDoesNotExist(InfoVersionColumn, lit(preparationResult.reportVersion)) + val withPartCols = addInfoColumns(result, cmd.reportDate, preparationResult.reportVersion) val recordCount: Long = result.lastCheckpointRowCount match { case None => withPartCols.count @@ -162,7 +156,7 @@ trait ConformanceExecution extends CommonJobExecution { val minBlockSize = configReader.getLongOption(CommonConfConstants.minPartitionSizeKey) val maxBlockSize = configReader.getLongOption(CommonConfConstants.maxPartitionSizeKey) - val withRepartitioning = repartitionDataFrame(result, minBlockSize, maxBlockSize) + val withRepartitioning = repartitionDataFrame(withPartCols, minBlockSize, maxBlockSize) withRepartitioning.write.parquet(preparationResult.pathCfg.publish.path) @@ -176,7 +170,7 @@ trait ConformanceExecution extends CommonJobExecution { menasCredentials.username, cmdLineArgs ) - withPartCols.writeInfoFile(preparationResult.pathCfg.publish.path)(publishFs) + withRepartitioning.writeInfoFile(preparationResult.pathCfg.publish.path)(publishFs) writePerformanceMetrics(preparationResult.performance, cmd) if (conformanceReader.isAutocleanStdFolderEnabled()) { diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformance.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformance.scala index 2b08bcae2..955c60758 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformance.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformance.scala @@ -21,6 +21,7 @@ import org.apache.commons.configuration2.Configuration import org.apache.hadoop.fs.FileSystem import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.StructType import org.apache.spark.sql.{DataFrame, SparkSession} import org.slf4j.{Logger, LoggerFactory} import za.co.absa.enceladus.common.Constants._ @@ -32,7 +33,7 @@ import za.co.absa.enceladus.dao.MenasDAO import za.co.absa.enceladus.dao.auth.{MenasCredentialsFactory, MenasKerberosCredentialsFactory, MenasPlainCredentialsFactory} import za.co.absa.enceladus.dao.rest.RestDaoFactory.AvailabilitySetup import za.co.absa.enceladus.dao.rest.{MenasConnectionStringParser, RestDaoFactory} -import za.co.absa.enceladus.model.Dataset +import za.co.absa.enceladus.model.{ConformedSchema, Dataset} import za.co.absa.enceladus.utils.fs.HadoopFsUtils import za.co.absa.enceladus.utils.validation.ValidationLevel import za.co.absa.hyperdrive.ingestor.api.transformer.{StreamTransformer, StreamTransformerFactory} @@ -68,8 +69,11 @@ class HyperConformance (menasBaseUrls: List[String], (implicit sparkSession: SparkSession, menasDAO: MenasDAO): DataFrame = { import za.co.absa.enceladus.utils.implicits.DataFrameImplicits.DataFrameEnhancements + val schema: StructType = menasDAO.getSchema(conformance.schemaName, conformance.schemaVersion) + val schemaFields = if (schema == null) List() else schema.fields.toList + val conformedSchema = ConformedSchema(schemaFields, conformance) val infoDateColumn = infoDateFactory.getInfoDateColumn(rawDf) - val infoVersionColumn = infoVersionFactory.getInfoVersionColumn(rawDf) + val infoVersionColumn = infoVersionFactory.getInfoVersionColumn(conformedSchema) // using HDFS implementation until HyperConformance is S3-ready implicit val hdfs: FileSystem = FileSystem.get(sparkSession.sparkContext.hadoopConfiguration) diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformanceAttributes.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformanceAttributes.scala index d348086c4..165ec439a 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformanceAttributes.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/HyperConformanceAttributes.scala @@ -58,7 +58,7 @@ trait HyperConformanceAttributes extends HasComponentAttributes { datasetVersionKey -> PropertyMetadata("Dataset version", None, required = true), reportDateKey -> - PropertyMetadata("Report date", Some("The current date is used by default "), required = false), + PropertyMetadata("Report date", Some("The current date is used by default"), required = false), reportVersionColumnKey -> PropertyMetadata("Report version column", Some("Taken from another column"), required = false), reportVersionKey -> diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/config/FilterFromConfig.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/config/FilterFromConfig.scala deleted file mode 100644 index 3aded1f71..000000000 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/config/FilterFromConfig.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2018 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.enceladus.conformance.config - -import java.text.ParseException - -import za.co.absa.enceladus.dao.rest.JsonSerializer -import za.co.absa.enceladus.model.dataFrameFilter.DataFrameFilter -import za.co.absa.enceladus.utils.config.ConfigReader - -import scala.util.{Failure, Success, Try} - -/** - * This is a helper object to allow configuration of Mapping tables filters before the UI is reasdy for that. - * Until then, the filters can be set via configuration. - */ -object FilterFromConfig { - private val configReader = new ConfigReader() - - private def dataFrameId(dataFrameName: String): String = { - s"dataframefilter.$dataFrameName" - } - - private def filterFromJson(dataFrameName: String, json: String): Option[DataFrameFilter] = { - val result = Try (JsonSerializer.fromJson[DataFrameFilter](json)) - result match { - case Failure(exception) => - throw new ParseException(s"$dataFrameName filter load failed: ${exception.getMessage}", 0) - case Success(filter) => Option(filter) - } - } - - private def readJson(configKey: String): Option[String] = { - configReader.getStringOption(configKey).filter(_.nonEmpty).map(_.replaceAllLiterally("'","\"")) - } - - def loadFilter(dataFrameName: String): Option[DataFrameFilter] = { - val filterJson = readJson(dataFrameId(dataFrameName)) - filterJson.filter(_.nonEmpty).flatMap(filterFromJson(dataFrameName, _)) - } -} diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/DynamicInterpreter.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/DynamicInterpreter.scala index 700e5a705..e277198a1 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/DynamicInterpreter.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/DynamicInterpreter.scala @@ -27,12 +27,11 @@ import za.co.absa.enceladus.conformance.config.ConformanceConfigParser import za.co.absa.enceladus.conformance.datasource.PartitioningUtils import za.co.absa.enceladus.conformance.interpreter.rules._ import za.co.absa.enceladus.conformance.interpreter.rules.custom.CustomConformanceRule -import za.co.absa.enceladus.conformance.interpreter.rules.mapping.{ - MappingRuleInterpreter, MappingRuleInterpreterBroadcast, MappingRuleInterpreterGroupExplode -} +import za.co.absa.enceladus.conformance.interpreter.rules.mapping.{MappingRuleInterpreter, MappingRuleInterpreterBroadcast, MappingRuleInterpreterGroupExplode} import za.co.absa.enceladus.dao.MenasDAO import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, _} import za.co.absa.enceladus.model.{Dataset => ConfDataset} +import za.co.absa.enceladus.utils.config.PathWithFs import za.co.absa.enceladus.utils.error.ErrorMessage import za.co.absa.enceladus.utils.explode.ExplosionContext import za.co.absa.enceladus.utils.fs.HadoopFsUtils @@ -236,7 +235,7 @@ case class DynamicInterpreter(implicit inputFs: FileSystem) { MappingRuleInterpreterBroadcast(rule, ictx.conformance) } else { //Only MappingRuleInterpreterBroadcast or MappingRuleInterpreterGroupExplode support multiple outputs - if (ictx.featureSwitches.experimentalMappingRuleEnabled || rule.additionalColumns.getOrElse(Map()).nonEmpty) { + if (ictx.featureSwitches.experimentalMappingRuleEnabled || rule.definedAdditionalColumns().nonEmpty) { log.info("Group explode strategy for mapping rules used") MappingRuleInterpreterGroupExplode(rule, ictx.conformance) } else { @@ -309,8 +308,13 @@ case class DynamicInterpreter(implicit inputFs: FileSystem) { val mappingTableDef = ictx.dao.getMappingTable(rule.mappingTable, rule.mappingTableVersion) val mappingTablePath = PartitioningUtils.getPartitionedPathName(mappingTableDef.hdfsPath, ictx.progArgs.reportDate) - val mappingTableSize = HadoopFsUtils.getOrCreate(inputFs).getDirectorySizeNoHidden(mappingTablePath) - (mappingTableSize / (1024 * 1024)).toInt + //accommodate different fs for the mapping table or different bucket + val mappingTableFs = PathWithFs.fromPath(mappingTablePath)(ictx.spark.sparkContext.hadoopConfiguration) + + val mappingTableSize = HadoopFsUtils.getOrCreate(mappingTableFs.fileSystem).getDirectorySizeNoHidden(mappingTableFs.path) + val mb = (mappingTableSize / (1024 * 1024)).toInt + log.debug(s"$mappingTablePath size: ${mb}MB") + mb } /** diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/CommonMappingRuleInterpreter.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/CommonMappingRuleInterpreter.scala index edbf4ac7b..3b6c19594 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/CommonMappingRuleInterpreter.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/CommonMappingRuleInterpreter.scala @@ -19,7 +19,6 @@ import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.slf4j.Logger -import za.co.absa.enceladus.conformance.config.FilterFromConfig import za.co.absa.enceladus.conformance.datasource.DataSource import za.co.absa.enceladus.conformance.interpreter.{ExplosionState, InterpreterContextArgs} import za.co.absa.enceladus.dao.MenasDAO @@ -66,13 +65,9 @@ trait CommonMappingRuleInterpreter { val mappingTableDef = dao.getMappingTable(rule.mappingTable, rule.mappingTableVersion) - val ruleFilter = if (rule.mappingTableFilter.nonEmpty) { - rule.mappingTableFilter - } else { - // This is a workaround until UI supports filter definition. Until then, the filters can be set via configuration. - FilterFromConfig.loadFilter(rule.mappingTable) - } + val ruleFilter = rule.mappingTableFilter val mappingTableFilter = mappingTableDef.filter.filterNot(_ => rule.getOverrideMappingTableOwnFilter) + // find the data frame from the mapping table val filter: Option[DataFrameFilter] = (ruleFilter, mappingTableFilter) match { case (Some(a), Some(b)) => Option(a and b) diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterBroadcast.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterBroadcast.scala index 67b49dacd..0267f396a 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterBroadcast.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterBroadcast.scala @@ -47,7 +47,7 @@ case class MappingRuleInterpreterBroadcast(rule: MappingConformanceRule, conform val broadcastedMt = spark.sparkContext.broadcast(mt) val errorUDF = BroadcastUtils.getErrorUdf(broadcastedMt, rule.allOutputColumns().keys.toSeq, mappings) - if (rule.additionalColumns.getOrElse(Map()).isEmpty) { + if (rule.definedAdditionalColumns().isEmpty) { val mappingUDF = BroadcastUtils.getMappingUdfForSingleOutput(broadcastedMt, defaultValues) val withMappedFieldsDf = NestedArrayTransformations.nestedExtendedStructAndErrorMap( diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterGroupExplode.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterGroupExplode.scala index f4463d7ea..ec872a21e 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterGroupExplode.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/mapping/MappingRuleInterpreterGroupExplode.scala @@ -51,7 +51,7 @@ case class MappingRuleInterpreterGroupExplode(rule: MappingConformanceRule, array(rule.attributeMappings.values.toSeq.map(arrCol(_).cast(StringType)): _*), typedLit(mappings)) - val withErrorsDf = if (rule.additionalColumns.getOrElse(Map()).isEmpty) { + val withErrorsDf = if (rule.definedAdditionalColumns().isEmpty) { val joined = joinDatasetAndMappingTable(mapTable, explodedDf) val placedDf = ExplodeTools.nestedRenameReplace(joined, rule.outputColumn, rule.outputColumn) diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/streaming/InfoVersionFactory.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/streaming/InfoVersionFactory.scala index 365e41dda..73114bade 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/streaming/InfoVersionFactory.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/streaming/InfoVersionFactory.scala @@ -16,28 +16,27 @@ package za.co.absa.enceladus.conformance.streaming import org.apache.commons.configuration2.Configuration -import org.apache.spark.sql.{Column, DataFrame} +import org.apache.spark.sql.Column import org.apache.spark.sql.functions.{col, lit} -import org.apache.spark.sql.types.StructField import za.co.absa.enceladus.conformance.streaming.InfoDateFactory.log -import za.co.absa.enceladus.utils.schema.SchemaUtils +import za.co.absa.enceladus.model.ConformedSchema sealed trait InfoVersionFactory { - def getInfoVersionColumn(df: DataFrame): Column + def getInfoVersionColumn(conformedSchema: ConformedSchema): Column } object InfoVersionFactory { private class InfoVersionLiteralFactory(reportVersion: Int) extends InfoVersionFactory { - override def getInfoVersionColumn(df: DataFrame): Column = lit(reportVersion) + override def getInfoVersionColumn(conformedSchema: ConformedSchema): Column = lit(reportVersion) } private class InfoVersionColumnFactory(columnName: String) extends InfoVersionFactory { - override def getInfoVersionColumn(df: DataFrame): Column = { - val dt: Option[StructField] = SchemaUtils.getField(columnName, df.schema) - dt match { - case Some(_) => col(columnName) - case None => throw new IllegalArgumentException(s"The specified info column does not exist: $columnName") + override def getInfoVersionColumn(conformedSchema: ConformedSchema): Column = { + if(conformedSchema.hasField(columnName)) { + col(columnName) + } else { + throw new IllegalArgumentException(s"The specified info column does not exist: $columnName") } } } diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala index 66f5d99c9..4dfb213d9 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala @@ -200,8 +200,8 @@ trait StandardizationExecution extends CommonJobExecution { val maxPartitionSize = configReader.getLongOption(CommonConfConstants.maxPartitionSizeKey) val withRepartitioning = if (cmd.isInstanceOf[StandardizationConfig]) { - repartitionDataFrame(standardizedDF, minPartitionSize, maxPartitionSize) - } else { + repartitionDataFrame(standardizedDF, minPartitionSize, maxPartitionSize) + } else { standardizedDF } withRepartitioning.write.parquet(preparationResult.pathCfg.standardization.path) @@ -224,10 +224,10 @@ trait StandardizationExecution extends CommonJobExecution { cmd.csvDelimiter.foreach(delimiter => Atum.setAdditionalInfo("csv_delimiter" -> delimiter)) log.info(s"infoFilePath = ${preparationResult.pathCfg.standardization.path}/_INFO") - standardizedDF.writeInfoFile(preparationResult.pathCfg.standardization.path)(stdFs) + withRepartitioning.writeInfoFile(preparationResult.pathCfg.standardization.path)(stdFs) writePerformanceMetrics(preparationResult.performance, cmd) log.info(s"$sourceId finished successfully") - standardizedDF + withRepartitioning } //scalastyle:off parameter.number diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceJob.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceJob.scala index f4b46e0b2..2bf6febec 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceJob.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceJob.scala @@ -50,6 +50,7 @@ object StandardizationAndConformanceJob extends StandardizationAndConformanceExe processStandardizationResult(args, standardized, preparationResult, schema, cmd, menasCredentials) // post processing deliberately rereads the output to make sure that outputted data is stable #1538 runPostProcessing(SourcePhase.Standardization, preparationResult, cmd) + standardized.unpersist() prepareConformance(preparationResult) val confInputData = readConformanceInputData(preparationResult.pathCfg) diff --git a/spark-jobs/src/test/resources/application.conf b/spark-jobs/src/test/resources/application.conf index cd362f17b..b4538b3fc 100644 --- a/spark-jobs/src/test/resources/application.conf +++ b/spark-jobs/src/test/resources/application.conf @@ -35,11 +35,5 @@ control.info.validation=warning #system-wide time zone timezone="UTC" -#Mapping tables filters -dataframefilter.Empty="" -dataframefilter.OK="{'_t':'EqualsFilter','columnName':'myColumn','value':'This value'}" -dataframefilter.Fail="Not a json" -dataframefilter.Complex="{'_t':'AndJoinedFilters','filterItems':[{'_t':'EqualsFilter','columnName':'myColumn','value':'This value'},{'_t':'DiffersFilter','columnName':'myColumn2','value':'2','valueType':'integer'}]}" - # info file prefix for dataset properties control.info.dataset.properties.prefix="ds_testing_" diff --git a/spark-jobs/src/test/resources/data/empty/_SUCCESS b/spark-jobs/src/test/resources/data/empty/_SUCCESS new file mode 100644 index 000000000..e69de29bb diff --git a/spark-jobs/src/test/scala/za/co/absa/enceladus/common/CommonExecutionSuite.scala b/spark-jobs/src/test/scala/za/co/absa/enceladus/common/CommonExecutionSuite.scala index d89f8d5a8..5f65d0112 100644 --- a/spark-jobs/src/test/scala/za/co/absa/enceladus/common/CommonExecutionSuite.scala +++ b/spark-jobs/src/test/scala/za/co/absa/enceladus/common/CommonExecutionSuite.scala @@ -15,6 +15,8 @@ package za.co.absa.enceladus.common +import org.apache.spark.sql.types.{StringType, StructType} +import org.apache.spark.sql.{DataFrame, SparkSession} import org.mockito.Mockito import org.mockito.scalatest.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec @@ -33,6 +35,9 @@ class CommonExecutionSuite extends AnyFlatSpec with Matchers with SparkTestBase prepareJob() } override protected def validatePaths(pathConfig: PathConfig): Unit = {} + override def repartitionDataFrame(df: DataFrame, minBlockSize: Option[Long], maxBlockSize: Option[Long]) + (implicit spark: SparkSession): DataFrame = + super.repartitionDataFrame(df, minBlockSize, maxBlockSize) } Seq( @@ -49,7 +54,6 @@ class CommonExecutionSuite extends AnyFlatSpec with Matchers with SparkTestBase Mockito.when(dao.getDataset("DatasetA", 1, ValidationLevel.ForRun)).thenReturn(dataset) doNothing.when(dao).authenticate() - val commonJob = new CommonJobExecutionTest val exceptionMessage = intercept[IllegalStateException](commonJob.testRun).getMessage @@ -59,4 +63,15 @@ class CommonExecutionSuite extends AnyFlatSpec with Matchers with SparkTestBase } } + "repartitionDataFrame" should "pass on empty data" in { + val schema = new StructType() + .add("not_important", StringType, nullable = true) + // reading the data from empty directory to get 0 partitions, even creating a DatFrame from an empty sequence gives 1 partition + val df = spark.read.schema(schema).parquet("src/test/resources/data/empty") + df.rdd.getNumPartitions shouldBe 0 // ensure there are 0 partitions for the test + val commonJob = new CommonJobExecutionTest + val result = commonJob.repartitionDataFrame(df, Option(1), Option(2)) + result shouldBe df + } + } diff --git a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/config/FilterFromConfigSuite.scala b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/config/FilterFromConfigSuite.scala deleted file mode 100644 index 617ecb661..000000000 --- a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/config/FilterFromConfigSuite.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2018 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.enceladus.conformance.config - -import java.text.ParseException - -import org.apache.spark.sql.types._ -import org.scalatest.funsuite.AnyFunSuite -import za.co.absa.enceladus.model.dataFrameFilter._ - -class FilterFromConfigSuite extends AnyFunSuite { - - test("Filter for dataset doesn't exist") { - assert(FilterFromConfig.loadFilter("NotExistent").isEmpty) - } - - test("Filter for dataset is empty") { - assert(FilterFromConfig.loadFilter("Empty").isEmpty) - } - - test("Filter for dataset is set") { - val valueType: String = null - val expected = EqualsFilter("myColumn", "This value", valueType) - val loaded = FilterFromConfig.loadFilter("OK").get - assert(loaded == expected) - assert(loaded.asInstanceOf[EqualsFilter].dataType == StringType) - } - - test("Filter for dataset is wrong") { - val filterName = "Fail" - - val except = intercept[ParseException] { - FilterFromConfig.loadFilter(filterName) - } - assert(except.getMessage.contains(s"$filterName filter load failed")) - } - - test("A complex filter") { - val valueType: String = null - val f1 = EqualsFilter("myColumn", "This value", valueType) - val f2 = DiffersFilter("myColumn2", "2", IntegerType) - val expected = AndJoinedFilters(Set(f1, f2)) - val loaded = FilterFromConfig.loadFilter("Complex").get - assert(loaded == expected) - val types = loaded.asInstanceOf[AndJoinedFilters].filterItems.map(item => item.asInstanceOf[SingleColumnAndValueFilter].dataType) - assert(types == Set(StringType, IntegerType)) - } -} diff --git a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/fixtures/StreamingFixture.scala b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/fixtures/StreamingFixture.scala index 591fe2608..9b33fbfaf 100644 --- a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/fixtures/StreamingFixture.scala +++ b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/fixtures/StreamingFixture.scala @@ -18,6 +18,7 @@ package za.co.absa.enceladus.conformance.interpreter.fixtures import org.apache.commons.configuration2.Configuration import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.execution.streaming.MemoryStream +import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row} import org.mockito.Mockito.lenient import org.scalatest.funsuite.AnyFunSuite @@ -38,13 +39,15 @@ trait StreamingFixture extends AnyFunSuite with SparkTestBase with MockitoSugar protected def testHyperConformanceFromConfig(input: DataFrame, sinkTableName: String, dataset: Dataset, - reportDate: String) + reportDate: String, + reportVersionColumnKeyProvided: String + ) (implicit menasDAO: MenasDAO): DataFrame = { val configStub: Configuration = mock[Configuration] when(configStub.containsKey(reportVersionKey)).thenReturn(false) when(configStub.containsKey(eventTimestampColumnKey)).thenReturn(false) lenient.when(configStub.containsKey(reportVersionColumnKey)).thenReturn(true) - when(configStub.getString(reportVersionColumnKey)).thenReturn("numerics.SmartObject.all_random") + when(configStub.getString(reportVersionColumnKey)).thenReturn(reportVersionColumnKeyProvided) when(configStub.containsKey(reportDateKey)).thenReturn(true) when(configStub.getString(reportDateKey)).thenReturn(reportDate) when(configStub.containsKey(datasetNameKey)).thenReturn(true) @@ -60,6 +63,10 @@ trait StreamingFixture extends AnyFunSuite with SparkTestBase with MockitoSugar when(configStub.getInt(menasUriRetryCountKey)).thenReturn(0) when(configStub.containsKey(menasAvailabilitySetupKey)).thenReturn(false) + when(menasDAO.getSchema(dataset.schemaName,dataset.schemaVersion)).thenReturn(StructType(Seq( + StructField("numerics.SmartObject.all_random", StringType) + ))) + val memoryStream = new MemoryStream[Row](1, spark.sqlContext)(RowEncoder(input.schema)) val hyperConformance = HyperConformance(configStub).asInstanceOf[HyperConformance] val source: DataFrame = memoryStream.toDF() diff --git a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/RulesSuite.scala b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/RulesSuite.scala index fca3d2f4a..1c1f81ad8 100644 --- a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/RulesSuite.scala +++ b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/RulesSuite.scala @@ -62,6 +62,14 @@ class RulesSuite extends AnyFunSuite with SparkTestBase { assert(roleCondGen.semanticEquals(roleCond)) } + test("Test empty join condition evaluates to true") { + val countryRule = EmployeeConformance.countryRule.copy(attributeMappings = Map.empty) + val countryCondGen = CommonMappingRuleInterpreter.getJoinCondition(countryRule).expr + val countryCond = lit(true).expr + + assert(countryCondGen.semanticEquals(countryCond)) + } + test("Infest strictest type int") { val colGen = dummyInterpreter.inferStrictestType("2").expr val colMan = lit(2).expr diff --git a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/testcasefactories/SimpleTestCaseFactory.scala b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/testcasefactories/SimpleTestCaseFactory.scala index 1bf1b17e7..b9908d478 100644 --- a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/testcasefactories/SimpleTestCaseFactory.scala +++ b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/interpreter/rules/testcasefactories/SimpleTestCaseFactory.scala @@ -15,7 +15,7 @@ package za.co.absa.enceladus.conformance.interpreter.rules.testcasefactories -import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.hadoop.fs.Path import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} import org.mockito.Mockito.{mock, when => mockWhen} @@ -25,7 +25,7 @@ import za.co.absa.enceladus.dao.MenasDAO import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory} import za.co.absa.enceladus.model.{Dataset, DefaultValue, MappingTable} -import za.co.absa.enceladus.utils.fs.{HadoopFsUtils, LocalFsUtils} +import za.co.absa.enceladus.utils.fs.LocalFsUtils import za.co.absa.enceladus.utils.testUtils.HadoopFsTestBase import za.co.absa.enceladus.utils.validation.ValidationLevel diff --git a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/streaming/HyperConformanceIntegrationSuite.scala b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/streaming/HyperConformanceIntegrationSuite.scala index 3e226656b..436d3a6a1 100644 --- a/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/streaming/HyperConformanceIntegrationSuite.scala +++ b/spark-jobs/src/test/scala/za/co/absa/enceladus/conformance/streaming/HyperConformanceIntegrationSuite.scala @@ -23,9 +23,11 @@ import za.co.absa.enceladus.conformance.interpreter.fixtures.{NestedStructsFixtu class HyperConformanceIntegrationSuite extends AnyFunSuite with StreamingFixture with NestedStructsFixture { + private val reportDateValue = "2020-05-23" + test("Test with catalyst workaround, literal factory") { val configuration = new PropertyListConfiguration() - configuration.addProperty(reportDateKey, "2020-05-23") + configuration.addProperty(reportDateKey, reportDateValue) configuration.addProperty(reportVersionKey, 1) implicit val infoDateFactory: InfoDateFactory = InfoDateFactory.getFactoryFromConfig(configuration) implicit val infoVersionFactory: InfoVersionFactory = InfoVersionFactory.getFactoryFromConfig(configuration) @@ -47,7 +49,7 @@ class HyperConformanceIntegrationSuite extends AnyFunSuite with StreamingFixture val df: DataFrame = testHyperConformanceFromConfig(standardizedDf, "result", nestedStructsDS, - reportDate = "2020-05-23" ) + reportDate = reportDateValue, "numerics.SmartObject.all_random" ) .orderBy("ID") assertResult(df.count())(20) @@ -59,6 +61,22 @@ class HyperConformanceIntegrationSuite extends AnyFunSuite with StreamingFixture assertResult(returned)(conformed) } + test("Test Hyperconformance from config, conformed column info") { + val df: DataFrame = testHyperConformanceFromConfig(standardizedDf, + "result", + nestedStructsDS, + reportDate = reportDateValue, "strings.all_random_upper" ) + .orderBy("ID") + + assertResult(df.count())(20) + val conformed = spark.read + .textFile("src/test/testData/nestedStructs/conformed_literal_conf_info_col.json") + .collect().mkString("\n") + val returned = df.toJSON.collect().mkString("\n") + + assertResult(returned)(conformed) + } + test("Test with catalyst workaround, event time factory") { val configuration = new PropertyListConfiguration() configuration.addProperty(eventTimestampColumnKey, "dates.date_format5") diff --git a/spark-jobs/src/test/testData/nestedStructs/conformed_literal_conf_info_col.json b/spark-jobs/src/test/testData/nestedStructs/conformed_literal_conf_info_col.json new file mode 100644 index 000000000..5b795b367 --- /dev/null +++ b/spark-jobs/src/test/testData/nestedStructs/conformed_literal_conf_info_col.json @@ -0,0 +1,20 @@ +{"ID":"1aWBmiwlkCZB8bNeEXCZX6OlSFedKvC0LtAP7QMzUg08XJhV8yMnFOyTQ3MFxaFo","dates":{"date_format1":"009, 09 Jan 2017 23:40:23 GMT+1","date_format2":"2017-01-05T13:18:35","date_format3":"Fri, 6 Jan 2017 10:15:43 +0100","date_format4":"2017-01-11T10:18:05+0100","date_format5":"01-11-2017 09:45","epoch":{"ibm":"31122121.221243","overflow_negative":-82273821210,"overflow_positive":911219864722,"random_negative":-1605659684,"random_positive":714866123,"zero":0}},"numerics":{"SmartObject":{"all_random":"43#sK]BDFo7kEc]vrY^A","whitespaces":"d w w a zv ri","with_new_lines":"Maxime repudiandae officia ex dolorum. Dicta suscipit aliquam ullam impedit doloremque animi ipsa. Sunt laborum qui cum quis quibusdam eum ducimus. Placeat ratione dignissimos esse maxime sit pariatur nostrum beatae enim deserunt voluptates sit. Modi quibusdam eum quam nesciunt amet sed totam aspernatur ea. Et qui rerum et dicta.\n\nFugiat eligendi nostrum consequuntur dolores doloremque possimus perferendis reprehenderit dolores vero aut ducimus numquam voluptas optio. Quia cupiditate est unde sed aspernatur. Rem quaerat qui eos labore rerum ea dolorum dolor quod sed non molestias. Et enim quis est atque perferendis rem. Nihil voluptate sit sit dolorem deleniti ut amet cupiditate accusamus aut deleniti. Iure sapiente labore consequatur enim et dolores voluptatem aut necessitatibus dolore non quod ut quod. Beatae culpa animi ut eos at fuga nobis. Tempore rerum voluptates ut necessitatibus velit dolor molestiae impedit ex id et tenetur assumenda et. Eaque aut a laboriosam ut dolorem sint ut quas.\n\nNihil veritatis aut excepturi rerum nulla rerum perspiciatis dolor autem. Praesentium cum velit saepe sunt tenetur quisquam enim aut inventore pariatur est suscipit ut ex delectus. Eligendi illum ea dolore eaque. Quis corrupti accusantium tenetur."},"big_negative":-402529611146737,"big_positive":16904479461635,"small_negative":-335,"small_positive":885,"zero":0,"small_positive_casted1":"885","small_negative_casted1":"-335","big_positive_casted1":"16904479461635","small_positive_negated":-885,"small_negative_negated":335,"big_positive_negated":-16904479461635,"big_negative_negated":402529611146737},"strings":{"all_random":"2HN6xwXB*nBe6Ndddv#0","whitespaces":" z di ","with_new_lines":"Id excepturi et ut qui eos ullam sunt placeat. In eius esse sed et et nobis quidem nihil itaque maiores sit omnis vitae sequi culpa. Quia excepturi quae ipsa maxime vero voluptatem exercitationem aut mollitia ipsa tempora temporibus quas error. Facilis sapiente culpa itaque. Exercitationem quia eum et in aspernatur non dicta optio aliquid vel ut dolorem facere itaque laboriosam. Nihil ut ea est officiis delectus autem et nostrum autem qui quae aut autem voluptas dolorum. Consequatur consequuntur aperiam cumque. Eveniet a amet et sit quos velit sit tempora vel aliquid ipsa quis. Occaecati harum non excepturi porro a nihil voluptatem qui inventore vel.\n\nUt corrupti ab maiores deserunt officiis. Excepturi eveniet nisi eius ut fugiat ex illum qui perspiciatis dolores provident ut quia beatae. Incidunt ut qui eum iusto amet modi excepturi. Rem delectus expedita omnis accusantium excepturi sed et error qui.\n\nEt rerum voluptatibus omnis rerum soluta. Laudantium molestiae quia quo praesentium suscipit sit temporibus. Asperiores perspiciatis deleniti sit nihil repellat soluta necessitatibus ad corporis laborum repellat.","with_new_lines_upper":"ID EXCEPTURI ET UT QUI EOS ULLAM SUNT PLACEAT. IN EIUS ESSE SED ET ET NOBIS QUIDEM NIHIL ITAQUE MAIORES SIT OMNIS VITAE SEQUI CULPA. QUIA EXCEPTURI QUAE IPSA MAXIME VERO VOLUPTATEM EXERCITATIONEM AUT MOLLITIA IPSA TEMPORA TEMPORIBUS QUAS ERROR. FACILIS SAPIENTE CULPA ITAQUE. EXERCITATIONEM QUIA EUM ET IN ASPERNATUR NON DICTA OPTIO ALIQUID VEL UT DOLOREM FACERE ITAQUE LABORIOSAM. NIHIL UT EA EST OFFICIIS DELECTUS AUTEM ET NOSTRUM AUTEM QUI QUAE AUT AUTEM VOLUPTAS DOLORUM. CONSEQUATUR CONSEQUUNTUR APERIAM CUMQUE. EVENIET A AMET ET SIT QUOS VELIT SIT TEMPORA VEL ALIQUID IPSA QUIS. OCCAECATI HARUM NON EXCEPTURI PORRO A NIHIL VOLUPTATEM QUI INVENTORE VEL.\n\nUT CORRUPTI AB MAIORES DESERUNT OFFICIIS. EXCEPTURI EVENIET NISI EIUS UT FUGIAT EX ILLUM QUI PERSPICIATIS DOLORES PROVIDENT UT QUIA BEATAE. INCIDUNT UT QUI EUM IUSTO AMET MODI EXCEPTURI. REM DELECTUS EXPEDITA OMNIS ACCUSANTIUM EXCEPTURI SED ET ERROR QUI.\n\nET RERUM VOLUPTATIBUS OMNIS RERUM SOLUTA. LAUDANTIUM MOLESTIAE QUIA QUO PRAESENTIUM SUSCIPIT SIT TEMPORIBUS. ASPERIORES PERSPICIATIS DELENITI SIT NIHIL REPELLAT SOLUTA NECESSITATIBUS AD CORPORIS LABORUM REPELLAT.","all_random_upper":"2HN6XWXB*NBE6NDDDV#0","whitespaces_upper":" Z DI "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"2HN6XWXB*NBE6NDDDV#0"} +{"ID":"45kQ9jb8XtpV2DWqMyNqhA7xAtDrbabpvUdkuJOSgHUFSIRUeknQWmVT5B4uS9Tm","dates":{"date_format1":"006, 06 Jan 2017 10:17:59 GMT+1","date_format2":"2017-01-04T09:57:35","date_format3":"Wed, 11 Jan 2017 04:46:49 +0100","date_format4":"2017-01-14T09:49:23+0100","date_format5":"01-17-2017 14:31","epoch":{"ibm":"14082062.101844","overflow_negative":-53651978336,"overflow_positive":995641328170,"random_negative":-2126585361,"random_positive":1823211439,"zero":0}},"numerics":{"SmartObject":{"all_random":"Z30*(UU#TtA@NahATZe%","whitespaces":" xwhqpy tw d w p ","with_new_lines":"Modi quibusdam omnis ut at corporis suscipit voluptas rerum reprehenderit ex pariatur. Dignissimos ab at reprehenderit qui explicabo aut est ea aliquam est veniam voluptas dolores. Minima et architecto ea maxime iste eaque nesciunt molestiae nemo dolores dolor reprehenderit qui. Ullam magnam optio quo qui nostrum.\n\nRerum dolor magnam impedit sit aut iusto quod qui eos. Voluptatem eius culpa ullam quo repellat occaecati quas quasi nam ipsum architecto nemo sapiente sunt. Vero expedita voluptatum deleniti. Ullam nihil et rem est quia autem aut rem. Quia omnis et similique repudiandae voluptate qui voluptatum et ipsam eos quod natus et deserunt illo.\n\nEt dolorem et consectetur minima nostrum quis dolorem. Et ut vitae rerum architecto dolor sit doloribus in consequatur fugiat. Quisquam tenetur in molestiae animi eaque sed est aut. Sapiente quasi quam quia voluptatem eaque fugit sapiente dolorem et ad maiores eum magni voluptatem. Vero impedit minima et occaecati nesciunt aut. Sit sint et aut."},"big_negative":-855212471318448,"big_positive":849206950782983,"small_negative":-437,"small_positive":446,"zero":0,"small_positive_casted1":"446","small_negative_casted1":"-437","big_positive_casted1":"849206950782983","small_positive_negated":-446,"small_negative_negated":437,"big_positive_negated":-849206950782983,"big_negative_negated":855212471318448},"strings":{"all_random":"FZo0%FBUU8f6HPYP9syS","whitespaces":" h s j p l","with_new_lines":"Neque in dolorem rerum corrupti maxime dolorum et eligendi. Esse accusantium et qui excepturi iste voluptas similique et. Quia rerum occaecati consequatur hic explicabo. Quos beatae laudantium voluptas ex quisquam est. Saepe aut voluptas voluptate saepe reprehenderit cumque nihil qui commodi atque et debitis laudantium aut itaque.\n\nQuasi consequatur id odit deserunt vitae consequatur quia magnam sint. Repudiandae maxime ipsa voluptas itaque neque consequatur id architecto provident aut. Ab quibusdam velit ducimus cum numquam animi eligendi. Nemo magnam non magni harum dicta repellendus quos enim vel facere.\n\nAd provident pariatur in ut officiis inventore tempore est a ut minima quibusdam quos consequatur iusto. Ea dolorum mollitia delectus. Reprehenderit qui numquam maiores alias dolores odit omnis. Facilis ut optio consequatur sint sit porro laborum illo est aperiam illum quas sit rerum.","with_new_lines_upper":"NEQUE IN DOLOREM RERUM CORRUPTI MAXIME DOLORUM ET ELIGENDI. ESSE ACCUSANTIUM ET QUI EXCEPTURI ISTE VOLUPTAS SIMILIQUE ET. QUIA RERUM OCCAECATI CONSEQUATUR HIC EXPLICABO. QUOS BEATAE LAUDANTIUM VOLUPTAS EX QUISQUAM EST. SAEPE AUT VOLUPTAS VOLUPTATE SAEPE REPREHENDERIT CUMQUE NIHIL QUI COMMODI ATQUE ET DEBITIS LAUDANTIUM AUT ITAQUE.\n\nQUASI CONSEQUATUR ID ODIT DESERUNT VITAE CONSEQUATUR QUIA MAGNAM SINT. REPUDIANDAE MAXIME IPSA VOLUPTAS ITAQUE NEQUE CONSEQUATUR ID ARCHITECTO PROVIDENT AUT. AB QUIBUSDAM VELIT DUCIMUS CUM NUMQUAM ANIMI ELIGENDI. NEMO MAGNAM NON MAGNI HARUM DICTA REPELLENDUS QUOS ENIM VEL FACERE.\n\nAD PROVIDENT PARIATUR IN UT OFFICIIS INVENTORE TEMPORE EST A UT MINIMA QUIBUSDAM QUOS CONSEQUATUR IUSTO. EA DOLORUM MOLLITIA DELECTUS. REPREHENDERIT QUI NUMQUAM MAIORES ALIAS DOLORES ODIT OMNIS. FACILIS UT OPTIO CONSEQUATUR SINT SIT PORRO LABORUM ILLO EST APERIAM ILLUM QUAS SIT RERUM.","all_random_upper":"FZO0%FBUU8F6HPYP9SYS","whitespaces_upper":" H S J P L"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"FZO0%FBUU8F6HPYP9SYS"} +{"ID":"5mktK1GUFOuRWHic1cmQ7hd1cL6XqeRK78JgIzDDQ4O31gTEMzvFVa7Riv7HdeGy","dates":{"date_format1":"014, 14 Jan 2017 09:26:38 GMT+1","date_format2":"2017-01-17T00:39:47","date_format3":"Fri, 6 Jan 2017 22:12:07 +0100","date_format4":"2017-01-19T18:31:33+0100","date_format5":"01-07-2017 01:25","epoch":{"ibm":"15012073.184727","overflow_negative":-65384614253,"overflow_positive":36056999340,"random_negative":-142314786,"random_positive":636966473,"zero":0}},"numerics":{"SmartObject":{"all_random":"NFakgabtV31$iqQ3ab07","whitespaces":" w tc k oq ","with_new_lines":"Nesciunt sunt non delectus odit aut odio qui a nesciunt. Cupiditate qui minus ut minus qui culpa molestiae ut sequi voluptates. Maiores dolorem corporis molestiae quasi. Autem officia rerum alias dolores saepe qui ipsum qui qui. Officia quis unde atque soluta excepturi blanditiis optio. Sit voluptas omnis nihil nemo quia voluptatem molestiae provident similique ipsum. Et aut quae veniam ut hic libero possimus voluptates eos deleniti veniam. Repellendus cupiditate ea est sint eveniet ea in numquam voluptatem dolorem labore placeat temporibus.\n\nOmnis beatae saepe quia et numquam modi eum deleniti debitis est eum officia eius. Eius consequatur dignissimos sit accusamus incidunt in facere ut id itaque a reprehenderit. Consequatur quia veritatis error beatae provident beatae qui rerum tempora asperiores voluptates debitis fuga. Repellendus alias est sunt et dolorem minima magnam eos. Hic omnis est ab debitis temporibus hic consequuntur. Quos minus nostrum consectetur itaque et ea quo aut aspernatur ipsum consequuntur ratione eos molestias.\n\nQuas incidunt ut et consequuntur quis et laboriosam officia aliquam impedit vel. Aut tempore similique quas. Molestias eos a debitis sint veniam impedit. Aut alias quos quis alias mollitia eos aut officiis voluptate unde mollitia similique laboriosam quo."},"big_negative":-783143645497786,"big_positive":921956159852521,"small_negative":-141,"small_positive":771,"zero":0,"small_positive_casted1":"771","small_negative_casted1":"-141","big_positive_casted1":"921956159852521","small_positive_negated":-771,"small_negative_negated":141,"big_positive_negated":-921956159852521,"big_negative_negated":783143645497786},"strings":{"all_random":"zgr]p]ki!KWRdD&%X&wi","whitespaces":"k c h e rs z ","with_new_lines":"Consequatur cum maiores sed laborum rem fugit cumque et aut sint quaerat facere veritatis. Enim in perspiciatis ut dolores possimus sint quia est qui hic. Vitae architecto ex quos. Ea non dolor quos nulla cum ipsa sed facere et voluptatem vero aut quasi hic ut. Quo harum rerum eos error iste odio et et. Est molestiae quod voluptatum et at unde nulla est sint quasi deleniti. Vel est minima quis voluptatem qui excepturi consequatur.\n\nNeque vel quia eaque. Similique dicta ut deserunt quo esse et sit atque. Quos autem itaque occaecati officia non sunt et adipisci ut saepe. Dolorem sint harum non sed ipsam quidem fuga cupiditate velit saepe.\n\nEt ea ratione ullam velit porro ut. Asperiores quae omnis autem hic. Et ut non voluptas pariatur non ea quia ut ipsum repudiandae. Praesentium fuga minima assumenda quia necessitatibus dolor repellat eos et laudantium quidem et. Dolores animi praesentium numquam a.","with_new_lines_upper":"CONSEQUATUR CUM MAIORES SED LABORUM REM FUGIT CUMQUE ET AUT SINT QUAERAT FACERE VERITATIS. ENIM IN PERSPICIATIS UT DOLORES POSSIMUS SINT QUIA EST QUI HIC. VITAE ARCHITECTO EX QUOS. EA NON DOLOR QUOS NULLA CUM IPSA SED FACERE ET VOLUPTATEM VERO AUT QUASI HIC UT. QUO HARUM RERUM EOS ERROR ISTE ODIO ET ET. EST MOLESTIAE QUOD VOLUPTATUM ET AT UNDE NULLA EST SINT QUASI DELENITI. VEL EST MINIMA QUIS VOLUPTATEM QUI EXCEPTURI CONSEQUATUR.\n\nNEQUE VEL QUIA EAQUE. SIMILIQUE DICTA UT DESERUNT QUO ESSE ET SIT ATQUE. QUOS AUTEM ITAQUE OCCAECATI OFFICIA NON SUNT ET ADIPISCI UT SAEPE. DOLOREM SINT HARUM NON SED IPSAM QUIDEM FUGA CUPIDITATE VELIT SAEPE.\n\nET EA RATIONE ULLAM VELIT PORRO UT. ASPERIORES QUAE OMNIS AUTEM HIC. ET UT NON VOLUPTAS PARIATUR NON EA QUIA UT IPSUM REPUDIANDAE. PRAESENTIUM FUGA MINIMA ASSUMENDA QUIA NECESSITATIBUS DOLOR REPELLAT EOS ET LAUDANTIUM QUIDEM ET. DOLORES ANIMI PRAESENTIUM NUMQUAM A.","all_random_upper":"ZGR]P]KI!KWRDD&%X&WI","whitespaces_upper":"K C H E RS Z "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"ZGR]P]KI!KWRDD&%X&WI"} +{"ID":"98KF4sntNuXp9MtIfzKtPH8Kcq2xFEjBRI4oUJtJAXftpFqO89Ie34yw4ZWqFYjn","dates":{"date_format1":"024, 24 Jan 2017 18:08:00 GMT+1","date_format2":"2017-01-27T05:33:40","date_format3":"Sun, 15 Jan 2017 18:00:43 +0100","date_format4":"2017-01-19T19:48:16+0100","date_format5":"01-18-2017 18:01","epoch":{"ibm":"16092182.134147","overflow_negative":-76474463410,"overflow_positive":848346721927,"random_negative":-1123482748,"random_positive":1903762053,"zero":0}},"numerics":{"SmartObject":{"all_random":"w@BQHwAq7F(gsIvyV$mn","whitespaces":" y x c lws ","with_new_lines":"Itaque a exercitationem et rerum incidunt quas blanditiis. Nostrum dolor dolorem et ab voluptas vel ipsum consequuntur autem sit eius quos. Recusandae et velit et fugiat neque et veniam aliquid velit. Dolores atque ratione dolores sed iure. Fugiat explicabo iusto consequatur quia. Voluptates asperiores non amet eaque repudiandae aut voluptatum. Nostrum facilis illo quod non. Necessitatibus repudiandae quia commodi accusantium iste libero explicabo. Fuga eveniet hic asperiores sed et tenetur itaque vel ipsum rerum non velit ut sapiente est.\n\nIpsam ratione unde enim ducimus consequatur quia et. Ut aut doloribus molestiae totam minima omnis consequuntur et consequatur modi temporibus esse odit. Mollitia sed omnis quo vel quo eum. Pariatur libero ratione voluptate eum aliquid quidem in ut id officiis sed voluptatem aliquid maxime ut. Praesentium impedit enim facere quia molestiae amet ducimus quas porro. Quisquam aut quis rerum eaque quia ut aliquid id velit molestiae cum. Et doloribus et pariatur qui deserunt saepe. Facere nesciunt velit impedit odit incidunt et reiciendis consequatur non qui. Facilis minus alias et.\n\nQuidem cupiditate doloremque assumenda harum dolor nisi adipisci quod saepe dolor. Eveniet rerum consequatur quam enim voluptatem qui debitis voluptatem possimus nihil possimus vero ea inventore. Libero amet dolorem repudiandae qui accusantium laudantium similique laborum error omnis excepturi quia voluptates laboriosam. Laborum perspiciatis officiis quis suscipit ullam aut ut et magnam distinctio ullam. Ut voluptate aut ea explicabo et quasi laborum incidunt magnam praesentium. Ut amet quibusdam quaerat aut distinctio exercitationem et cupiditate et animi et omnis doloremque temporibus repudiandae. Aperiam nostrum blanditiis quisquam. Atque at omnis et eveniet tenetur eum ullam nostrum ratione dicta non ex. Nobis recusandae recusandae delectus facere laborum dolores aliquam totam unde ipsa nesciunt aliquid in."},"big_negative":-756122246367585,"big_positive":594088171930711,"small_negative":-68,"small_positive":200,"zero":0,"small_positive_casted1":"200","small_negative_casted1":"-68","big_positive_casted1":"594088171930711","small_positive_negated":-200,"small_negative_negated":68,"big_positive_negated":-594088171930711,"big_negative_negated":756122246367585},"strings":{"all_random":"eGUS!471Epb&y^r4w8FH","whitespaces":"kh iq t fv o ","with_new_lines":"Facilis accusantium voluptas quia rerum alias maiores repudiandae saepe rerum inventore atque. Nihil optio eos est accusantium quidem at provident voluptatibus aut cum hic et quas perferendis. Voluptates vitae veniam sunt ut aliquid aut vitae explicabo. Rem et ut voluptas.\n\nEt iure minima soluta quo modi explicabo. Voluptatem ab iste molestiae sed nihil dolorem architecto dignissimos aut animi suscipit. Laboriosam id reiciendis voluptatum nesciunt error. Corporis voluptatibus vel et id cumque veritatis omnis sunt et.\n\nNulla at perspiciatis deserunt rerum ut ab harum pariatur qui. Nobis ut enim ad at et voluptate minus cum quidem sequi sed asperiores officia velit maxime. Sint est ratione qui repellendus aspernatur et voluptatum ab voluptates consequuntur aliquid ipsam. Minus laboriosam adipisci dolorem quasi facilis. Iure ipsam omnis optio dolor et. Quo maxime repellendus ipsa iusto enim eos quia est quas nemo eligendi laudantium non. Molestias voluptatum fugit quo voluptate numquam. Nemo debitis labore et ipsam modi rerum quia voluptas doloribus sint in minima rem.","with_new_lines_upper":"FACILIS ACCUSANTIUM VOLUPTAS QUIA RERUM ALIAS MAIORES REPUDIANDAE SAEPE RERUM INVENTORE ATQUE. NIHIL OPTIO EOS EST ACCUSANTIUM QUIDEM AT PROVIDENT VOLUPTATIBUS AUT CUM HIC ET QUAS PERFERENDIS. VOLUPTATES VITAE VENIAM SUNT UT ALIQUID AUT VITAE EXPLICABO. REM ET UT VOLUPTAS.\n\nET IURE MINIMA SOLUTA QUO MODI EXPLICABO. VOLUPTATEM AB ISTE MOLESTIAE SED NIHIL DOLOREM ARCHITECTO DIGNISSIMOS AUT ANIMI SUSCIPIT. LABORIOSAM ID REICIENDIS VOLUPTATUM NESCIUNT ERROR. CORPORIS VOLUPTATIBUS VEL ET ID CUMQUE VERITATIS OMNIS SUNT ET.\n\nNULLA AT PERSPICIATIS DESERUNT RERUM UT AB HARUM PARIATUR QUI. NOBIS UT ENIM AD AT ET VOLUPTATE MINUS CUM QUIDEM SEQUI SED ASPERIORES OFFICIA VELIT MAXIME. SINT EST RATIONE QUI REPELLENDUS ASPERNATUR ET VOLUPTATUM AB VOLUPTATES CONSEQUUNTUR ALIQUID IPSAM. MINUS LABORIOSAM ADIPISCI DOLOREM QUASI FACILIS. IURE IPSAM OMNIS OPTIO DOLOR ET. QUO MAXIME REPELLENDUS IPSA IUSTO ENIM EOS QUIA EST QUAS NEMO ELIGENDI LAUDANTIUM NON. MOLESTIAS VOLUPTATUM FUGIT QUO VOLUPTATE NUMQUAM. NEMO DEBITIS LABORE ET IPSAM MODI RERUM QUIA VOLUPTAS DOLORIBUS SINT IN MINIMA REM.","all_random_upper":"EGUS!471EPB&Y^R4W8FH","whitespaces_upper":"KH IQ T FV O "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"EGUS!471EPB&Y^R4W8FH"} +{"ID":"GsxvM73c6kfGnJJ2bR8ezcIx8NPby8nZAJlzK351AHF9YSiMmsH9FLUCiKRO0HBR","dates":{"date_format1":"018, 18 Jan 2017 02:11:38 GMT+1","date_format2":"2017-01-15T08:05:28","date_format3":"Wed, 18 Jan 2017 13:35:11 +0100","date_format4":"2017-01-17T03:47:48+0100","date_format5":"01-27-2017 16:30","epoch":{"ibm":"18082100.125405","overflow_negative":-18947043201,"overflow_positive":559723816542,"random_negative":-429924121,"random_positive":425597894,"zero":0}},"numerics":{"SmartObject":{"all_random":"mr4zyICN&U0xqB0ChHwM","whitespaces":" ala d lm ","with_new_lines":"Quisquam fugit qui aliquid non consequatur explicabo odio amet quo quia harum non assumenda cumque necessitatibus. Unde eius architecto nesciunt sit. Ducimus quis explicabo ut repellendus voluptas optio. Sit dolores voluptates quia autem animi voluptate. Eligendi eveniet aut et et asperiores amet nemo maxime quisquam qui quis placeat praesentium molestiae ab.\n\nExercitationem beatae ut aut adipisci et iusto eveniet autem maiores enim nostrum fuga possimus. Et reiciendis ut enim velit a dicta animi at eos voluptates praesentium eum omnis. Quis delectus doloribus dignissimos omnis accusamus minus. A assumenda nostrum sint iste. Saepe a voluptatibus est voluptatem vel ducimus explicabo aut fugit enim autem recusandae.\n\nProvident soluta veritatis a dolore reprehenderit atque pariatur et. Id aut doloremque qui rerum reprehenderit eligendi aut dolorum et dolores. Illo perspiciatis voluptatem veniam. Quia fugiat molestiae sed. Dolor qui beatae doloribus nemo non. Sed perferendis deleniti et fugiat qui minima. Ex quod animi dolores eos ullam excepturi amet minima quibusdam necessitatibus delectus."},"big_negative":-21167345327894,"big_positive":6741070543289,"small_negative":-156,"small_positive":207,"zero":0,"small_positive_casted1":"207","small_negative_casted1":"-156","big_positive_casted1":"6741070543289","small_positive_negated":-207,"small_negative_negated":156,"big_positive_negated":-6741070543289,"big_negative_negated":21167345327894},"strings":{"all_random":"qF$WaeXXOguuu^a&TnWk","whitespaces":"xi iai m u bhicj ","with_new_lines":"Quia similique harum laudantium totam. Dicta eos voluptatem voluptas sit rerum voluptatem libero possimus ut repellendus dolor voluptatem. Iure et sit praesentium voluptatibus sit. Tenetur omnis rerum itaque minus exercitationem iusto voluptate ut repellendus ipsum. Eos qui dignissimos consequatur iste rerum et eos laborum et. Saepe quaerat qui sunt molestiae mollitia. Officia rem id est molestiae molestiae ipsa omnis esse temporibus minus corrupti. Odio voluptatem sunt deleniti aut architecto repudiandae error dolor vero at est nobis voluptatem. Quae vero quia aliquam ea accusantium nihil et optio expedita molestiae consequatur et temporibus minus.\n\nMinima a dolorem quo praesentium voluptatem fugit quis corrupti asperiores culpa dignissimos. Iusto ducimus quia quis dicta accusamus perferendis hic sint et corrupti eaque aut. Delectus iste sed nobis eligendi nostrum rem aliquam saepe praesentium consequatur eum et.\n\nFacere eligendi et amet tenetur et. Incidunt deleniti consequatur id voluptas non sunt rerum omnis. Aperiam ut deserunt in dolorem porro molestias quia quidem accusantium ea et placeat non rerum.","with_new_lines_upper":"QUIA SIMILIQUE HARUM LAUDANTIUM TOTAM. DICTA EOS VOLUPTATEM VOLUPTAS SIT RERUM VOLUPTATEM LIBERO POSSIMUS UT REPELLENDUS DOLOR VOLUPTATEM. IURE ET SIT PRAESENTIUM VOLUPTATIBUS SIT. TENETUR OMNIS RERUM ITAQUE MINUS EXERCITATIONEM IUSTO VOLUPTATE UT REPELLENDUS IPSUM. EOS QUI DIGNISSIMOS CONSEQUATUR ISTE RERUM ET EOS LABORUM ET. SAEPE QUAERAT QUI SUNT MOLESTIAE MOLLITIA. OFFICIA REM ID EST MOLESTIAE MOLESTIAE IPSA OMNIS ESSE TEMPORIBUS MINUS CORRUPTI. ODIO VOLUPTATEM SUNT DELENITI AUT ARCHITECTO REPUDIANDAE ERROR DOLOR VERO AT EST NOBIS VOLUPTATEM. QUAE VERO QUIA ALIQUAM EA ACCUSANTIUM NIHIL ET OPTIO EXPEDITA MOLESTIAE CONSEQUATUR ET TEMPORIBUS MINUS.\n\nMINIMA A DOLOREM QUO PRAESENTIUM VOLUPTATEM FUGIT QUIS CORRUPTI ASPERIORES CULPA DIGNISSIMOS. IUSTO DUCIMUS QUIA QUIS DICTA ACCUSAMUS PERFERENDIS HIC SINT ET CORRUPTI EAQUE AUT. DELECTUS ISTE SED NOBIS ELIGENDI NOSTRUM REM ALIQUAM SAEPE PRAESENTIUM CONSEQUATUR EUM ET.\n\nFACERE ELIGENDI ET AMET TENETUR ET. INCIDUNT DELENITI CONSEQUATUR ID VOLUPTAS NON SUNT RERUM OMNIS. APERIAM UT DESERUNT IN DOLOREM PORRO MOLESTIAS QUIA QUIDEM ACCUSANTIUM EA ET PLACEAT NON RERUM.","all_random_upper":"QF$WAEXXOGUUU^A&TNWK","whitespaces_upper":"XI IAI M U BHICJ "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"QF$WAEXXOGUUU^A&TNWK"} +{"ID":"J0c1z4Dj4bBuMMeXHfEKNlGyYbfZIFCqkVACNAsinEPaUnXBwkChIvS2gQkL2FLt","dates":{"date_format1":"028, 28 Jan 2017 23:45:39 GMT+1","date_format2":"2017-01-18T14:50:29","date_format3":"Sun, 29 Jan 2017 03:36:40 +0100","date_format4":"2017-01-24T13:41:57+0100","date_format5":"01-07-2017 15:49","epoch":{"ibm":"25081900.024543","overflow_negative":-4846640306,"overflow_positive":142430475045,"random_negative":-864800827,"random_positive":698068562,"zero":0}},"numerics":{"SmartObject":{"all_random":"r&GzMe9drHH2Rw3ut#1k","whitespaces":" z ue fr ","with_new_lines":"Illum vel dolores enim corporis fugiat et facilis qui saepe dolores autem occaecati. Beatae aliquid consequuntur et ut repellat. Sed culpa voluptas aut saepe voluptatem nostrum molestias velit reprehenderit alias consequatur consequatur deleniti fugit. Iusto et blanditiis delectus et facere. Necessitatibus assumenda atque debitis iusto. Quia maxime aliquid ipsam quam. Cupiditate illo ut est error qui aut. Eius tempora rerum deleniti dolorem qui quidem quae exercitationem eos.\n\nIllum consequatur culpa quibusdam voluptatem totam voluptatibus illum laudantium quidem voluptatem quos commodi unde. Odio et fugiat ex consequatur quas ut rerum dolores. Molestiae eum architecto et voluptas provident illum nihil porro. Facere et et non blanditiis repellendus debitis autem voluptatem et sunt itaque maiores fuga. Qui qui aliquid enim dolorum quo qui laborum error voluptatem id ab. Occaecati qui cumque ducimus quia distinctio quam voluptates iusto et. Ipsam sunt et eligendi porro saepe necessitatibus officia quaerat et qui odit velit officiis. Laboriosam sit rem aliquid sunt quo.\n\nOptio qui facere consectetur optio ullam totam est ab itaque commodi magnam maxime quas expedita qui. Doloremque aut similique accusantium at facere. Fuga modi at quod neque ut quam architecto quia est itaque repellat. Modi voluptatem illo architecto dignissimos maiores. Quae numquam impedit occaecati magnam consequatur veniam ex veniam quisquam inventore. Corporis animi at voluptatem porro magnam fugit saepe ut optio ducimus voluptatem qui alias et fugiat. Rerum omnis dignissimos non ut voluptatem."},"big_negative":-754228964140557,"big_positive":298786847296599,"small_negative":-666,"small_positive":235,"zero":0,"small_positive_casted1":"235","small_negative_casted1":"-666","big_positive_casted1":"298786847296599","small_positive_negated":-235,"small_negative_negated":666,"big_positive_negated":-298786847296599,"big_negative_negated":754228964140557},"strings":{"all_random":"]#0F9X54q)ya[e0FTSRp","whitespaces":" ow ds z cl ","with_new_lines":"Quibusdam et aut quis architecto non aperiam architecto ea odit eveniet aspernatur voluptatum velit occaecati nobis. Porro est dolorem iusto aut nihil. Et aliquid impedit nemo minima maiores maxime repudiandae vel impedit voluptatum corrupti molestiae quia quia. Deserunt odio explicabo est et repellat soluta itaque. Mollitia qui quisquam explicabo dolores voluptatem voluptatem. Autem voluptas iste rem eum harum earum molestiae.\n\nEx autem sint aliquid aut fuga est. Est provident ea et saepe nobis ex et vel tempora dolore quaerat corporis est quasi. Saepe et quas enim illum voluptatem sapiente. Tenetur nihil architecto quod. Fuga natus quia eaque sunt voluptatem. Aut molestiae qui porro veritatis quidem ipsa iure illum earum quaerat mollitia ipsam veniam. Maiores architecto suscipit modi necessitatibus et quidem. Sint ipsum et fugiat aut voluptatem praesentium quasi inventore et qui.\n\nId corrupti ad et ab voluptas laborum. Voluptatum est dolorem possimus rerum est saepe ullam mollitia architecto facilis voluptatem officia. Aliquid quam et ad laudantium qui ut ratione. A culpa voluptatem dolorum sunt suscipit et ut eum sequi corporis perferendis corrupti ea.","with_new_lines_upper":"QUIBUSDAM ET AUT QUIS ARCHITECTO NON APERIAM ARCHITECTO EA ODIT EVENIET ASPERNATUR VOLUPTATUM VELIT OCCAECATI NOBIS. PORRO EST DOLOREM IUSTO AUT NIHIL. ET ALIQUID IMPEDIT NEMO MINIMA MAIORES MAXIME REPUDIANDAE VEL IMPEDIT VOLUPTATUM CORRUPTI MOLESTIAE QUIA QUIA. DESERUNT ODIO EXPLICABO EST ET REPELLAT SOLUTA ITAQUE. MOLLITIA QUI QUISQUAM EXPLICABO DOLORES VOLUPTATEM VOLUPTATEM. AUTEM VOLUPTAS ISTE REM EUM HARUM EARUM MOLESTIAE.\n\nEX AUTEM SINT ALIQUID AUT FUGA EST. EST PROVIDENT EA ET SAEPE NOBIS EX ET VEL TEMPORA DOLORE QUAERAT CORPORIS EST QUASI. SAEPE ET QUAS ENIM ILLUM VOLUPTATEM SAPIENTE. TENETUR NIHIL ARCHITECTO QUOD. FUGA NATUS QUIA EAQUE SUNT VOLUPTATEM. AUT MOLESTIAE QUI PORRO VERITATIS QUIDEM IPSA IURE ILLUM EARUM QUAERAT MOLLITIA IPSAM VENIAM. MAIORES ARCHITECTO SUSCIPIT MODI NECESSITATIBUS ET QUIDEM. SINT IPSUM ET FUGIAT AUT VOLUPTATEM PRAESENTIUM QUASI INVENTORE ET QUI.\n\nID CORRUPTI AD ET AB VOLUPTAS LABORUM. VOLUPTATUM EST DOLOREM POSSIMUS RERUM EST SAEPE ULLAM MOLLITIA ARCHITECTO FACILIS VOLUPTATEM OFFICIA. ALIQUID QUAM ET AD LAUDANTIUM QUI UT RATIONE. A CULPA VOLUPTATEM DOLORUM SUNT SUSCIPIT ET UT EUM SEQUI CORPORIS PERFERENDIS CORRUPTI EA.","all_random_upper":"]#0F9X54Q)YA[E0FTSRP","whitespaces_upper":" OW DS Z CL "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"]#0F9X54Q)YA[E0FTSRP"} +{"ID":"MU56SrFI2eW8ghwmERh8xCfACF6TEGwu44JTpuRpSOXAalzm82rmFgKY405UKEfr","dates":{"date_format1":"009, 09 Jan 2017 09:17:33 GMT+1","date_format2":"2017-01-04T11:08:05","date_format3":"Thu, 19 Jan 2017 14:42:56 +0100","date_format4":"2017-01-19T14:58:05+0100","date_format5":"01-12-2017 05:15","epoch":{"ibm":"30122073.094048","overflow_negative":-62197623203,"overflow_positive":552552370159,"random_negative":-2017949157,"random_positive":2142554516,"zero":0}},"numerics":{"SmartObject":{"all_random":"9Mhm1Rb%2A31D(3xzbYy","whitespaces":" m k ","with_new_lines":"Autem sed quis perferendis vel. Cupiditate est rerum quo adipisci. Sint omnis aut repellat voluptas velit dolores ut perferendis. Enim atque ea numquam. Ut ut totam provident quaerat. Illo qui sit nobis consequatur ea. Quia rerum praesentium nihil facilis ad ad neque nostrum iure est consequatur dolor dolores.\n\nVoluptatem omnis quia dolores ipsum esse et veniam. Temporibus aut dolorem iure ut unde quam molestiae molestiae quia corporis quibusdam. Excepturi aliquid omnis id temporibus. Voluptatum ut laudantium consequatur et qui voluptatem ab quis eos. Enim repellendus quo consequuntur earum neque est sequi vitae at totam sit est. Et esse omnis laboriosam est qui non culpa animi error dicta voluptatem non sed.\n\nQuisquam molestias quis repudiandae magnam sequi similique non aut iste corporis odit assumenda ea. Autem et fugit quo ut ullam omnis illum voluptate architecto. Molestiae illo temporibus autem iure. Praesentium aut ut facilis numquam ipsam odit consectetur modi facilis enim praesentium sint sed."},"big_negative":-703224505589638,"big_positive":592517494751902,"small_negative":-363,"small_positive":909,"zero":0,"small_positive_casted1":"909","small_negative_casted1":"-363","big_positive_casted1":"592517494751902","small_positive_negated":-909,"small_negative_negated":363,"big_positive_negated":-592517494751902,"big_negative_negated":703224505589638},"strings":{"all_random":"#$vxD7pziM@2b#H@SvM8","whitespaces":"j q w v a ","with_new_lines":"Architecto corporis consectetur tenetur est nulla hic et quo sed sed laborum. Qui porro earum ut eligendi eligendi qui blanditiis quidem. Ducimus tenetur dignissimos cupiditate labore velit vero maxime.\n\nEligendi dicta sit iste tempore quo exercitationem sed maiores animi eius. Quos iure et cupiditate temporibus tenetur blanditiis. Eos itaque officia distinctio velit nisi perspiciatis asperiores modi provident quod repudiandae voluptatibus esse. Repellat quia sed facilis quo quia veritatis dolor non.\n\nEaque ex enim quaerat dolor id. Sed repudiandae aut aspernatur quae eveniet ducimus esse sit est exercitationem qui reprehenderit. Omnis voluptatem ut ut accusamus accusamus voluptatem placeat occaecati odio rerum. Dicta adipisci necessitatibus cumque. Aut enim accusantium et ad et qui veritatis aut aut. Dolorem accusantium non laboriosam inventore delectus ut possimus quo non veritatis occaecati. Et et cumque earum ipsam consequuntur ratione sint voluptas omnis magni illum et voluptates. Aliquid non magni voluptatem architecto consequuntur dolores quia qui reprehenderit corrupti in. Repudiandae modi et sunt nulla.","with_new_lines_upper":"ARCHITECTO CORPORIS CONSECTETUR TENETUR EST NULLA HIC ET QUO SED SED LABORUM. QUI PORRO EARUM UT ELIGENDI ELIGENDI QUI BLANDITIIS QUIDEM. DUCIMUS TENETUR DIGNISSIMOS CUPIDITATE LABORE VELIT VERO MAXIME.\n\nELIGENDI DICTA SIT ISTE TEMPORE QUO EXERCITATIONEM SED MAIORES ANIMI EIUS. QUOS IURE ET CUPIDITATE TEMPORIBUS TENETUR BLANDITIIS. EOS ITAQUE OFFICIA DISTINCTIO VELIT NISI PERSPICIATIS ASPERIORES MODI PROVIDENT QUOD REPUDIANDAE VOLUPTATIBUS ESSE. REPELLAT QUIA SED FACILIS QUO QUIA VERITATIS DOLOR NON.\n\nEAQUE EX ENIM QUAERAT DOLOR ID. SED REPUDIANDAE AUT ASPERNATUR QUAE EVENIET DUCIMUS ESSE SIT EST EXERCITATIONEM QUI REPREHENDERIT. OMNIS VOLUPTATEM UT UT ACCUSAMUS ACCUSAMUS VOLUPTATEM PLACEAT OCCAECATI ODIO RERUM. DICTA ADIPISCI NECESSITATIBUS CUMQUE. AUT ENIM ACCUSANTIUM ET AD ET QUI VERITATIS AUT AUT. DOLOREM ACCUSANTIUM NON LABORIOSAM INVENTORE DELECTUS UT POSSIMUS QUO NON VERITATIS OCCAECATI. ET ET CUMQUE EARUM IPSAM CONSEQUUNTUR RATIONE SINT VOLUPTAS OMNIS MAGNI ILLUM ET VOLUPTATES. ALIQUID NON MAGNI VOLUPTATEM ARCHITECTO CONSEQUUNTUR DOLORES QUIA QUI REPREHENDERIT CORRUPTI IN. REPUDIANDAE MODI ET SUNT NULLA.","all_random_upper":"#$VXD7PZIM@2B#H@SVM8","whitespaces_upper":"J Q W V A "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"#$VXD7PZIM@2B#H@SVM8"} +{"ID":"O4Ah3F90sm83fjNa6Y1WI96T4vOHTH9HkI0erEvZW3KG1wXtKXo1hCgzUJ1nH8ZF","dates":{"date_format1":"022, 22 Jan 2017 12:47:55 GMT+1","date_format2":"2017-01-29T11:26:02","date_format3":"Wed, 18 Jan 2017 11:15:52 +0100","date_format4":"2017-01-13T22:05:44+0100","date_format5":"01-20-2017 12:04","epoch":{"ibm":"05062170.163303","overflow_negative":-41512350426,"overflow_positive":599893060239,"random_negative":-2140475137,"random_positive":1959355774,"zero":0}},"numerics":{"SmartObject":{"all_random":"pFAxdyJwNhUNn4]@Sety","whitespaces":"yy w m cd iw","with_new_lines":"Tenetur rerum ad eos aut quo et quo et voluptates et cumque. Totam ut vitae est ad dolor quam quam quia similique aliquid adipisci omnis. Consequatur nihil aperiam eos reprehenderit iusto voluptates aliquid. Ab odio culpa sunt dolore minima consectetur explicabo consequatur perspiciatis est distinctio modi et. Esse voluptatem aut ea odio est porro asperiores dolores consequatur. Et voluptatem excepturi voluptates eaque delectus odit dolorem corporis. Et modi incidunt sit qui rem non laboriosam eos eos. Error laborum in modi porro.\n\nBlanditiis dolorem tempore et voluptas. Fugiat autem ducimus rerum rerum culpa. Laudantium voluptatem et nisi itaque fugiat perspiciatis temporibus vitae dolor est aperiam itaque explicabo iure aut. Voluptatem voluptas enim nesciunt nobis quo eum eos sed. Ex quod distinctio porro aut autem quibusdam atque ad. Et nobis iure sapiente corporis. Officiis neque neque dolores voluptatem expedita et minima. Dolore ullam distinctio quasi cum accusantium velit ut nobis. Et sapiente vel autem iure quia.\n\nQui odit qui et voluptatum nihil dignissimos. Exercitationem ipsa aspernatur blanditiis eaque deserunt aliquid maiores soluta itaque et aspernatur asperiores voluptatum ducimus. Deserunt ut rerum accusamus repudiandae ut tempora ut magnam consequatur ipsum repudiandae. Doloremque maiores autem accusamus quis quia dolor unde et ex. Tempora vel velit quos odio eum esse ullam rerum sunt voluptas dolores."},"big_negative":-837705846023288,"big_positive":790236165808513,"small_negative":-469,"small_positive":952,"zero":0,"small_positive_casted1":"952","small_negative_casted1":"-469","big_positive_casted1":"790236165808513","small_positive_negated":-952,"small_negative_negated":469,"big_positive_negated":-790236165808513,"big_negative_negated":837705846023288},"strings":{"all_random":"U9p$]rD#$)96c3bSU6Ls","whitespaces":" c kw j ml","with_new_lines":"Dolorem odio architecto est eos fugiat possimus sapiente assumenda eum et omnis et. Praesentium libero autem atque nostrum nesciunt quis vitae saepe est quia. Rerum occaecati sapiente eum ut consequuntur id sit maiores et omnis ex laborum dolorem. Facere commodi perspiciatis voluptatem vitae modi eum ut sint aliquid. Quisquam impedit aut tenetur error ad.\n\nSunt vitae reiciendis sapiente ipsa nobis et eveniet voluptatem aut animi omnis. Recusandae quasi ea atque eos est qui aperiam error doloremque aut dolorum aut ratione. Autem magni quia et laboriosam tenetur quidem.\n\nDoloremque minus quis perferendis molestias perferendis illo nostrum possimus voluptates aut similique nemo eos. Sapiente suscipit numquam id. Consequatur qui sapiente ea accusamus. Ea excepturi unde dolor hic aut ullam dolorem maxime cumque provident et qui. Sit et praesentium consequatur dolor atque nemo enim. Tempore fugit ab rerum. Quidem tempore ratione ut possimus doloremque quod et atque.","with_new_lines_upper":"DOLOREM ODIO ARCHITECTO EST EOS FUGIAT POSSIMUS SAPIENTE ASSUMENDA EUM ET OMNIS ET. PRAESENTIUM LIBERO AUTEM ATQUE NOSTRUM NESCIUNT QUIS VITAE SAEPE EST QUIA. RERUM OCCAECATI SAPIENTE EUM UT CONSEQUUNTUR ID SIT MAIORES ET OMNIS EX LABORUM DOLOREM. FACERE COMMODI PERSPICIATIS VOLUPTATEM VITAE MODI EUM UT SINT ALIQUID. QUISQUAM IMPEDIT AUT TENETUR ERROR AD.\n\nSUNT VITAE REICIENDIS SAPIENTE IPSA NOBIS ET EVENIET VOLUPTATEM AUT ANIMI OMNIS. RECUSANDAE QUASI EA ATQUE EOS EST QUI APERIAM ERROR DOLOREMQUE AUT DOLORUM AUT RATIONE. AUTEM MAGNI QUIA ET LABORIOSAM TENETUR QUIDEM.\n\nDOLOREMQUE MINUS QUIS PERFERENDIS MOLESTIAS PERFERENDIS ILLO NOSTRUM POSSIMUS VOLUPTATES AUT SIMILIQUE NEMO EOS. SAPIENTE SUSCIPIT NUMQUAM ID. CONSEQUATUR QUI SAPIENTE EA ACCUSAMUS. EA EXCEPTURI UNDE DOLOR HIC AUT ULLAM DOLOREM MAXIME CUMQUE PROVIDENT ET QUI. SIT ET PRAESENTIUM CONSEQUATUR DOLOR ATQUE NEMO ENIM. TEMPORE FUGIT AB RERUM. QUIDEM TEMPORE RATIONE UT POSSIMUS DOLOREMQUE QUOD ET ATQUE.","all_random_upper":"U9P$]RD#$)96C3BSU6LS","whitespaces_upper":" C KW J ML"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"U9P$]RD#$)96C3BSU6LS"} +{"ID":"PtMbWp3btIB8DtJzFMD4yzo2UWjScrcGeElCrUWgIE1eh0ashM03gyyySNGNqL1f","dates":{"date_format1":"025, 25 Jan 2017 11:32:57 GMT+1","date_format2":"2017-01-20T15:31:23","date_format3":"Mon, 23 Jan 2017 14:17:17 +0100","date_format4":"2017-01-11T12:56:09+0100","date_format5":"01-22-2017 07:33","epoch":{"ibm":"08112090.232624","overflow_negative":-93278114732,"overflow_positive":284580294330,"random_negative":-161671971,"random_positive":1076591524,"zero":0}},"numerics":{"SmartObject":{"all_random":"yQOnC%NP^f3cLda!efkQ","whitespaces":" t r e t z ","with_new_lines":"Aut vel error eius omnis. Et cumque eius cupiditate ut esse distinctio dolorem suscipit sint minima nulla quae et omnis. Qui ullam fugit quia facere officia vel repudiandae. Natus nemo modi et labore labore nulla. Quos eos architecto consequuntur aut veritatis dolorem necessitatibus quae aut vitae quasi recusandae officiis velit nihil.\n\nOccaecati quos nobis est delectus voluptatem magni. Neque dicta ducimus qui fugiat ex animi autem et rem ea. Cum omnis consectetur incidunt est ea suscipit quos. Voluptas qui omnis numquam quibusdam et et assumenda aliquam explicabo consequatur esse accusantium. Qui dicta et sed necessitatibus dicta aut. Et et et est quam veniam voluptatem maxime quos repellendus consequatur quisquam molestias id.\n\nLibero sint consequatur sed sapiente aut asperiores est. Et et impedit labore quo ducimus eum. Dolores culpa numquam nihil et totam eius aut voluptatibus vel nesciunt. Neque assumenda quaerat optio aut quisquam et. Ipsum impedit quae est perferendis neque quia nesciunt iste dolores est rem alias voluptas. Id non esse et neque."},"big_negative":-642294164196098,"big_positive":101359425794559,"small_negative":-846,"small_positive":239,"zero":0,"small_positive_casted1":"239","small_negative_casted1":"-846","big_positive_casted1":"101359425794559","small_positive_negated":-239,"small_negative_negated":846,"big_positive_negated":-101359425794559,"big_negative_negated":642294164196098},"strings":{"all_random":"1UX@E)KEIy4Xl3vOAeT3","whitespaces":"k c b lc ","with_new_lines":"Consectetur et qui est voluptatum itaque nesciunt nisi. Recusandae est beatae dicta dicta facere hic atque nisi aut natus modi assumenda. Nesciunt in a in quae ab fuga laboriosam quia ea. Nam qui mollitia debitis tenetur et et voluptatem rem facilis laborum id eos.\n\nQuam magni itaque ipsam est qui reiciendis itaque dignissimos et vel et vitae facilis maiores. Ratione fugit quod odio dicta voluptates atque laudantium rem ut et iure laborum quos exercitationem enim. Dolorum ut quidem omnis nisi. Voluptatem ab laudantium ducimus dolor autem eius.\n\nUt pariatur officia aspernatur fugiat dolorem dignissimos adipisci esse ut neque sint eius est quia enim. Eos quo quod assumenda commodi officia suscipit sunt minus in optio nobis aut molestiae aperiam. Amet sit libero dolor aut deleniti autem cum officiis molestias aut. Illo impedit ullam dolorem expedita dolor culpa sapiente ipsum et quo voluptatem odit necessitatibus consequatur.","with_new_lines_upper":"CONSECTETUR ET QUI EST VOLUPTATUM ITAQUE NESCIUNT NISI. RECUSANDAE EST BEATAE DICTA DICTA FACERE HIC ATQUE NISI AUT NATUS MODI ASSUMENDA. NESCIUNT IN A IN QUAE AB FUGA LABORIOSAM QUIA EA. NAM QUI MOLLITIA DEBITIS TENETUR ET ET VOLUPTATEM REM FACILIS LABORUM ID EOS.\n\nQUAM MAGNI ITAQUE IPSAM EST QUI REICIENDIS ITAQUE DIGNISSIMOS ET VEL ET VITAE FACILIS MAIORES. RATIONE FUGIT QUOD ODIO DICTA VOLUPTATES ATQUE LAUDANTIUM REM UT ET IURE LABORUM QUOS EXERCITATIONEM ENIM. DOLORUM UT QUIDEM OMNIS NISI. VOLUPTATEM AB LAUDANTIUM DUCIMUS DOLOR AUTEM EIUS.\n\nUT PARIATUR OFFICIA ASPERNATUR FUGIAT DOLOREM DIGNISSIMOS ADIPISCI ESSE UT NEQUE SINT EIUS EST QUIA ENIM. EOS QUO QUOD ASSUMENDA COMMODI OFFICIA SUSCIPIT SUNT MINUS IN OPTIO NOBIS AUT MOLESTIAE APERIAM. AMET SIT LIBERO DOLOR AUT DELENITI AUTEM CUM OFFICIIS MOLESTIAS AUT. ILLO IMPEDIT ULLAM DOLOREM EXPEDITA DOLOR CULPA SAPIENTE IPSUM ET QUO VOLUPTATEM ODIT NECESSITATIBUS CONSEQUATUR.","all_random_upper":"1UX@E)KEIY4XL3VOAET3","whitespaces_upper":"K C B LC "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"1UX@E)KEIY4XL3VOAET3"} +{"ID":"Q6ErqaV1EPSnrh1mn71mFMslXzxGF9q5wdwBalOS9TfHkIt5fzmihosNNV1Hm3m4","dates":{"date_format1":"014, 14 Jan 2017 00:49:38 GMT+1","date_format2":"2017-01-01T14:58:56","date_format3":"Tue, 17 Jan 2017 11:11:44 +0100","date_format4":"2017-01-13T01:50:43+0100","date_format5":"01-21-2017 05:33","epoch":{"ibm":"12011939.213516","overflow_negative":-63177790359,"overflow_positive":955792767130,"random_negative":-873701701,"random_positive":932829157,"zero":0}},"numerics":{"SmartObject":{"all_random":"Yyftzp375D&Uc7^)6REC","whitespaces":"x t y tgk ","with_new_lines":"Consequuntur impedit est voluptates repellat. Eos expedita quaerat aspernatur aspernatur. Repellendus pariatur sint nisi explicabo quisquam est id dignissimos sed fugit ea. Consectetur libero sunt quaerat sed est et cum nisi voluptatibus omnis quo rerum et distinctio. Quae placeat dolores voluptates tempora inventore sunt ea ut et et magnam. Rerum vel quia aut non sed velit deserunt aut. Alias officia adipisci sint ut ut facere suscipit et id.\n\nIpsam iusto explicabo eum repudiandae. Et dolor quasi consequatur rerum eaque laboriosam beatae dolores nam. Eveniet illo qui ducimus quidem dignissimos illum dolorem porro. Est sunt dolorem ab rem eos minima ex vero. Quis atque qui accusantium cupiditate error. Nemo ut ut praesentium et qui est omnis minus eveniet eaque iure neque et. Consequatur et libero cumque incidunt voluptatem voluptatem rerum aut voluptatem repellat eum vitae.\n\nA delectus veniam officia nam aut expedita omnis distinctio ratione itaque aliquam rerum qui quia. Totam veniam sapiente eligendi possimus et in occaecati non. Dolorem necessitatibus eum dolor reiciendis voluptatem sint fugit aperiam esse. Eum sapiente iusto ab aut ipsum consequatur ut quibusdam id quidem soluta. Quae quasi sint cum voluptas qui."},"big_negative":-939362949106272,"big_positive":791949411397450,"small_negative":-618,"small_positive":550,"zero":0,"small_positive_casted1":"550","small_negative_casted1":"-618","big_positive_casted1":"791949411397450","small_positive_negated":-550,"small_negative_negated":618,"big_positive_negated":-791949411397450,"big_negative_negated":939362949106272},"strings":{"all_random":"GvTalZII(^Fk$h[HU!88","whitespaces":" t e t rp z p","with_new_lines":"Voluptatem est omnis odit officiis et. Laudantium inventore voluptatibus itaque provident commodi suscipit modi est et dolores et doloremque quas sed. Commodi perferendis illum omnis officia natus qui consequatur recusandae. Quo eum velit sint et modi. Sequi tempora fugiat voluptatem perferendis laborum accusantium culpa excepturi. Eos ex quisquam culpa atque voluptates eos illum repudiandae eum. Perferendis facere dolores et velit vel.\n\nRem nulla tempora ut neque voluptas ad sint non repellat modi quae. Rerum delectus recusandae qui rem labore pariatur qui voluptatem eligendi cum ipsam recusandae ratione et. Qui rerum omnis deleniti quo ex qui molestiae nihil maxime ipsum. Perferendis placeat ut ipsum sapiente rerum fuga facere et iusto omnis voluptas assumenda saepe. Sit voluptate asperiores deserunt incidunt et pariatur nesciunt minima deserunt nam. Quas tempore ea quia necessitatibus aut qui asperiores perspiciatis qui est quasi et. Est ea ad ut maxime fugiat ut repellat quidem aperiam hic dolor eum tenetur sed. Ut nihil architecto nemo eum fugit.\n\nSint voluptatem libero rem molestias. Ipsa molestiae et praesentium qui laudantium porro adipisci. Numquam qui exercitationem necessitatibus nulla velit. Eos fugit aut doloremque dolores sit minima consequatur tenetur consequuntur est consectetur. Ullam voluptatum repellendus magnam voluptatem delectus accusantium ad magni et autem iure officia dolor quibusdam.","with_new_lines_upper":"VOLUPTATEM EST OMNIS ODIT OFFICIIS ET. LAUDANTIUM INVENTORE VOLUPTATIBUS ITAQUE PROVIDENT COMMODI SUSCIPIT MODI EST ET DOLORES ET DOLOREMQUE QUAS SED. COMMODI PERFERENDIS ILLUM OMNIS OFFICIA NATUS QUI CONSEQUATUR RECUSANDAE. QUO EUM VELIT SINT ET MODI. SEQUI TEMPORA FUGIAT VOLUPTATEM PERFERENDIS LABORUM ACCUSANTIUM CULPA EXCEPTURI. EOS EX QUISQUAM CULPA ATQUE VOLUPTATES EOS ILLUM REPUDIANDAE EUM. PERFERENDIS FACERE DOLORES ET VELIT VEL.\n\nREM NULLA TEMPORA UT NEQUE VOLUPTAS AD SINT NON REPELLAT MODI QUAE. RERUM DELECTUS RECUSANDAE QUI REM LABORE PARIATUR QUI VOLUPTATEM ELIGENDI CUM IPSAM RECUSANDAE RATIONE ET. QUI RERUM OMNIS DELENITI QUO EX QUI MOLESTIAE NIHIL MAXIME IPSUM. PERFERENDIS PLACEAT UT IPSUM SAPIENTE RERUM FUGA FACERE ET IUSTO OMNIS VOLUPTAS ASSUMENDA SAEPE. SIT VOLUPTATE ASPERIORES DESERUNT INCIDUNT ET PARIATUR NESCIUNT MINIMA DESERUNT NAM. QUAS TEMPORE EA QUIA NECESSITATIBUS AUT QUI ASPERIORES PERSPICIATIS QUI EST QUASI ET. EST EA AD UT MAXIME FUGIAT UT REPELLAT QUIDEM APERIAM HIC DOLOR EUM TENETUR SED. UT NIHIL ARCHITECTO NEMO EUM FUGIT.\n\nSINT VOLUPTATEM LIBERO REM MOLESTIAS. IPSA MOLESTIAE ET PRAESENTIUM QUI LAUDANTIUM PORRO ADIPISCI. NUMQUAM QUI EXERCITATIONEM NECESSITATIBUS NULLA VELIT. EOS FUGIT AUT DOLOREMQUE DOLORES SIT MINIMA CONSEQUATUR TENETUR CONSEQUUNTUR EST CONSECTETUR. ULLAM VOLUPTATUM REPELLENDUS MAGNAM VOLUPTATEM DELECTUS ACCUSANTIUM AD MAGNI ET AUTEM IURE OFFICIA DOLOR QUIBUSDAM.","all_random_upper":"GVTALZII(^FK$H[HU!88","whitespaces_upper":" T E T RP Z P"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"GVTALZII(^FK$H[HU!88"} +{"ID":"Wn3fmwTUNEzpDrCWjJuhmQks8BrfIwpKYcw0pNzuXy9klVjjEp5OStQVJFQGHyF2","dates":{"date_format1":"013, 13 Jan 2017 00:03:19 GMT+1","date_format2":"2017-01-07T23:31:54","date_format3":"Tue, 10 Jan 2017 00:27:26 +0100","date_format4":"2017-01-08T07:54:11+0100","date_format5":"01-24-2017 23:13","epoch":{"ibm":"23122033.121221","overflow_negative":-13889955259,"overflow_positive":834233552094,"random_negative":-1701775404,"random_positive":870124027,"zero":0}},"numerics":{"SmartObject":{"all_random":"j@qKy]fMGDelXj8dyWvO","whitespaces":" v k wy rv y e ","with_new_lines":"Et ut harum necessitatibus temporibus ad vero minima. Fugiat enim sit magnam unde. Doloribus voluptatem odio consequuntur labore officiis fugit quae aliquam minima aut itaque quisquam consequatur aut. Voluptatem temporibus eum in ad quia dignissimos quo reprehenderit vel suscipit ratione asperiores perferendis. Fugiat quod aliquid et. Aut neque nemo est culpa vero illum soluta. Ut vero quisquam quia recusandae.\n\nAut esse quisquam nam corporis sit itaque est laborum qui earum fuga itaque in. Dolorem et quaerat assumenda odit fugit culpa minus modi laudantium repellendus tenetur accusamus iusto neque exercitationem. Similique a sapiente ipsa aut dignissimos corporis quia eaque neque fugiat. Aspernatur quo beatae commodi dolorem accusantium exercitationem quibusdam quam debitis alias id rerum nihil enim eos.\n\nEaque quis neque quo est. Est adipisci distinctio omnis sint aspernatur qui sapiente unde vel quidem. Adipisci odit eos ut beatae. Impedit corrupti culpa consequatur tempore ut nihil quae."},"big_negative":-399003411789975,"big_positive":634724217882648,"small_negative":-286,"small_positive":941,"zero":0,"small_positive_casted1":"941","small_negative_casted1":"-286","big_positive_casted1":"634724217882648","small_positive_negated":-941,"small_negative_negated":286,"big_positive_negated":-634724217882648,"big_negative_negated":399003411789975},"strings":{"all_random":"WC[puZYHi*%0KutOQdQJ","whitespaces":"sde jv u ","with_new_lines":"Fugiat tempore vitae mollitia quos. Quia hic et et ab ut aut est perspiciatis ut unde. Suscipit qui perferendis adipisci impedit ipsum veritatis aperiam delectus quaerat et assumenda blanditiis beatae. Aliquam dolor vel iste. Sequi non enim nihil nisi rem consequatur optio voluptatem ad qui.\n\nAccusantium nemo excepturi laudantium magni veniam non voluptatum. Corporis molestiae autem ut sed. Ea odio aut velit eum cumque. Ut impedit omnis quaerat nesciunt eaque eos amet autem quidem. Quaerat mollitia occaecati natus ad voluptatem dolores eligendi impedit laudantium. Ullam voluptatum aut nulla provident sed officiis eos voluptatem architecto tenetur voluptatem enim autem. Quod iste non est rerum culpa neque magni reprehenderit in eaque fuga quisquam dolor labore minima. Cum earum aspernatur similique est aut et eveniet enim. Iste alias omnis rem ut optio animi voluptates.\n\nIusto tenetur eos quibusdam est nisi ipsam molestiae nisi impedit nobis mollitia. Minus est temporibus aut omnis a minus nobis architecto odio itaque aut distinctio error quis. Quia officiis suscipit non dicta.","with_new_lines_upper":"FUGIAT TEMPORE VITAE MOLLITIA QUOS. QUIA HIC ET ET AB UT AUT EST PERSPICIATIS UT UNDE. SUSCIPIT QUI PERFERENDIS ADIPISCI IMPEDIT IPSUM VERITATIS APERIAM DELECTUS QUAERAT ET ASSUMENDA BLANDITIIS BEATAE. ALIQUAM DOLOR VEL ISTE. SEQUI NON ENIM NIHIL NISI REM CONSEQUATUR OPTIO VOLUPTATEM AD QUI.\n\nACCUSANTIUM NEMO EXCEPTURI LAUDANTIUM MAGNI VENIAM NON VOLUPTATUM. CORPORIS MOLESTIAE AUTEM UT SED. EA ODIO AUT VELIT EUM CUMQUE. UT IMPEDIT OMNIS QUAERAT NESCIUNT EAQUE EOS AMET AUTEM QUIDEM. QUAERAT MOLLITIA OCCAECATI NATUS AD VOLUPTATEM DOLORES ELIGENDI IMPEDIT LAUDANTIUM. ULLAM VOLUPTATUM AUT NULLA PROVIDENT SED OFFICIIS EOS VOLUPTATEM ARCHITECTO TENETUR VOLUPTATEM ENIM AUTEM. QUOD ISTE NON EST RERUM CULPA NEQUE MAGNI REPREHENDERIT IN EAQUE FUGA QUISQUAM DOLOR LABORE MINIMA. CUM EARUM ASPERNATUR SIMILIQUE EST AUT ET EVENIET ENIM. ISTE ALIAS OMNIS REM UT OPTIO ANIMI VOLUPTATES.\n\nIUSTO TENETUR EOS QUIBUSDAM EST NISI IPSAM MOLESTIAE NISI IMPEDIT NOBIS MOLLITIA. MINUS EST TEMPORIBUS AUT OMNIS A MINUS NOBIS ARCHITECTO ODIO ITAQUE AUT DISTINCTIO ERROR QUIS. QUIA OFFICIIS SUSCIPIT NON DICTA.","all_random_upper":"WC[PUZYHI*%0KUTOQDQJ","whitespaces_upper":"SDE JV U "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"WC[PUZYHI*%0KUTOQDQJ"} +{"ID":"Y3N4w0o5Rul4WwW5gRfj90WpDl4BnWcZv3M8sRGcU4mj07XvC29VqMMEEJuEIXo3","dates":{"date_format1":"015, 15 Jan 2017 09:05:51 GMT+1","date_format2":"2017-01-17T02:00:02","date_format3":"Wed, 25 Jan 2017 01:18:27 +0100","date_format4":"2017-01-11T16:31:17+0100","date_format5":"01-19-2017 13:58","epoch":{"ibm":"24032082.103316","overflow_negative":-87989020778,"overflow_positive":943467953930,"random_negative":-1433633112,"random_positive":1347545941,"zero":0}},"numerics":{"SmartObject":{"all_random":"$9fG9Rw2evPg@Ar)DQ(Y","whitespaces":" j gh y cq ","with_new_lines":"Qui minus ea provident aliquam temporibus. Facilis veritatis sit officiis quo vel sed sed in laborum. Quia culpa enim in sed velit quae inventore impedit totam voluptate dicta. Voluptas illum quos placeat quae itaque accusamus incidunt non eligendi sint quisquam dolor eius iste. Sit consequatur tenetur velit ipsum ut ratione qui totam et voluptatem fuga ut repellendus dolores sint. Tempora illo cum rerum quia enim veniam et. Nihil omnis amet labore aut tempore.\n\nEst dolore quos nisi nihil commodi. Voluptatem quia laudantium autem eaque reiciendis. Earum ut nihil et qui ullam. Laborum maiores itaque dolorum qui at ad tempora et deleniti tempore ratione. Ut quidem qui non quaerat et voluptatem enim aperiam qui adipisci adipisci est. Qui accusantium blanditiis maiores et quis culpa neque. Debitis et est voluptatem quia sit.\n\nQuis quia quae dicta. Id sed molestiae vitae debitis eum earum voluptas sit reprehenderit quia non. Necessitatibus commodi laboriosam adipisci ratione vero. Dolores perspiciatis ullam voluptatum dolore eius non illum dolorem provident autem deleniti labore corporis officia."},"big_negative":-684718954884696,"big_positive":768990048149640,"small_negative":-134,"small_positive":268,"zero":0,"small_positive_casted1":"268","small_negative_casted1":"-134","big_positive_casted1":"768990048149640","small_positive_negated":-268,"small_negative_negated":134,"big_positive_negated":-768990048149640,"big_negative_negated":684718954884696},"strings":{"all_random":"dA96FCG*pb$8%oedpjm5","whitespaces":" ifvmx w rc","with_new_lines":"Quia numquam deserunt delectus rem est totam ea culpa quas excepturi est. Architecto ab sit reprehenderit laudantium aut sapiente adipisci non cupiditate adipisci repellat eligendi. Aperiam enim repudiandae laudantium ut assumenda quo rerum asperiores rem odit odit necessitatibus fugiat.\n\nEt tempore quam aut sequi. Quia consequatur et exercitationem illum esse suscipit. Iste aut nihil nostrum quibusdam ea odit dolor expedita itaque. Aut et et minima ipsum sit dignissimos ad. Dolores voluptates hic aut autem dolor delectus asperiores laudantium voluptate. Minima fugiat voluptatem et recusandae asperiores nulla qui laborum sit est porro illum ea est ullam. Ullam doloribus odio quisquam laborum. Vitae quo repellat laudantium quibusdam sequi enim dolor odit quibusdam ipsum rerum. Amet alias voluptatem ut omnis tenetur et voluptatibus temporibus ut iure sunt enim eos.\n\nQuaerat magni aut repellat numquam et enim neque rerum quisquam explicabo facere nam. Debitis quaerat nobis suscipit repellendus aut expedita voluptatem voluptatibus laboriosam dignissimos. Dicta ratione fugiat est labore adipisci qui aut velit dolorum occaecati dolores. Deleniti commodi autem mollitia sunt sequi et qui quo.","with_new_lines_upper":"QUIA NUMQUAM DESERUNT DELECTUS REM EST TOTAM EA CULPA QUAS EXCEPTURI EST. ARCHITECTO AB SIT REPREHENDERIT LAUDANTIUM AUT SAPIENTE ADIPISCI NON CUPIDITATE ADIPISCI REPELLAT ELIGENDI. APERIAM ENIM REPUDIANDAE LAUDANTIUM UT ASSUMENDA QUO RERUM ASPERIORES REM ODIT ODIT NECESSITATIBUS FUGIAT.\n\nET TEMPORE QUAM AUT SEQUI. QUIA CONSEQUATUR ET EXERCITATIONEM ILLUM ESSE SUSCIPIT. ISTE AUT NIHIL NOSTRUM QUIBUSDAM EA ODIT DOLOR EXPEDITA ITAQUE. AUT ET ET MINIMA IPSUM SIT DIGNISSIMOS AD. DOLORES VOLUPTATES HIC AUT AUTEM DOLOR DELECTUS ASPERIORES LAUDANTIUM VOLUPTATE. MINIMA FUGIAT VOLUPTATEM ET RECUSANDAE ASPERIORES NULLA QUI LABORUM SIT EST PORRO ILLUM EA EST ULLAM. ULLAM DOLORIBUS ODIO QUISQUAM LABORUM. VITAE QUO REPELLAT LAUDANTIUM QUIBUSDAM SEQUI ENIM DOLOR ODIT QUIBUSDAM IPSUM RERUM. AMET ALIAS VOLUPTATEM UT OMNIS TENETUR ET VOLUPTATIBUS TEMPORIBUS UT IURE SUNT ENIM EOS.\n\nQUAERAT MAGNI AUT REPELLAT NUMQUAM ET ENIM NEQUE RERUM QUISQUAM EXPLICABO FACERE NAM. DEBITIS QUAERAT NOBIS SUSCIPIT REPELLENDUS AUT EXPEDITA VOLUPTATEM VOLUPTATIBUS LABORIOSAM DIGNISSIMOS. DICTA RATIONE FUGIAT EST LABORE ADIPISCI QUI AUT VELIT DOLORUM OCCAECATI DOLORES. DELENITI COMMODI AUTEM MOLLITIA SUNT SEQUI ET QUI QUO.","all_random_upper":"DA96FCG*PB$8%OEDPJM5","whitespaces_upper":" IFVMX W RC"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"DA96FCG*PB$8%OEDPJM5"} +{"ID":"fJWkrTHmF9Dy0Ebf5TKV2I4Ky2RdSageoF9mw1MQLTNDagX2xESbT7VbqjEJhyGU","dates":{"date_format1":"009, 09 Jan 2017 07:06:14 GMT+1","date_format2":"2017-01-12T13:51:55","date_format3":"Fri, 13 Jan 2017 05:32:05 +0100","date_format4":"2017-01-11T03:33:18+0100","date_format5":"01-02-2017 19:35","epoch":{"ibm":"26052072.020200","overflow_negative":-28179187981,"overflow_positive":582963229667,"random_negative":-2105169562,"random_positive":1107748934,"zero":0}},"numerics":{"SmartObject":{"all_random":"rvPVt$Y#chul4C$AkdKX","whitespaces":"t a mfy bkf x","with_new_lines":"Aspernatur iusto velit qui adipisci atque aliquam aliquam impedit laborum minus ratione. Et soluta iure deserunt autem et qui sapiente id voluptatem aliquam. Cumque incidunt et itaque enim voluptatem nesciunt quo similique debitis aliquam id omnis. Voluptatum illo ex quas animi ratione repellendus et. Ut deserunt ad dolores temporibus. Unde laborum dolore cupiditate molestiae nihil similique rerum deleniti. Perferendis aspernatur itaque optio fuga suscipit autem ratione sed explicabo sequi autem sint.\n\nFuga quibusdam molestias quo iusto quis dignissimos et ratione numquam nulla amet eos illo natus occaecati. Quae et esse consectetur commodi in maiores optio eius voluptatem animi asperiores nihil. Minima nostrum sit et esse eum ut unde dicta consectetur qui laboriosam occaecati assumenda. Ea quo ut corrupti atque reprehenderit laborum rerum perspiciatis. Impedit ut tempore earum aut amet tempore aperiam hic ab sed est odio neque consectetur. Ut est quis doloremque adipisci inventore. Saepe qui quia ut consequuntur quia voluptas porro voluptatum et eius excepturi. Error excepturi omnis non asperiores consectetur nihil suscipit culpa debitis laboriosam. Eos velit qui et maiores cupiditate ex totam molestiae eveniet.\n\nLibero voluptates qui earum quidem consequatur molestias ipsa et inventore autem. Fugit saepe ut aut est sit molestiae dicta delectus laborum dolorem rerum impedit. Non totam iusto quia. Vitae enim nihil et aut officiis qui nulla ut voluptatem iusto suscipit deserunt qui. Dolore molestias sed hic facilis nam sequi blanditiis fugit deserunt nulla quia voluptatum."},"big_negative":-161176863305841,"big_positive":669223368251997,"small_negative":-660,"small_positive":722,"zero":0,"small_positive_casted1":"722","small_negative_casted1":"-660","big_positive_casted1":"669223368251997","small_positive_negated":-722,"small_negative_negated":660,"big_positive_negated":-669223368251997,"big_negative_negated":161176863305841},"strings":{"all_random":"DESebo8d%fL9sX@AzVin","whitespaces":" q bb l ","with_new_lines":"Culpa repellat nesciunt accusantium mollitia fuga. Nesciunt iusto est dignissimos eveniet consequatur molestiae voluptate sapiente architecto sit eius ab earum. Consequatur atque laborum eius deleniti sunt et officiis suscipit tempora quibusdam. Beatae et minima et. Fuga tenetur vel cumque eos perferendis. Minima deserunt nostrum excepturi qui possimus adipisci ratione tenetur praesentium quia et temporibus. Dolorem expedita possimus corrupti ratione dignissimos aliquam voluptas officiis ad impedit ex sit deserunt illo.\n\nConsequatur dolorem et odit maiores sit tempore repudiandae amet facilis. Amet et ipsam unde ratione adipisci voluptas adipisci inventore omnis nobis excepturi dolore tenetur modi. Minima temporibus officiis consectetur qui accusamus quia. Molestiae iure quibusdam totam animi. Minus et autem est.\n\nQuis dolorem a illum et quas quae sit architecto perferendis dolorum. Et natus omnis rerum omnis. Ab sapiente quam tenetur facilis dicta omnis repellendus accusamus voluptates. Harum repellat vel nihil et porro. Suscipit pariatur adipisci dolorem. Modi ut sint et ducimus voluptatem voluptate consequatur et recusandae corporis amet cum error doloribus dolore.","with_new_lines_upper":"CULPA REPELLAT NESCIUNT ACCUSANTIUM MOLLITIA FUGA. NESCIUNT IUSTO EST DIGNISSIMOS EVENIET CONSEQUATUR MOLESTIAE VOLUPTATE SAPIENTE ARCHITECTO SIT EIUS AB EARUM. CONSEQUATUR ATQUE LABORUM EIUS DELENITI SUNT ET OFFICIIS SUSCIPIT TEMPORA QUIBUSDAM. BEATAE ET MINIMA ET. FUGA TENETUR VEL CUMQUE EOS PERFERENDIS. MINIMA DESERUNT NOSTRUM EXCEPTURI QUI POSSIMUS ADIPISCI RATIONE TENETUR PRAESENTIUM QUIA ET TEMPORIBUS. DOLOREM EXPEDITA POSSIMUS CORRUPTI RATIONE DIGNISSIMOS ALIQUAM VOLUPTAS OFFICIIS AD IMPEDIT EX SIT DESERUNT ILLO.\n\nCONSEQUATUR DOLOREM ET ODIT MAIORES SIT TEMPORE REPUDIANDAE AMET FACILIS. AMET ET IPSAM UNDE RATIONE ADIPISCI VOLUPTAS ADIPISCI INVENTORE OMNIS NOBIS EXCEPTURI DOLORE TENETUR MODI. MINIMA TEMPORIBUS OFFICIIS CONSECTETUR QUI ACCUSAMUS QUIA. MOLESTIAE IURE QUIBUSDAM TOTAM ANIMI. MINUS ET AUTEM EST.\n\nQUIS DOLOREM A ILLUM ET QUAS QUAE SIT ARCHITECTO PERFERENDIS DOLORUM. ET NATUS OMNIS RERUM OMNIS. AB SAPIENTE QUAM TENETUR FACILIS DICTA OMNIS REPELLENDUS ACCUSAMUS VOLUPTATES. HARUM REPELLAT VEL NIHIL ET PORRO. SUSCIPIT PARIATUR ADIPISCI DOLOREM. MODI UT SINT ET DUCIMUS VOLUPTATEM VOLUPTATE CONSEQUATUR ET RECUSANDAE CORPORIS AMET CUM ERROR DOLORIBUS DOLORE.","all_random_upper":"DESEBO8D%FL9SX@AZVIN","whitespaces_upper":" Q BB L "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"DESEBO8D%FL9SX@AZVIN"} +{"ID":"gNFKJ6qe6wd1lGrjooLrruDokqjaVmBfwdcv2SBqrqv0UUlATjvtflnZcBDhHDyd","dates":{"date_format1":"003, 03 Jan 2017 12:20:55 GMT+1","date_format2":"2017-01-14T14:20:36","date_format3":"Mon, 23 Jan 2017 08:01:42 +0100","date_format4":"2017-01-18T05:24:45+0100","date_format5":"01-08-2017 00:44","epoch":{"ibm":"06082179.092348","overflow_negative":-17148196670,"overflow_positive":889245411926,"random_negative":-369502968,"random_positive":1578154726,"zero":0}},"numerics":{"SmartObject":{"all_random":"NX3US$kJr@!T7cTE%34#","whitespaces":"rq hjc r yq r f","with_new_lines":"Inventore et minus rerum quia dolor sunt sit vitae possimus non et nihil reprehenderit modi ea. Repellendus aut nemo odio nostrum consequuntur corporis. Odio officia tempore animi et beatae quia animi. Modi dolores minima quod quae labore culpa molestiae quod laboriosam quaerat quos sapiente voluptatum sunt.\n\nEst nihil quod nisi nemo rem ipsa harum nulla iusto nisi qui tempora dolorum sunt. Quae earum occaecati voluptate similique et nulla iusto. Assumenda quidem quae omnis id necessitatibus sed et ut quis reiciendis soluta adipisci exercitationem rerum expedita. Totam ea temporibus modi quisquam quisquam quo vitae voluptatem praesentium. Qui officia placeat ratione sit et aut adipisci laudantium rerum enim ut officiis voluptas et similique. Voluptatum ea quod aut sit inventore esse velit quae quia.\n\nEaque ducimus doloremque est qui quam reiciendis sint facilis. Et laborum id et. Sunt molestiae ab eum soluta voluptatibus dignissimos perspiciatis et sed nobis laboriosam neque necessitatibus qui. Esse officiis et sint ut hic. Sapiente ut voluptatem rerum corporis temporibus praesentium. Quis quia est consequatur quia suscipit non unde molestiae porro ullam odit atque earum omnis. Error est eum maiores nesciunt officiis error laboriosam. Occaecati a perferendis facilis perspiciatis. Architecto qui commodi necessitatibus est eos ut eum beatae deserunt recusandae."},"big_negative":-658158151699432,"big_positive":574505272908509,"small_negative":-594,"small_positive":429,"zero":0,"small_positive_casted1":"429","small_negative_casted1":"-594","big_positive_casted1":"574505272908509","small_positive_negated":-429,"small_negative_negated":594,"big_positive_negated":-574505272908509,"big_negative_negated":658158151699432},"strings":{"all_random":"vHX6N[inTV[&YrIaTto5","whitespaces":" bge xo f s q g","with_new_lines":"Velit velit ad ex fuga sit quis. Velit enim repudiandae ipsum vero qui perferendis occaecati eum quae et consequuntur beatae voluptas. Esse impedit quo quasi exercitationem tenetur exercitationem repellendus. Iste repellat id architecto ut molestiae. Eius neque aspernatur qui saepe ad qui beatae earum ut est sunt ipsum dolor. Earum dolorem autem aliquam voluptatum.\n\nQui dolorum voluptatibus tempora explicabo. Et in ipsam numquam aliquam saepe necessitatibus modi repudiandae maiores et pariatur molestiae eum aut. Nobis qui exercitationem culpa iure nam nihil porro perferendis praesentium ullam. Ut hic tempora sint in est. Sunt sed temporibus quis ut iste reprehenderit in sunt laudantium. Possimus sequi quia recusandae voluptas non quos facilis minus perferendis.\n\nQuisquam laboriosam rerum nam eligendi omnis consequuntur. Aut ab praesentium reprehenderit aspernatur sed occaecati illo dolorum ut ratione aut labore odit rerum. Sapiente quia ex nihil incidunt inventore libero sed recusandae omnis consequatur enim. Dolores enim expedita ea. Nostrum aspernatur earum omnis.","with_new_lines_upper":"VELIT VELIT AD EX FUGA SIT QUIS. VELIT ENIM REPUDIANDAE IPSUM VERO QUI PERFERENDIS OCCAECATI EUM QUAE ET CONSEQUUNTUR BEATAE VOLUPTAS. ESSE IMPEDIT QUO QUASI EXERCITATIONEM TENETUR EXERCITATIONEM REPELLENDUS. ISTE REPELLAT ID ARCHITECTO UT MOLESTIAE. EIUS NEQUE ASPERNATUR QUI SAEPE AD QUI BEATAE EARUM UT EST SUNT IPSUM DOLOR. EARUM DOLOREM AUTEM ALIQUAM VOLUPTATUM.\n\nQUI DOLORUM VOLUPTATIBUS TEMPORA EXPLICABO. ET IN IPSAM NUMQUAM ALIQUAM SAEPE NECESSITATIBUS MODI REPUDIANDAE MAIORES ET PARIATUR MOLESTIAE EUM AUT. NOBIS QUI EXERCITATIONEM CULPA IURE NAM NIHIL PORRO PERFERENDIS PRAESENTIUM ULLAM. UT HIC TEMPORA SINT IN EST. SUNT SED TEMPORIBUS QUIS UT ISTE REPREHENDERIT IN SUNT LAUDANTIUM. POSSIMUS SEQUI QUIA RECUSANDAE VOLUPTAS NON QUOS FACILIS MINUS PERFERENDIS.\n\nQUISQUAM LABORIOSAM RERUM NAM ELIGENDI OMNIS CONSEQUUNTUR. AUT AB PRAESENTIUM REPREHENDERIT ASPERNATUR SED OCCAECATI ILLO DOLORUM UT RATIONE AUT LABORE ODIT RERUM. SAPIENTE QUIA EX NIHIL INCIDUNT INVENTORE LIBERO SED RECUSANDAE OMNIS CONSEQUATUR ENIM. DOLORES ENIM EXPEDITA EA. NOSTRUM ASPERNATUR EARUM OMNIS.","all_random_upper":"VHX6N[INTV[&YRIATTO5","whitespaces_upper":" BGE XO F S Q G"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"VHX6N[INTV[&YRIATTO5"} +{"ID":"tmNkka1IcmnHFIbFP8hpMqryfgUNz27snLisD6SwBekahrpAUGfWlRsbVH0m1oWW","dates":{"date_format1":"029, 29 Jan 2017 09:19:36 GMT+1","date_format2":"2017-01-28T09:03:57","date_format3":"Sun, 1 Jan 2017 17:41:05 +0100","date_format4":"2017-01-11T05:33:27+0100","date_format5":"01-08-2017 14:23","epoch":{"ibm":"16011980.213951","overflow_negative":-79913531539,"overflow_positive":264283192064,"random_negative":-586600374,"random_positive":1981899766,"zero":0}},"numerics":{"SmartObject":{"all_random":"kQ!&D5&Kz(*y)hw[a0Wc","whitespaces":" sdc j o cab","with_new_lines":"Blanditiis a debitis expedita velit ullam enim odit atque. Explicabo est veniam laudantium eum dignissimos et aut fugiat cum expedita quam quasi laborum alias. Eum tenetur voluptatem rem et sit. Quos tempore nemo voluptate provident alias eius in corporis velit nesciunt officia. Delectus facere a veritatis. Nihil quis similique totam ipsam cum aut labore dolorem molestiae voluptatibus nihil voluptas et est consequatur. Laborum qui beatae dolores quasi iusto. Enim ut quos asperiores magni nobis.\n\nMinima et eum eos sit labore fuga vel voluptatem quaerat ab ut. Quia iste velit ea. Suscipit quam sunt velit vel et aut quasi consequatur nemo. Fugit et vero laudantium voluptas nulla est vitae aliquam voluptatem. Aut aut commodi enim et similique necessitatibus rem repudiandae et nisi nesciunt deserunt. Voluptatibus tenetur laboriosam aut tenetur et nulla in id provident molestiae alias. Ullam fuga corporis voluptatem culpa iure asperiores corrupti laboriosam eos magnam nostrum eius id reprehenderit.\n\nCumque consequuntur consequatur iure id corporis necessitatibus possimus et ullam repellendus quod. Et modi omnis voluptatum aut ipsam architecto sapiente voluptatem atque eligendi quia distinctio et. Omnis ut sint ut ad odit non quia consequatur eum. Provident eum qui nihil."},"big_negative":-922649228419401,"big_positive":622312963632063,"small_negative":-864,"small_positive":66,"zero":0,"small_positive_casted1":"66","small_negative_casted1":"-864","big_positive_casted1":"622312963632063","small_positive_negated":-66,"small_negative_negated":864,"big_positive_negated":-622312963632063,"big_negative_negated":922649228419401},"strings":{"all_random":"GmRdQlE4Avn1hSlVPAH#","whitespaces":" c sa yv drf","with_new_lines":"Laboriosam modi numquam reprehenderit. Nihil blanditiis culpa eos sed et ipsum laudantium non repellat non. Voluptatem non aspernatur sit cumque cum aut suscipit nisi. Dignissimos porro dolor facilis et architecto non tenetur qui est culpa rerum. Voluptatem ratione provident rerum et excepturi ratione voluptatibus neque sed at illum nesciunt nobis magni adipisci. In eum quo ea eius voluptas maxime qui tempora quae sint. Ducimus voluptatum est veritatis molestiae neque dolore omnis expedita quae qui quibusdam veritatis. Repudiandae necessitatibus aut saepe quia assumenda est dolorem dolor ipsa ipsam explicabo numquam.\n\nDelectus ullam provident nesciunt quam dignissimos sequi porro aperiam quos labore. Iusto vitae et sunt enim architecto ducimus quia. Velit odio nostrum amet et id excepturi praesentium voluptatibus dignissimos exercitationem. Dignissimos consequuntur ipsa qui corporis aliquid cumque odio aut explicabo in modi et reprehenderit voluptatibus distinctio. Eum aut omnis totam quis sint voluptatum eius recusandae quaerat perferendis.\n\nSoluta id aliquam ut id recusandae et numquam aperiam ut optio ad nesciunt doloribus deleniti aut. Voluptatem fuga delectus sequi minus omnis ut. Amet qui voluptas quisquam suscipit. Sunt expedita quidem ex ducimus commodi quasi commodi labore eaque occaecati quod est. Rerum vitae quae assumenda dolorem debitis eos delectus amet excepturi aut culpa alias. Sunt omnis commodi culpa laborum et quia autem culpa et quae magnam laudantium. Accusamus reprehenderit expedita ex deleniti voluptas atque.","with_new_lines_upper":"LABORIOSAM MODI NUMQUAM REPREHENDERIT. NIHIL BLANDITIIS CULPA EOS SED ET IPSUM LAUDANTIUM NON REPELLAT NON. VOLUPTATEM NON ASPERNATUR SIT CUMQUE CUM AUT SUSCIPIT NISI. DIGNISSIMOS PORRO DOLOR FACILIS ET ARCHITECTO NON TENETUR QUI EST CULPA RERUM. VOLUPTATEM RATIONE PROVIDENT RERUM ET EXCEPTURI RATIONE VOLUPTATIBUS NEQUE SED AT ILLUM NESCIUNT NOBIS MAGNI ADIPISCI. IN EUM QUO EA EIUS VOLUPTAS MAXIME QUI TEMPORA QUAE SINT. DUCIMUS VOLUPTATUM EST VERITATIS MOLESTIAE NEQUE DOLORE OMNIS EXPEDITA QUAE QUI QUIBUSDAM VERITATIS. REPUDIANDAE NECESSITATIBUS AUT SAEPE QUIA ASSUMENDA EST DOLOREM DOLOR IPSA IPSAM EXPLICABO NUMQUAM.\n\nDELECTUS ULLAM PROVIDENT NESCIUNT QUAM DIGNISSIMOS SEQUI PORRO APERIAM QUOS LABORE. IUSTO VITAE ET SUNT ENIM ARCHITECTO DUCIMUS QUIA. VELIT ODIO NOSTRUM AMET ET ID EXCEPTURI PRAESENTIUM VOLUPTATIBUS DIGNISSIMOS EXERCITATIONEM. DIGNISSIMOS CONSEQUUNTUR IPSA QUI CORPORIS ALIQUID CUMQUE ODIO AUT EXPLICABO IN MODI ET REPREHENDERIT VOLUPTATIBUS DISTINCTIO. EUM AUT OMNIS TOTAM QUIS SINT VOLUPTATUM EIUS RECUSANDAE QUAERAT PERFERENDIS.\n\nSOLUTA ID ALIQUAM UT ID RECUSANDAE ET NUMQUAM APERIAM UT OPTIO AD NESCIUNT DOLORIBUS DELENITI AUT. VOLUPTATEM FUGA DELECTUS SEQUI MINUS OMNIS UT. AMET QUI VOLUPTAS QUISQUAM SUSCIPIT. SUNT EXPEDITA QUIDEM EX DUCIMUS COMMODI QUASI COMMODI LABORE EAQUE OCCAECATI QUOD EST. RERUM VITAE QUAE ASSUMENDA DOLOREM DEBITIS EOS DELECTUS AMET EXCEPTURI AUT CULPA ALIAS. SUNT OMNIS COMMODI CULPA LABORUM ET QUIA AUTEM CULPA ET QUAE MAGNAM LAUDANTIUM. ACCUSAMUS REPREHENDERIT EXPEDITA EX DELENITI VOLUPTAS ATQUE.","all_random_upper":"GMRDQLE4AVN1HSLVPAH#","whitespaces_upper":" C SA YV DRF"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"GMRDQLE4AVN1HSLVPAH#"} +{"ID":"vG8IG0B9VLO2YqFggLIHH0cw5vecJunmB1b9ngv0yg5yZ39Ps6Hf3NH6mK2c1Iyq","dates":{"date_format1":"016, 16 Jan 2017 09:02:58 GMT+1","date_format2":"2017-01-01T23:44:24","date_format3":"Mon, 23 Jan 2017 11:56:36 +0100","date_format4":"2017-01-17T12:37:34+0100","date_format5":"01-02-2017 08:56","epoch":{"ibm":"05022123.171955","overflow_negative":-3174941341,"overflow_positive":989758550516,"random_negative":-907625661,"random_positive":731107142,"zero":0}},"numerics":{"SmartObject":{"all_random":"(nh4m!rL$NL5^h$(fzib","whitespaces":" jv y ","with_new_lines":"Dolores magni temporibus sed aut blanditiis magni fugiat distinctio magni in sunt nihil repudiandae molestiae. Quos molestiae impedit earum sunt consectetur rerum necessitatibus ut ex. Ipsum et architecto veritatis hic enim deserunt minus. Dolore cupiditate laudantium itaque quia odit quis at hic qui maxime quos.\n\nAut architecto harum ipsa repellat quibusdam maxime. Numquam nesciunt laudantium at. Molestiae alias aut vitae veritatis consectetur quia rerum.\n\nQuidem est laborum neque id quidem nulla eum sint voluptatem quia repudiandae sequi provident ullam excepturi. Ut cupiditate quos hic eos. Tenetur dolore sed enim dolorem magni accusamus quidem vel qui quaerat facere consectetur quam voluptatem. Veniam inventore asperiores sunt sit. Molestiae explicabo et ut."},"big_negative":-887851267652913,"big_positive":38876546092228,"small_negative":-672,"small_positive":941,"zero":0,"small_positive_casted1":"941","small_negative_casted1":"-672","big_positive_casted1":"38876546092228","small_positive_negated":-941,"small_negative_negated":672,"big_positive_negated":-38876546092228,"big_negative_negated":887851267652913},"strings":{"all_random":"rY&n9UnVcD*KS]jPBpa[","whitespaces":" rw xfv ","with_new_lines":"Accusamus quia vel deleniti. Sit velit labore ad iure sunt nemo incidunt autem beatae velit. Voluptas asperiores architecto aut aut corrupti qui explicabo sit. Praesentium et optio consequuntur quidem dignissimos mollitia consequatur autem. Deserunt hic labore nemo et sunt autem esse repudiandae saepe natus tempora. Corporis ex odit dolor saepe excepturi et aliquam aut expedita voluptas ut quis ut quaerat deserunt. Aut dolores facere repellat. Nemo officiis excepturi minus amet est incidunt. Doloribus dolores tempora quidem quis.\n\nQuod perferendis sit ullam sint qui. Praesentium sit delectus laborum nemo perspiciatis. Ut laboriosam animi ea aspernatur unde voluptas accusamus tenetur aut ea illum amet nihil quam ipsum. Voluptates tempora pariatur repellendus dolores quidem ab aut qui sapiente dolorem ipsum. Natus qui impedit excepturi odio voluptatem tempora sequi. Quae beatae qui cum sunt corrupti et enim est nesciunt doloremque iusto illum qui. Eos sapiente et aspernatur tempora. Eum consequuntur quod eum voluptatem velit excepturi accusamus ullam.\n\nDolor et alias qui libero deserunt dolorem id suscipit esse. Blanditiis ipsa in quaerat explicabo quae facere. Velit delectus temporibus asperiores qui qui autem non perspiciatis unde nisi architecto et ipsum non. Ut exercitationem quod recusandae error eos neque aut rerum eligendi ullam eligendi voluptate. Sunt maxime molestiae accusamus in sed aliquam temporibus voluptatem asperiores pariatur non ratione. Aliquid aliquam neque est similique voluptas magni odit inventore.","with_new_lines_upper":"ACCUSAMUS QUIA VEL DELENITI. SIT VELIT LABORE AD IURE SUNT NEMO INCIDUNT AUTEM BEATAE VELIT. VOLUPTAS ASPERIORES ARCHITECTO AUT AUT CORRUPTI QUI EXPLICABO SIT. PRAESENTIUM ET OPTIO CONSEQUUNTUR QUIDEM DIGNISSIMOS MOLLITIA CONSEQUATUR AUTEM. DESERUNT HIC LABORE NEMO ET SUNT AUTEM ESSE REPUDIANDAE SAEPE NATUS TEMPORA. CORPORIS EX ODIT DOLOR SAEPE EXCEPTURI ET ALIQUAM AUT EXPEDITA VOLUPTAS UT QUIS UT QUAERAT DESERUNT. AUT DOLORES FACERE REPELLAT. NEMO OFFICIIS EXCEPTURI MINUS AMET EST INCIDUNT. DOLORIBUS DOLORES TEMPORA QUIDEM QUIS.\n\nQUOD PERFERENDIS SIT ULLAM SINT QUI. PRAESENTIUM SIT DELECTUS LABORUM NEMO PERSPICIATIS. UT LABORIOSAM ANIMI EA ASPERNATUR UNDE VOLUPTAS ACCUSAMUS TENETUR AUT EA ILLUM AMET NIHIL QUAM IPSUM. VOLUPTATES TEMPORA PARIATUR REPELLENDUS DOLORES QUIDEM AB AUT QUI SAPIENTE DOLOREM IPSUM. NATUS QUI IMPEDIT EXCEPTURI ODIO VOLUPTATEM TEMPORA SEQUI. QUAE BEATAE QUI CUM SUNT CORRUPTI ET ENIM EST NESCIUNT DOLOREMQUE IUSTO ILLUM QUI. EOS SAPIENTE ET ASPERNATUR TEMPORA. EUM CONSEQUUNTUR QUOD EUM VOLUPTATEM VELIT EXCEPTURI ACCUSAMUS ULLAM.\n\nDOLOR ET ALIAS QUI LIBERO DESERUNT DOLOREM ID SUSCIPIT ESSE. BLANDITIIS IPSA IN QUAERAT EXPLICABO QUAE FACERE. VELIT DELECTUS TEMPORIBUS ASPERIORES QUI QUI AUTEM NON PERSPICIATIS UNDE NISI ARCHITECTO ET IPSUM NON. UT EXERCITATIONEM QUOD RECUSANDAE ERROR EOS NEQUE AUT RERUM ELIGENDI ULLAM ELIGENDI VOLUPTATE. SUNT MAXIME MOLESTIAE ACCUSAMUS IN SED ALIQUAM TEMPORIBUS VOLUPTATEM ASPERIORES PARIATUR NON RATIONE. ALIQUID ALIQUAM NEQUE EST SIMILIQUE VOLUPTAS MAGNI ODIT INVENTORE.","all_random_upper":"RY&N9UNVCD*KS]JPBPA[","whitespaces_upper":" RW XFV "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"RY&N9UNVCD*KS]JPBPA["} +{"ID":"wgQiORjaVzmzGML9tisIicRYBaf5CL9XfH1sr43JW9y6TRvxCvgTsNQ7dPIgor85","dates":{"date_format1":"021, 21 Jan 2017 16:07:07 GMT+1","date_format2":"2017-01-02T15:15:52","date_format3":"Wed, 25 Jan 2017 06:15:52 +0100","date_format4":"2017-01-25T02:06:57+0100","date_format5":"01-08-2017 18:38","epoch":{"ibm":"04121996.213104","overflow_negative":-49496782187,"overflow_positive":725956158401,"random_negative":-1446029591,"random_positive":2234641,"zero":0}},"numerics":{"SmartObject":{"all_random":"2KY]KA!cR]fsC$YmhMhy","whitespaces":" k eu z c u ","with_new_lines":"Ea omnis eligendi eos et tempore sit facilis occaecati eaque et. Est rem nulla expedita doloribus qui qui. Placeat quisquam impedit quae sit est. Ut fugit quo architecto. Officia expedita similique optio ullam quod rerum libero nam perspiciatis aut. Vel dolorem quisquam magni beatae et optio distinctio maxime libero optio autem. Voluptates est hic non quia dolore aut temporibus tempore iste voluptatem laboriosam. Necessitatibus autem minus reiciendis accusamus in architecto omnis tenetur et.\n\nOmnis in ipsum nihil minima rerum nobis similique et nihil. Quia voluptatem nobis molestias deserunt molestias voluptas eum. Sint nam quo eum aut cum facere et non est ipsum eaque velit rerum. Molestias eum tempora optio magnam quis. Est odio molestias excepturi adipisci. Ea non atque quod labore et ut dolorem quos dolorem sed repellat.\n\nEaque nihil ratione porro. Tempora quibusdam sed omnis rerum laudantium minima modi nobis rerum quas quo libero excepturi nulla perspiciatis. Sint fugiat suscipit fugit est enim sit assumenda ut ullam dolores voluptas."},"big_negative":-161717972030100,"big_positive":546512456067624,"small_negative":-402,"small_positive":260,"zero":0,"small_positive_casted1":"260","small_negative_casted1":"-402","big_positive_casted1":"546512456067624","small_positive_negated":-260,"small_negative_negated":402,"big_positive_negated":-546512456067624,"big_negative_negated":161717972030100},"strings":{"all_random":"A7mmkGyNv)aln4DCRut5","whitespaces":" m u t f tq ","with_new_lines":"Et eligendi blanditiis iusto dolore blanditiis ad. Eius deserunt aperiam iure quae alias in veritatis voluptates et fuga. Temporibus sequi accusantium laudantium et suscipit et cum excepturi molestias unde fuga est quibusdam. Mollitia delectus omnis rem itaque deleniti autem quasi et sint esse reprehenderit est magnam est sed. Fuga enim omnis facilis id velit labore voluptate. Id in officiis at exercitationem dolores molestiae et quo voluptatem ex veniam modi ut repudiandae. Ratione id quam in eum qui labore dolor excepturi dolor autem libero quia voluptas nostrum voluptas. Reiciendis voluptatem qui adipisci architecto corrupti nulla ut et tempora commodi maiores cum quidem.\n\nIn debitis et adipisci sequi. Facere nesciunt explicabo et enim molestiae et fuga odit doloribus qui quia quia dolorem aut modi. Ullam ut dolore est. Culpa voluptatum voluptas sed sint sunt. Quisquam animi rerum sed est sint commodi sit adipisci. Suscipit qui amet et sunt ab eum necessitatibus laboriosam magnam praesentium molestiae aut ut minima.\n\nQui molestias numquam pariatur veniam adipisci. Aliquam et consequuntur sit est ducimus aut odit libero quisquam ea placeat sit odit labore. Voluptatem eligendi natus ullam non et tempora adipisci quasi adipisci voluptatem vel commodi. Officiis id quidem iure ea nesciunt sunt omnis pariatur. Illum voluptates sint suscipit consectetur soluta molestiae ducimus sit qui.","with_new_lines_upper":"ET ELIGENDI BLANDITIIS IUSTO DOLORE BLANDITIIS AD. EIUS DESERUNT APERIAM IURE QUAE ALIAS IN VERITATIS VOLUPTATES ET FUGA. TEMPORIBUS SEQUI ACCUSANTIUM LAUDANTIUM ET SUSCIPIT ET CUM EXCEPTURI MOLESTIAS UNDE FUGA EST QUIBUSDAM. MOLLITIA DELECTUS OMNIS REM ITAQUE DELENITI AUTEM QUASI ET SINT ESSE REPREHENDERIT EST MAGNAM EST SED. FUGA ENIM OMNIS FACILIS ID VELIT LABORE VOLUPTATE. ID IN OFFICIIS AT EXERCITATIONEM DOLORES MOLESTIAE ET QUO VOLUPTATEM EX VENIAM MODI UT REPUDIANDAE. RATIONE ID QUAM IN EUM QUI LABORE DOLOR EXCEPTURI DOLOR AUTEM LIBERO QUIA VOLUPTAS NOSTRUM VOLUPTAS. REICIENDIS VOLUPTATEM QUI ADIPISCI ARCHITECTO CORRUPTI NULLA UT ET TEMPORA COMMODI MAIORES CUM QUIDEM.\n\nIN DEBITIS ET ADIPISCI SEQUI. FACERE NESCIUNT EXPLICABO ET ENIM MOLESTIAE ET FUGA ODIT DOLORIBUS QUI QUIA QUIA DOLOREM AUT MODI. ULLAM UT DOLORE EST. CULPA VOLUPTATUM VOLUPTAS SED SINT SUNT. QUISQUAM ANIMI RERUM SED EST SINT COMMODI SIT ADIPISCI. SUSCIPIT QUI AMET ET SUNT AB EUM NECESSITATIBUS LABORIOSAM MAGNAM PRAESENTIUM MOLESTIAE AUT UT MINIMA.\n\nQUI MOLESTIAS NUMQUAM PARIATUR VENIAM ADIPISCI. ALIQUAM ET CONSEQUUNTUR SIT EST DUCIMUS AUT ODIT LIBERO QUISQUAM EA PLACEAT SIT ODIT LABORE. VOLUPTATEM ELIGENDI NATUS ULLAM NON ET TEMPORA ADIPISCI QUASI ADIPISCI VOLUPTATEM VEL COMMODI. OFFICIIS ID QUIDEM IURE EA NESCIUNT SUNT OMNIS PARIATUR. ILLUM VOLUPTATES SINT SUSCIPIT CONSECTETUR SOLUTA MOLESTIAE DUCIMUS SIT QUI.","all_random_upper":"A7MMKGYNV)ALN4DCRUT5","whitespaces_upper":" M U T F TQ "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"A7MMKGYNV)ALN4DCRUT5"} +{"ID":"x0PJ1ni75i5nNZ6fTzLaVouOGrrVOsQa0pSPSXDI1PZ4jCD2Ru8s9F3G0yaenker","dates":{"date_format1":"025, 25 Jan 2017 13:35:33 GMT+1","date_format2":"2017-01-08T10:46:05","date_format3":"Mon, 23 Jan 2017 11:29:38 +0100","date_format4":"2017-01-02T11:48:26+0100","date_format5":"01-07-2017 08:24","epoch":{"ibm":"23102061.172405","overflow_negative":-98158493367,"overflow_positive":376728263471,"random_negative":-1804178033,"random_positive":573704926,"zero":0}},"numerics":{"SmartObject":{"all_random":"R4Cx9IIXDBSEvlPW@2M)","whitespaces":" j a mwc ","with_new_lines":"Reprehenderit ab vel ipsum omnis repellendus ex debitis excepturi fuga aut aspernatur incidunt omnis ex facilis. Cum quaerat qui ducimus nesciunt enim inventore. Consequatur aut officiis cum corporis aut harum reprehenderit in aliquid sit iste velit pariatur. Occaecati voluptatum earum libero sit velit earum et quia et quos saepe. Eveniet sit quia corrupti officiis voluptatum maxime provident voluptas ipsam magnam sapiente dolor architecto libero. Aut est sapiente cumque alias quidem cupiditate est optio numquam et consequatur amet iste qui. Enim aut molestiae neque perferendis. Voluptas dolor non harum eum nesciunt nulla fuga quae id labore ea incidunt asperiores.\n\nExpedita quibusdam aut placeat praesentium placeat consectetur veritatis dignissimos molestiae itaque sunt aliquid quaerat at. Corrupti rerum qui blanditiis natus ad. Quidem qui qui veritatis error sint dicta aut sunt quam unde ipsam ad dolorem sit reiciendis. Blanditiis enim et est saepe aut eaque corrupti tempora vitae itaque officiis ad ipsam. Vitae modi beatae minima omnis iusto quos magni consectetur quam.\n\nCulpa ipsum architecto totam ex ab culpa et nihil dolor id optio praesentium ut et. Cumque aut aspernatur accusamus sit ipsa cum porro voluptatem occaecati ut minima sit ut. Et assumenda voluptatem laudantium modi optio qui voluptate officiis. Ab aut quam est omnis dolorum commodi temporibus accusantium."},"big_negative":-909463012558308,"big_positive":154024782904132,"small_negative":-506,"small_positive":966,"zero":0,"small_positive_casted1":"966","small_negative_casted1":"-506","big_positive_casted1":"154024782904132","small_positive_negated":-966,"small_negative_negated":506,"big_positive_negated":-154024782904132,"big_negative_negated":909463012558308},"strings":{"all_random":"rX^1Vm6Apx9ilP83$VbA","whitespaces":" eh s g m iu","with_new_lines":"Voluptatem nihil et beatae ut. Aut ipsam et qui at aut. Totam quia ea eius quia sunt id unde porro minus eaque doloribus sit delectus. Ullam ea adipisci aut at. Nihil quos enim voluptates error rerum optio omnis dolorem eligendi voluptas qui et.\n\nEst sit ea eligendi quis magni atque quidem est. Cumque quia ut autem id vitae non fugit inventore sint. Autem asperiores voluptas rerum assumenda laboriosam corrupti ullam accusantium sed quia odit vero quaerat. Laudantium repellendus omnis quia dicta quisquam possimus magni ad porro mollitia. Enim animi enim maxime cupiditate sapiente. Dolorem iste necessitatibus ut aliquam veritatis architecto consequatur id. Illum repudiandae sint ea sint eius maiores quibusdam hic voluptatem est non exercitationem quis.\n\nMaiores quasi laborum voluptas dicta. Dolor dolorem amet autem iste laborum voluptatibus voluptatibus commodi porro laborum quasi vero. Voluptatem architecto nobis expedita. Ratione sequi eum autem est voluptates laudantium corporis.","with_new_lines_upper":"VOLUPTATEM NIHIL ET BEATAE UT. AUT IPSAM ET QUI AT AUT. TOTAM QUIA EA EIUS QUIA SUNT ID UNDE PORRO MINUS EAQUE DOLORIBUS SIT DELECTUS. ULLAM EA ADIPISCI AUT AT. NIHIL QUOS ENIM VOLUPTATES ERROR RERUM OPTIO OMNIS DOLOREM ELIGENDI VOLUPTAS QUI ET.\n\nEST SIT EA ELIGENDI QUIS MAGNI ATQUE QUIDEM EST. CUMQUE QUIA UT AUTEM ID VITAE NON FUGIT INVENTORE SINT. AUTEM ASPERIORES VOLUPTAS RERUM ASSUMENDA LABORIOSAM CORRUPTI ULLAM ACCUSANTIUM SED QUIA ODIT VERO QUAERAT. LAUDANTIUM REPELLENDUS OMNIS QUIA DICTA QUISQUAM POSSIMUS MAGNI AD PORRO MOLLITIA. ENIM ANIMI ENIM MAXIME CUPIDITATE SAPIENTE. DOLOREM ISTE NECESSITATIBUS UT ALIQUAM VERITATIS ARCHITECTO CONSEQUATUR ID. ILLUM REPUDIANDAE SINT EA SINT EIUS MAIORES QUIBUSDAM HIC VOLUPTATEM EST NON EXERCITATIONEM QUIS.\n\nMAIORES QUASI LABORUM VOLUPTAS DICTA. DOLOR DOLOREM AMET AUTEM ISTE LABORUM VOLUPTATIBUS VOLUPTATIBUS COMMODI PORRO LABORUM QUASI VERO. VOLUPTATEM ARCHITECTO NOBIS EXPEDITA. RATIONE SEQUI EUM AUTEM EST VOLUPTATES LAUDANTIUM CORPORIS.","all_random_upper":"RX^1VM6APX9ILP83$VBA","whitespaces_upper":" EH S G M IU"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"RX^1VM6APX9ILP83$VBA"} +{"ID":"xifcQGcR1ZpN3nQpueMJ68YU69uvVeJIPVGXjVpiD6lOAmVOgJzk1K2SAmeuiRPz","dates":{"date_format1":"028, 28 Jan 2017 13:20:12 GMT+1","date_format2":"2017-01-06T07:39:06","date_format3":"Thu, 26 Jan 2017 18:04:22 +0100","date_format4":"2017-01-09T11:40:54+0100","date_format5":"01-07-2017 07:53","epoch":{"ibm":"07122121.091023","overflow_negative":-79706968488,"overflow_positive":262234174436,"random_negative":-1597371550,"random_positive":1824260297,"zero":0}},"numerics":{"SmartObject":{"all_random":"QXTHAZ8E8YJKmOKEdg]T","whitespaces":"xm wre m l ","with_new_lines":"Voluptatem laudantium consequatur ipsum odit fuga asperiores sapiente. Ab corrupti molestias dicta fugit aperiam. Velit non rerum laudantium quod laudantium et dolor sit rem reiciendis quia. Eos dicta illum consectetur velit nisi voluptas reiciendis quasi. Nesciunt quidem rem nisi et omnis veritatis sequi.\n\nAmet nulla et occaecati voluptas et quidem maiores soluta laudantium doloribus veniam illo provident. Eum sit laboriosam sint libero tenetur aspernatur qui aut culpa explicabo. Voluptatem ut voluptas et dolorem vel nihil asperiores harum esse voluptatem. Ut nihil magnam inventore quod illum saepe totam. Delectus deserunt sunt aspernatur dolor totam et at aut doloribus facere. Repudiandae quia et deserunt dolores ut quia dicta aliquid quasi officia quas quibusdam quis minus est. Maiores praesentium fugiat aut adipisci recusandae ut ratione.\n\nLabore ab voluptate non sapiente amet voluptatem qui minima repellat cum esse consequatur accusantium. Qui qui eos est et qui nulla fugit laborum. Ut qui est officia tempora voluptatum fugit aut quisquam est. Sed aut quasi libero ipsum voluptas voluptas. Quia voluptates fugiat quisquam perspiciatis. Repellat nulla quae in ut qui magnam animi ad et inventore aspernatur numquam illo est."},"big_negative":-685211018991098,"big_positive":396082908903782,"small_negative":-520,"small_positive":25,"zero":0,"small_positive_casted1":"25","small_negative_casted1":"-520","big_positive_casted1":"396082908903782","small_positive_negated":-25,"small_negative_negated":520,"big_positive_negated":-396082908903782,"big_negative_negated":685211018991098},"strings":{"all_random":"q]jST1N)q^[T%#skEz&(","whitespaces":" o czx u r ud","with_new_lines":"Temporibus quis quos rerum fuga alias est totam repellat et error velit itaque consequatur aut esse. Minus eos quia optio incidunt. Qui aliquid est aut esse eligendi minima qui adipisci harum odio est. A ipsam id eum odit sed optio. Ea fugit pariatur ut itaque quo quasi maiores et et. Et et et ut dolores aut omnis vel velit ut accusamus voluptate autem dolore aperiam. Saepe reprehenderit ad sit animi qui magni laudantium fuga numquam corporis et sit.\n\nSit rerum labore optio distinctio consectetur consectetur enim architecto temporibus aperiam eum eos consequatur officia quibusdam. Veritatis est ad qui sapiente qui blanditiis rerum quae voluptatum omnis. Debitis minima ab et veritatis facilis nihil at voluptatum atque voluptate est quod. Ut ut illo atque molestiae quos saepe autem eos voluptatum ipsam corrupti. Perferendis sed cum placeat enim quaerat sint fugiat. Necessitatibus eos ex nisi molestiae blanditiis et soluta id vero.\n\nConsequatur porro a aut cum iure at vero in. Suscipit pariatur consequuntur sed consequatur quae eum. Iure error est aut cumque deserunt. Nam provident quia ab consectetur praesentium aut doloribus est aut est quia veniam ipsum. Nostrum autem saepe dolorem quidem quod molestiae voluptas quas quia laudantium omnis voluptatum. At rerum ullam aut doloremque facilis aut provident minima labore maiores ea in eveniet aut accusantium.","with_new_lines_upper":"TEMPORIBUS QUIS QUOS RERUM FUGA ALIAS EST TOTAM REPELLAT ET ERROR VELIT ITAQUE CONSEQUATUR AUT ESSE. MINUS EOS QUIA OPTIO INCIDUNT. QUI ALIQUID EST AUT ESSE ELIGENDI MINIMA QUI ADIPISCI HARUM ODIO EST. A IPSAM ID EUM ODIT SED OPTIO. EA FUGIT PARIATUR UT ITAQUE QUO QUASI MAIORES ET ET. ET ET ET UT DOLORES AUT OMNIS VEL VELIT UT ACCUSAMUS VOLUPTATE AUTEM DOLORE APERIAM. SAEPE REPREHENDERIT AD SIT ANIMI QUI MAGNI LAUDANTIUM FUGA NUMQUAM CORPORIS ET SIT.\n\nSIT RERUM LABORE OPTIO DISTINCTIO CONSECTETUR CONSECTETUR ENIM ARCHITECTO TEMPORIBUS APERIAM EUM EOS CONSEQUATUR OFFICIA QUIBUSDAM. VERITATIS EST AD QUI SAPIENTE QUI BLANDITIIS RERUM QUAE VOLUPTATUM OMNIS. DEBITIS MINIMA AB ET VERITATIS FACILIS NIHIL AT VOLUPTATUM ATQUE VOLUPTATE EST QUOD. UT UT ILLO ATQUE MOLESTIAE QUOS SAEPE AUTEM EOS VOLUPTATUM IPSAM CORRUPTI. PERFERENDIS SED CUM PLACEAT ENIM QUAERAT SINT FUGIAT. NECESSITATIBUS EOS EX NISI MOLESTIAE BLANDITIIS ET SOLUTA ID VERO.\n\nCONSEQUATUR PORRO A AUT CUM IURE AT VERO IN. SUSCIPIT PARIATUR CONSEQUUNTUR SED CONSEQUATUR QUAE EUM. IURE ERROR EST AUT CUMQUE DESERUNT. NAM PROVIDENT QUIA AB CONSECTETUR PRAESENTIUM AUT DOLORIBUS EST AUT EST QUIA VENIAM IPSUM. NOSTRUM AUTEM SAEPE DOLOREM QUIDEM QUOD MOLESTIAE VOLUPTAS QUAS QUIA LAUDANTIUM OMNIS VOLUPTATUM. AT RERUM ULLAM AUT DOLOREMQUE FACILIS AUT PROVIDENT MINIMA LABORE MAIORES EA IN EVENIET AUT ACCUSANTIUM.","all_random_upper":"Q]JST1N)Q^[T%#SKEZ&(","whitespaces_upper":" O CZX U R UD"},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"Q]JST1N)Q^[T%#SKEZ&("} +{"ID":"yoE2kpBXMzywDb4K6MkDGgOzdO1Ysr74Udt3UAUVNBjdBiVnnCu9s7Yr3isAAx6d","dates":{"date_format1":"024, 24 Jan 2017 07:05:28 GMT+1","date_format2":"2017-01-06T09:46:01","date_format3":"Sat, 28 Jan 2017 13:21:13 +0100","date_format4":"2017-01-28T09:21:11+0100","date_format5":"01-26-2017 05:47","epoch":{"ibm":"30092053.211026","overflow_negative":-89552179742,"overflow_positive":825279530076,"random_negative":-849422971,"random_positive":2116050965,"zero":0}},"numerics":{"SmartObject":{"all_random":"ZCZCLX4eeHMK@A)[DZ2w","whitespaces":" f c wp a z l ","with_new_lines":"Et voluptas tempora sint consequatur ut consequuntur. Aut deleniti aut qui. Voluptatem quo aut et tempore voluptatibus ipsa sint assumenda. Ea est quisquam recusandae. Tempore molestiae rerum voluptatem qui quam commodi recusandae sed aliquam et modi et. Perferendis cum quo vel beatae eum cumque est. Ad nemo itaque sunt accusamus doloremque dolor sint veniam aut accusantium iusto voluptatum voluptas. Voluptas voluptatum inventore dignissimos fugiat odio ducimus aut neque sunt similique qui cupiditate eligendi et dolor. Modi quo modi ab ex id eaque consectetur expedita porro consequatur unde velit.\n\nMinus porro cum dolorem odio. Omnis tenetur dolor corrupti consequatur eius. Ut sit vitae quo. Voluptate quod molestias ut quo voluptas deserunt possimus est facere sunt vitae doloremque omnis. Voluptatem laboriosam animi qui nobis tenetur sed ea qui omnis tempore fugiat tempora enim et unde. Repellendus commodi sunt ipsum consequuntur. Voluptatem ullam omnis consequatur est itaque aut esse ipsa vel dolor error nulla ut. In numquam exercitationem voluptatem vero labore et.\n\nBlanditiis ipsam illum aut dolorem sed voluptatem quod ullam vero. Rerum esse dicta et neque labore officia minus. Impedit eum omnis quibusdam qui voluptas sint quia. Molestiae quos hic adipisci vitae. Quis animi exercitationem inventore fugiat tempora minus. Ut enim aut veniam alias quas asperiores pariatur ex omnis dolorem atque esse."},"big_negative":-329442716592574,"big_positive":80572169057771,"small_negative":-116,"small_positive":901,"zero":0,"small_positive_casted1":"901","small_negative_casted1":"-116","big_positive_casted1":"80572169057771","small_positive_negated":-901,"small_negative_negated":116,"big_positive_negated":-80572169057771,"big_negative_negated":329442716592574},"strings":{"all_random":"DtnrnY0gOMNyaHpOHjUt","whitespaces":" d fdr hxg ","with_new_lines":"Dolores ipsam ipsum error et consequatur ut excepturi. Minus ab consequatur magni quis provident dicta ullam. Dicta aut nihil vitae provident laborum enim. Non consequatur est asperiores sunt veniam quas culpa repellendus neque minima dolores at et expedita. Reiciendis molestiae debitis et consequuntur laudantium tenetur qui assumenda.\n\nVoluptatem optio quis quibusdam eos voluptates pariatur ut nobis ducimus similique modi magnam aut. Vero et libero sunt accusamus similique eveniet aut. Ratione placeat aut provident totam repudiandae blanditiis unde expedita sit amet qui saepe.\n\nPorro accusantium nostrum illo molestiae quam voluptatem rem libero hic sed magnam blanditiis est animi. Aut eum dicta eveniet quod ad non velit sed laudantium. Excepturi debitis ut et ex qui labore sit.","with_new_lines_upper":"DOLORES IPSAM IPSUM ERROR ET CONSEQUATUR UT EXCEPTURI. MINUS AB CONSEQUATUR MAGNI QUIS PROVIDENT DICTA ULLAM. DICTA AUT NIHIL VITAE PROVIDENT LABORUM ENIM. NON CONSEQUATUR EST ASPERIORES SUNT VENIAM QUAS CULPA REPELLENDUS NEQUE MINIMA DOLORES AT ET EXPEDITA. REICIENDIS MOLESTIAE DEBITIS ET CONSEQUUNTUR LAUDANTIUM TENETUR QUI ASSUMENDA.\n\nVOLUPTATEM OPTIO QUIS QUIBUSDAM EOS VOLUPTATES PARIATUR UT NOBIS DUCIMUS SIMILIQUE MODI MAGNAM AUT. VERO ET LIBERO SUNT ACCUSAMUS SIMILIQUE EVENIET AUT. RATIONE PLACEAT AUT PROVIDENT TOTAM REPUDIANDAE BLANDITIIS UNDE EXPEDITA SIT AMET QUI SAEPE.\n\nPORRO ACCUSANTIUM NOSTRUM ILLO MOLESTIAE QUAM VOLUPTATEM REM LIBERO HIC SED MAGNAM BLANDITIIS EST ANIMI. AUT EUM DICTA EVENIET QUOD AD NON VELIT SED LAUDANTIUM. EXCEPTURI DEBITIS UT ET EX QUI LABORE SIT.","all_random_upper":"DTNRNY0GOMNYAHPOHJUT","whitespaces_upper":" D FDR HXG "},"errCol":[],"enceladus_info_date":"2020-05-23","enceladus_info_date_string":"2020-05-23","enceladus_info_version":"DTNRNY0GOMNYAHPOHJUT"} diff --git a/utils/src/main/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTable.scala b/utils/src/main/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTable.scala index a53215ae9..2b8763396 100644 --- a/utils/src/main/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTable.scala +++ b/utils/src/main/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTable.scala @@ -87,9 +87,6 @@ object LocalMappingTable { } private def validateKeyFields(mappingTableDf: DataFrame, keyFields: Seq[String]): Unit = { - if (keyFields.isEmpty) { - throw new IllegalArgumentException("No join key fields are provided for the mapping table.") - } keyFields.foreach(field => { SchemaUtils.getFieldType(field, mappingTableDf.schema) match { case Some(_: ArrayType) => throw new IllegalArgumentException(s"Join condition field cannot be an array: $field.") diff --git a/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/BroadcastUtilsSuite.scala b/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/BroadcastUtilsSuite.scala index df5d4953a..00b45faff 100644 --- a/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/BroadcastUtilsSuite.scala +++ b/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/BroadcastUtilsSuite.scala @@ -498,12 +498,6 @@ class BroadcastUtilsSuite extends AnyWordSpec with SparkTestBase with LoggerTest "throw an exception" when { - "a join without key fields is attempted" in { - intercept[IllegalArgumentException] { - LocalMappingTable(dfMt, Nil, Map(""->"val")) - } - } - "a join with more than 10 fields attempted" in { val localMt = LocalMappingTable(dfMt, Seq("id", "id", "id", "id", "id", "id", "id", "id", "id", "id", "id"), Map(""->"val")) diff --git a/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTableSuite.scala b/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTableSuite.scala index 379122846..250d96a21 100644 --- a/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTableSuite.scala +++ b/utils/src/test/scala/za/co/absa/enceladus/utils/broadcast/LocalMappingTableSuite.scala @@ -105,15 +105,16 @@ class LocalMappingTableSuite extends AnyWordSpec with SparkTestBase { assert(localMt.keyTypes(1).isInstanceOf[NumericType]) assert(localMt.getRowWithDefault(Seq(1, 21), 0).isInstanceOf[Row]) } - } - "throw an exception" when { - "no join keys are provided" in { - intercept[IllegalArgumentException] { - LocalMappingTable(dfMt, Nil, Map(""->"val")) - } + "no join condition" in { + val localMt = LocalMappingTable(dfMt, Nil, Map(""->"val")) + assert(localMt.keyTypes.length == 0) + assert(localMt.rowCount == 1) + assert(localMt.outputColumns.values.toSeq == Seq("val")) } + } + "throw an exception" when { "a join key does not exists in the schema" in { intercept[IllegalArgumentException] { LocalMappingTable(dfMt, Seq("dummy"), Map(""->"val"))