diff --git a/dao/pom.xml b/dao/pom.xml index a65e84833..66c9b1322 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -13,7 +13,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 dao @@ -43,17 +42,17 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} + ${jackson.spark.version} com.fasterxml.jackson.module jackson-module-scala_${scala.compat.version} - ${jackson.version} + ${jackson.spark.version} com.fasterxml.jackson.datatype jackson-datatype-jsr310 - ${jackson.version} + ${jackson.spark.datatype.version} org.springframework.security.kerberos diff --git a/data-model/pom.xml b/data-model/pom.xml index 18c6354f3..45b26fe15 100644 --- a/data-model/pom.xml +++ b/data-model/pom.xml @@ -35,12 +35,12 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} + ${jackson.spark.version} com.fasterxml.jackson.module jackson-module-scala_${scala.compat.version} - ${jackson.version} + ${jackson.spark.version} com.typesafe @@ -58,11 +58,6 @@ utils ${project.parent.version} - - io.swagger.core.v3 - swagger-annotations - ${swagger.annotations.version} - diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 000000000..40897e680 --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + examples + jar + + + za.co.absa.enceladus + parent + 3.1.0-SNAPSHOT + + + + ${project.parent.basedir}/scalastyle-config.xml + + + + + za.co.absa.enceladus + spark-jobs + ${project.version} + + + com.github.scopt + scopt_${scala.compat.version} + ${scopt.version} + + + + + + + org.scalastyle + scalastyle-maven-plugin + + + net.alchim31.maven + scala-maven-plugin + ${scala.maven.plugin.version} + + + + compile + testCompile + + + + + + -Xfatal-warnings + -unchecked + -deprecation + -feature + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.plugin.version} + + + copy + validate + + + + + + + run + + + + + + + diff --git a/menas/pom.xml b/menas/pom.xml index 922c0b2be..61583e713 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -12,7 +12,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 @@ -25,6 +24,279 @@ 3.1.0-SNAPSHOT + + 1.8 + 1.0.4 + ${project.basedir}/ui + ${project.parent.basedir}/scalastyle-config.xml + 2.2.0 + + + + + + org.scala-lang + scala-library + ${scala.version} + + + org.scala-lang.modules + scala-xml_${scala.compat.version} + ${scala.xml.version} + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + + org.apache.spark + spark-core_${scala.compat.version} + ${spark.version} + compile + + + org.slf4j + slf4j-log4j12 + + + javax.validation + validation-api + + + + + + org.apache.spark + spark-avro_${scala.compat.version} + ${spark.version} + compile + + + org.apache.spark + spark-sql_${scala.compat.version} + ${spark.version} + compile + + + org.apache.httpcomponents + httpclient + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + compile + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.compat.version} + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.google.code.gson + gson + ${gson.version} + + + + org.mongodb.scala + mongo-scala-driver_${scala.compat.version} + ${mongo.scala.driver.version} + + + io.github.cbartosiak + bson-codecs-jsr310 + ${bson.codec.jsr310.version} + + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-to-slf4j + ${log4j.version} + + + org.springframework.boot + spring-boot-starter-web + ${spring.version} + + + ch.qos.logback + logback-classic + + + + + org.springframework.boot + spring-boot-starter-actuator + ${spring.version} + + + + org.springframework.boot + spring-boot-starter-security + ${spring.version} + + + org.springframework.security + spring-security-ldap + ${spring.ldap.version} + + + org.springframework.security.kerberos + spring-security-kerberos-web + ${spring.kerberos.version} + + + org.springframework.security.kerberos + spring-security-kerberos-client + ${spring.kerberos.version} + + + io.jsonwebtoken + jjwt-api + ${jjwt.version} + + + io.jsonwebtoken + jjwt-impl + ${jjwt.version} + runtime + + + io.jsonwebtoken + jjwt-jackson + ${jjwt.version} + + + + org.apache.htrace + htrace-core + ${htrace.version} + + + + za.co.absa.enceladus + data-model + ${project.version} + + + za.co.absa.enceladus + migrations + ${project.version} + + + za.co.absa.enceladus + utils + ${project.version} + + + org.slf4j + slf4j-log4j12 + + + + + org.scala-lang.modules + scala-java8-compat_${scala.compat.version} + ${scala.java8.compat.version} + + + + org.webjars + openui5 + ${openui5.version} + + + org.webjars.bower + lodash + ${lodash.version} + + + org.webjars + momentjs + ${momentjs.version} + + + org.webjars + webjars-locator-core + ${webjars.locator.version} + + + org.apache.oozie + oozie-client + ${oozie.version} + + + org.apache.oozie + oozie-hadoop-auth + + + + + + org.webjars.bower + chart.js + ${bower.chart.js.version} + + + + za.co.absa.cobrix + spark-cobol_${scala.compat.version} + ${cobrix.version} + + + + com.google.guava + guava + ${guava.version} + + + za.co.absa + atum-model_${scala.compat.version} + ${atum.version} + + + + org.springframework.boot + spring-boot-starter-test + ${spring.version} + test + + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + ${embedded.mongo.version} + test + + @@ -32,106 +304,47 @@ dockerfile-maven-plugin ${dockerfile.maven.version} - ${dockerfile.menas.repository} + ${dockerfile.repository} ${dockerfile.tag} - ${dockerfile.menas.repository} - ${dockerfile.tag} + ${project.build.finalName}.war + ${dockerfile.property.file} - - org.apache.maven.plugins - maven-clean-plugin - 3.1.0 - - - - ${basedir}/ui/dist - ${basedir}/ui/node_modules - - - + + org.scalastyle + scalastyle-maven-plugin - com.github.eirslett - frontend-maven-plugin - 1.7.6 + org.springframework.boot + spring-boot-maven-plugin + ${spring.version} - target - ui - v16.19.0 + za.co.absa.enceladus.menas.Application - - - install node and npm - - install-node-and-npm - - - - install - - npm - - - install - - - - sync version to package.json - - npm - - - version ${project.parent.version} --allow-same-version - - - - ui5 build - - npm - - - run ui5-build - - - - clean node_modules - - npm - - - prune --production - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven.antrun.plugin.version} + net.alchim31.maven + scala-maven-plugin + ${scala.maven.plugin.version} - Clean up after generating component preload - generate-resources - - - - - - - - - - run + compile + testCompile + + + -Xfatal-warnings + -unchecked + -deprecation + -feature + + - org.apache.maven.plugins maven-war-plugin @@ -140,7 +353,7 @@ false - ui/dist + ui / false @@ -153,10 +366,115 @@ + + org.apache.maven.plugins + maven-resources-plugin + + false + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.plugin.version} + + + copy + validate + + + + + + + + run + + + + - + + generateComponentPreload + + + + com.github.eirslett + frontend-maven-plugin + 1.7.6 + + + install node and npm + + install-node-and-npm + + generate-resources + + + install grunt-openui5 + + npm + + + install grunt grunt-cli grunt-openui5 --save-dev + + + + generate preload + + grunt + + + openui5_preload + + + + + ui + v10.16.0 + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.plugin.version} + + + Clean up after generating component preload + generate-resources + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.plugin.version} + + + + true + + + + + + + license-check @@ -168,6 +486,7 @@ **/node_modules/** + ui/3rdParty/cRonstrue/cronstrue.min.js ui/dist/** diff --git a/migrations-cli/pom.xml b/migrations-cli/pom.xml index 8e8bcf914..3f398e3ca 100644 --- a/migrations-cli/pom.xml +++ b/migrations-cli/pom.xml @@ -27,6 +27,7 @@ ${project.parent.basedir}/scalastyle-config.xml + 3.5.3 1.7.16 @@ -50,7 +51,6 @@ org.json4s json4s-jackson_${scala.compat.version} ${json4s.version} - provided org.slf4j diff --git a/migrations/pom.xml b/migrations/pom.xml index 200219201..b4793a897 100644 --- a/migrations/pom.xml +++ b/migrations/pom.xml @@ -51,7 +51,7 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} + ${jackson.version} diff --git a/plugins-builtin/pom.xml b/plugins-builtin/pom.xml index 0b1a7dafe..58508cc35 100644 --- a/plugins-builtin/pom.xml +++ b/plugins-builtin/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 plugins-builtin jar + za.co.absa.enceladus parent 3.1.0-SNAPSHOT + ${project.parent.basedir}/scalastyle-config.xml - 7.3.0 + 5.2.2 3.1.0 + za.co.absa.enceladus plugins-api ${project.version} - - com.fasterxml.jackson.module - jackson-module-scala_${scala.compat.version} - ${jackson.version} - za.co.absa.enceladus utils @@ -53,6 +51,7 @@ kafka-avro-serializer ${kafka.avro.serializer.version} + com.github.tomakehurst @@ -73,6 +72,7 @@ test + @@ -100,15 +100,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - ${maven.compiler.plugin.version} - - ${maven.compiler.source} - ${maven.compiler.target} - - diff --git a/pom.xml b/pom.xml index 443a007bd..b884e4ee2 100644 --- a/pom.xml +++ b/pom.xml @@ -12,22 +12,27 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 za.co.absa.enceladus parent 3.1.0-SNAPSHOT pom + Enceladus Enceladus is a Dynamic Conformance Engine which allows data from different formats to be standardized to parquet and conformed to group-accepted common reference. https://github.com/AbsaOSS/enceladus 2018 + scm:git:git://github.com/AbsaOSS/enceladus.git scm:git:ssh://github.com:AbsaOSS/enceladus.git http://github.com/AbsaOSS/enceladus/tree/master HEAD + Apache License, Version 2.0 @@ -35,10 +40,12 @@ repo + ABSA Group Limited https://www.absa.africa + lokm01 @@ -88,19 +95,8 @@ Europe/Prague https://github.com/AdrianOlosutean - - lsulak - Ladislav Šulák - Europe/Prague - https://github.com/lsulak - - - jakipatryk - Bartlomiej Baj - Europe/Warsaw - https://github.com/jakipatryk - + ossrh @@ -111,6 +107,7 @@ https://oss.sonatype.org/service/local/staging/deploy/maven2/ + confluent @@ -120,33 +117,29 @@ + dao data-model + examples menas - migrations migrations-cli + migrations plugins-api plugins-builtin - rest-api spark-jobs utils + - -Xmx4000m -Dfile.encoding=UTF-8 UTF-8 UTF-8 - ${default.scalatest.argLine} - true - false + 1.8 + 1.8 1.4.13 1.16.0 - 0.8.10 1.8 - 3.8.1 - 1.8 - 1.8 1.6 3.2.0 0.12 @@ -154,29 +147,25 @@ 3.0.1 2.18.1 2.2 - 4.8.1 - 2.0.2 + 4.4.0 1.3.0 - 2.4 - 6.2.0 - 1.1.0 - 0.4.0 - 3.9.0 + 3.1.1 + 0.0.27 + 3.7.0 2.7.3 3.5.4 - 2.6.0 + 2.4.2 2.0.2 2.8.2 - 30.1-jre - 0.2.0 - 3.3.4 + 27.0.1-jre + 2.8.5 3.1.0-incubating 4.4.1 - 2.14.1 - 2.14.1 + 2.10.4 + 2.10.4 + 2.9.8 0.10.7 - 3.5.3 4.11 0-10 4.17.10 @@ -186,48 +175,46 @@ 2.22.2 3.6.4 2.7.0 + 4.3.0 1.67.1 - 1.12.2 + 1.10.0 0.1.6 - 2.12 + 2.11 0.9.0 - 2.12.17 + 2.11.12 + 2.0.0 3.2.2 4.0.0 - 3.2 - 0.2.0 - 1.0.0 + 2.4 0.2.2 0.4.0 - 3.2.2 + 2.4.4 0.5.0 - 3.8.6 + 2.4.16 + 0.3.9 1.3.5 1.0.1.RELEASE 5.0.14.RELEASE 2.0.0.RELEASE - 1.33 - 1.6.14 - 2.2.8 1.3.1 0.7.1 0.35 - 2.27.2 + 2.26.3 true ${project.basedir}/scalastyle-config.xml - docker.io/absaoss/menas - docker.io/absaoss/menas-ui + docker.io/absaoss/menas ${project.version} - 1 docker + true WARN ERROR [%p] %d{yy/MM/dd HH:mm:ss} %c{4}: %m%n false + org.scala-lang @@ -291,15 +278,6 @@ org.apache.httpcomponents httpclient - - - jackson-databind - com.fasterxml.jackson.core - - - jackson-annotations - com.fasterxml.jackson.core - @@ -316,10 +294,6 @@ org.apache.httpcomponents httpclient - - com.fasterxml.jackson.core - jackson-databind - @@ -375,6 +349,7 @@ test + src/main/scala src/test/scala @@ -406,11 +381,21 @@ + + + maven-surefire-plugin + ${maven.surefire.plugin.version} + + true + + + org.scalatest scalatest-maven-plugin ${scalatest.maven.version} + ${project.build.directory}/surefire-reports . WDF TestSuite.txt - ${scalatest.argLine} - ${span.scale.factor} + -Xmx4000m -Dfile.encoding=UTF-8 @@ -429,7 +413,6 @@ (?<!IntegrationSuite) - ${skip.unit.tests} @@ -452,6 +435,7 @@ true true + master develop @@ -463,10 +447,12 @@ true 1 + Update versions for hotfix v@{version} Update for next development version @{version} Update to hotfix version @{version} + Update versions for release v@{version} Update for next development version @{version} Update for next development version @{version} @@ -484,9 +470,13 @@ - org.jacoco - jacoco-maven-plugin - ${jacoco.version} + org.scoverage + scoverage-maven-plugin + ${scoverage.maven.plugin.version} + + ${scala.version} + true + @@ -502,6 +492,7 @@ + integration @@ -606,49 +597,5 @@ - - code-coverage - - ${default.scalatest.argLine} ${argLine} - false - false - - - - - za.co.absa.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - - - - - jacoco-prepare-agent - - prepare-agent - - - - jacoco-report - - report - - - ${project.name} - ${scala.version} - true - true - - - - - - - diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/MongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/MongoRepository.scala index ccc3e2c69..6c3fbc169 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/MongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/MongoRepository.scala @@ -48,7 +48,7 @@ abstract class MongoRepository[C](mongoDb: MongoDatabase)(implicit ct: ClassTag[ } def count(): Future[Long] = { - collection.countDocuments().toFuture() + collection.estimatedDocumentCount().toFuture() } private[repositories] def getNameFilter(name: String): Bson = { diff --git a/spark-jobs/pom.xml b/spark-jobs/pom.xml index 1401efcff..9a94233de 100644 --- a/spark-jobs/pom.xml +++ b/spark-jobs/pom.xml @@ -12,21 +12,24 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 spark-jobs jar + za.co.absa.enceladus parent 3.1.0-SNAPSHOT + ${project.parent.basedir}/scalastyle-config.xml 4.5.2 - 3.0.2 + 2.4.5 + @@ -44,17 +47,14 @@ dao ${project.version} + za.co.absa atum_${scala.compat.version} ${atum.version} - - za.co.absa - spark-data-standardization_${scala.compat.version} - ${spark.data.standardization.version} - + com.databricks @@ -64,18 +64,14 @@ za.co.absa fixed-width_${scala.compat.version} - ${fixed.width.version} + 0.1.0 za.co.absa.cobrix spark-cobol_${scala.compat.version} ${cobrix.version} - - za.co.absa - spark-partition-sizing-spark${spark.compat.version}_${scala.compat.version} - 0.2.0 - + za.co.absa.hyperdrive @@ -83,16 +79,13 @@ ${hyperdrive.version} provided + za.co.absa.commons commons_${scala.compat.version} ${absa.commons.version} - - za.co.absa - spark-commons-spark${spark.compat.version}_${scala.compat.version} - ${absa.spark.commons.version} - + com.typesafe @@ -114,11 +107,55 @@ spark-sql-kafka-0-10_${scala.compat.version} ${spark.sql.kafka.version} + + - com.github.mrpowers - spark-fast-tests_${scala.compat.version} - ${spark.fast.tests.version} - test + za.co.absa.spline + spline-core + ${spline.version} + + + net.jpountz.lz4 + lz4 + + + + + za.co.absa.spline + spline-core-spark-adapter-${spark.compat.version} + ${spline.version} + + + net.jpountz.lz4 + lz4 + + + + + za.co.absa.spline + spline-persistence-mongo + ${spline.version} + + + net.jpountz.lz4 + lz4 + + + + + za.co.absa.spline + spline-persistence-hdfs + ${spline.version} + + + net.jpountz.lz4 + lz4 + + + org.apache.hadoop + hadoop-common + + @@ -161,10 +198,9 @@ *:* - +--> org.apache.hadoop:* org.scala-lang:scala-actors org.scala-lang:scala-compiler @@ -174,18 +210,13 @@ spark-submit -(remove this)-packages org.apache.spark:spark-sql-kafka-0-10_2.11: - com.fasterxml.jackson @@ -215,6 +246,25 @@ java.lang.NoClassDefFoundError: com/fasterxml/jackson/datatype/jsr310/JavaTimeMo + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.plugin.version} + + + copy + validate + + + + + + + run + + + + org.apache.maven.plugins maven-jar-plugin diff --git a/utils/pom.xml b/utils/pom.xml index fcaaddbfd..fdd0f2a7b 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -13,18 +13,22 @@ ~ limitations under the License. --> - + 4.0.0 utils jar + za.co.absa.enceladus parent 3.1.0-SNAPSHOT + ${project.parent.basedir}/scalastyle-config.xml + org.scalatest @@ -53,50 +57,15 @@ commons_${scala.compat.version} ${absa.commons.version} - - za.co.absa - spark-commons-spark${spark.compat.version}_${scala.compat.version} - ${absa.spark.commons.version} - - - za.co.absa - spark-commons-test_${scala.compat.version} - ${absa.spark.commons.version} - - - za.co.absa - spark-data-standardization_${scala.compat.version} - ${spark.data.standardization.version} - org.scalatest scalatest-flatspec_${scala.compat.version} ${scalatest.version} test - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - com.google.guava - guava - ${guava.version} - test - - - com.github.mrpowers - spark-fast-tests_${scala.compat.version} - ${spark.fast.tests.version} - test - + + @@ -134,7 +103,7 @@ validate - + @@ -143,15 +112,7 @@ - - org.apache.maven.plugins - maven-compiler-plugin - ${maven.compiler.plugin.version} - - ${maven.compiler.source} - ${maven.compiler.target} - - +