From f1798b2cdbadcb3b41da8ea94ba1f5e318e809a2 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 7 Mar 2022 15:35:23 +0100 Subject: [PATCH] #1693 API v3: VersionedModel v3 init - VersionedModelControllerV3 base contours, empty DatasetControllerV3 to test --- .../versionedModel/VersionedSummary.scala | 2 + .../enceladus/rest_api/SpringFoxConfig.scala | 10 +- .../controllers/v3/DatasetControllerV3.scala | 34 ++++ .../v3/VersionedModelControllerV3.scala | 147 ++++++++++++++++++ .../VersionedMongoRepository.scala | 15 +- .../services/VersionedModelService.scala | 4 + .../rest_api/utils/implicits/package.scala | 2 +- 7 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index 87a0d365d..bc28bfbb0 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -16,3 +16,5 @@ package za.co.absa.enceladus.model.versionedModel case class VersionedSummary(_id: String, latestVersion: Int) + +case class VersionsList(_id: String, versions: Seq[Int]) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala index 6121ad047..b5980e65c 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala @@ -39,11 +39,17 @@ class SpringFoxConfig extends ProjectMetadata { } private def filteredPaths: Predicate[String] = - or[String](regex("/api/dataset.*"), regex("/api/schema.*"), + or[String]( + // api v2 + regex("/api/dataset.*"), regex("/api/schema.*"), regex("/api/mappingTable.*"), regex("/api/properties.*"), regex("/api/monitoring.*"),regex("/api/runs.*"), regex("/api/user.*"), regex("/api/spark.*"), - regex("/api/configuration.*") + regex("/api/configuration.*"), + + // api v3 + regex("/api-v3/datasets.*") + ) private def apiInfo = diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala new file mode 100644 index 000000000..ac207ca02 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.rest_api.services.DatasetService + +@RestController +@RequestMapping(path = Array("/api-v3/datasets")) +class DatasetControllerV3 @Autowired()(datasetService: DatasetService) + extends VersionedModelControllerV3(datasetService) { + + // TODO + // /{datasetName}/{version}/rules + // /{datasetName}/{version}/rules/{index} + // /{datasetName}/{version}/rules + +} + + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala new file mode 100644 index 000000000..7193c95ca --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -0,0 +1,147 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.controllers.v3 + +import com.mongodb.client.result.UpdateResult +import org.springframework.http.HttpStatus +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.security.core.userdetails.UserDetails +import org.springframework.web.bind.annotation._ +import za.co.absa.enceladus.model.menas.audit._ +import za.co.absa.enceladus.model.versionedModel._ +import za.co.absa.enceladus.model.{ExportableObject, UsedIn} +import za.co.absa.enceladus.rest_api.controllers.BaseController +import za.co.absa.enceladus.rest_api.exceptions.NotFoundException +import za.co.absa.enceladus.rest_api.services.VersionedModelService + +import java.util.Optional +import java.util.concurrent.CompletableFuture + +abstract class VersionedModelControllerV3[C <: VersionedModel with Product + with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { + + import za.co.absa.enceladus.rest_api.utils.implicits._ + + import scala.concurrent.ExecutionContext.Implicits.global + + // todo maybe offset/limit? + @GetMapping(Array("")) + @ResponseStatus(HttpStatus.OK) + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption) + } + + @GetMapping(Array("/{name}")) + @ResponseStatus(HttpStatus.OK) + def getVersionsList(@PathVariable name: String): CompletableFuture[Seq[Int]] = { + versionedModelService.getAllVersionsValues(name) + } + + @GetMapping(Array("/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + def getVersionDetail(@PathVariable name: String, + @PathVariable version: Int): CompletableFuture[C] = { + versionedModelService.getVersion(name, version).map { + case Some(entity) => entity + case None => throw notFound() + } + } + + @GetMapping(Array("/{name}/latest")) + @ResponseStatus(HttpStatus.OK) + def getLatestDetail(@PathVariable name: String): CompletableFuture[C] = { + versionedModelService.getLatestVersion(name).map { + case Some(entity) => entity + case None => throw NotFoundException() + } + } + + @GetMapping(Array("/{name}/audit-trail")) + @ResponseStatus(HttpStatus.OK) + def getAuditTrail(@PathVariable name: String): CompletableFuture[AuditTrail] = { + versionedModelService.getAuditTrail(name) + } + + @GetMapping(Array("/{name}/{version}/used-in")) + @ResponseStatus(HttpStatus.OK) + def usedIn(@PathVariable name: String, + @PathVariable version: Int): CompletableFuture[UsedIn] = { + versionedModelService.getUsedIn(name, Some(version)) + } + + @GetMapping(Array("/{name}/{version}/export")) + @ResponseStatus(HttpStatus.OK) + def exportSingleEntity(@PathVariable name: String, @PathVariable version: Int): CompletableFuture[String] = { + versionedModelService.exportSingleItem(name, version) + } + + @GetMapping(Array("/{name}/export")) + @ResponseStatus(HttpStatus.OK) + def exportLatestEntity(@PathVariable name: String): CompletableFuture[String] = { + versionedModelService.exportLatestItem(name) + } + + @PostMapping(Array("/{name}/import")) + @ResponseStatus(HttpStatus.CREATED) + def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, + @PathVariable name: String, + @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = { + // todo check that the name pathVar and object conform + versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { + case Some(entity) => entity // todo redo to have header Location present + case None => throw notFound() + } + } + + @PostMapping(Array("")) + @ResponseStatus(HttpStatus.CREATED) + def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C): CompletableFuture[C] = { + versionedModelService.isDisabled(item.name).flatMap { isDisabled => + if (isDisabled) { + versionedModelService.recreate(principal.getUsername, item) + } else { + versionedModelService.create(item, principal.getUsername) + } + }.map { + case Some(entity) => entity // todo redo to have header Location present + case None => throw notFound() + } + } + + @PutMapping(Array("")) + @ResponseStatus(HttpStatus.OK) + def edit(@AuthenticationPrincipal user: UserDetails, + @RequestBody item: C): CompletableFuture[C] = { + versionedModelService.update(user.getUsername, item).map { + case Some(entity) => entity + case None => throw notFound() + } + } + + @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @ResponseStatus(HttpStatus.OK) + def disable(@PathVariable name: String, + @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { + val v = if (version.isPresent) { + // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException + Some(version.get.toInt) + } else { + None + } + versionedModelService.disableVersion(name, v) + } + +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index d59d7d6ad..32fad03d3 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -16,7 +16,6 @@ package za.co.absa.enceladus.rest_api.repositories import java.time.ZonedDateTime - import org.mongodb.scala._ import org.mongodb.scala.bson._ import org.mongodb.scala.bson.collection.immutable.Document @@ -28,7 +27,7 @@ import org.mongodb.scala.model.Updates._ import org.mongodb.scala.model._ import org.mongodb.scala.result.UpdateResult import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionsList} import scala.concurrent.Future import scala.reflect.ClassTag @@ -93,6 +92,18 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).headOption().map(_.map(_.latestVersion)) } + def getAllVersionsValues(name: String): Future[Seq[Int]] = { + val pipeline = Seq( + filter(getNameFilter(name)), + Aggregates.sort(Sorts.ascending("version")), + Aggregates.group("$name", Accumulators.push("versions", "$version")) // all versions into single array + ) + collection.aggregate[VersionsList](pipeline).headOption().map { + case None => Seq.empty + case Some(verList) => verList.versions + } + } + def getAllVersions(name: String, inclDisabled: Boolean = false): Future[Seq[C]] = { val filter = if (inclDisabled) getNameFilter(name) else getNameFilterEnabled(name) collection diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index b11fae9d0..04928607f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -56,6 +56,10 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.getAllVersions(name) } + def getAllVersionsValues(name: String): Future[Seq[Int]] = { + versionedMongoRepository.getAllVersionsValues(name) + } + def getLatestVersion(name: String): Future[Option[C]] = { versionedMongoRepository.getLatestVersionValue(name).flatMap({ case Some(version) => getVersion(name, version) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala index 897bc0d17..c6aae6588 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala @@ -60,7 +60,7 @@ package object implicits { classOf[Run], classOf[Schema], classOf[SchemaField], classOf[SplineReference], classOf[RunSummary], classOf[RunDatasetNameGroupedSummary], classOf[RunDatasetVersionGroupedSummary], classOf[RuntimeConfig], classOf[OozieSchedule], classOf[OozieScheduleInstance], classOf[ScheduleTiming], classOf[DataFormat], - classOf[UserInfo], classOf[VersionedSummary], classOf[MenasAttachment], classOf[MenasReference], + classOf[UserInfo], classOf[VersionedSummary], classOf[VersionsList], classOf[MenasAttachment], classOf[MenasReference], classOf[PropertyDefinition], classOf[PropertyType], classOf[Essentiality], classOf[LandingPageInformation], classOf[TodaysRunsStatistics], classOf[DataFrameFilter]