From 0c9bf7311af3a3599fba1aaa104f0f1304a66f06 Mon Sep 17 00:00:00 2001 From: Daniel K Date: Tue, 24 May 2022 11:44:04 +0200 Subject: [PATCH] Feature/1693 api v3 delete recreate (#2055) * #1693 API v3: VersionedModel v3 disable/enable + usedIn - integrationTest cases added - 2 levels of .../used-in exist ( /{name}/used-in & /{name}/{version}/used-in). The former is used in disable checking. - UsedIn - empty/nonEmpty normalization V2 alternated between None and Some(Seq.empty) in various places, V3 consistently returns None for no references groups. - `NamedLatestVersion` generalized into a multipurpose `NamedVersion`. Small updates, thanks @benedeki - API v3 summary (NamedVersion) now contains `disabled` information - mainly on GET ...{/name} - disable fail due to nonEmpty used in now carries a wrapper with an error message (`UsedIn` wrapped in `EntityInUseException`) - Future {throw x}` replaced with `Future.failed(x)` in rest_api, cleanup --- .../za/co/absa/enceladus/model/UsedIn.scala | 41 ++- .../model/test/factories/DatasetFactory.scala | 6 +- .../model/versionedModel/NamedVersion.scala | 21 ++ .../versionedModel/VersionedSummary.scala | 14 +- .../co/absa/enceladus/model/UsedInTest.scala | 73 +++++ .../controllers/DatasetController.scala | 14 +- .../rest_api/controllers/HDFSController.scala | 5 +- .../PropertyDefinitionController.scala | 9 +- .../controllers/RestExceptionHandler.scala | 13 +- .../VersionedModelController.scala | 5 +- .../controllers/v3/DatasetControllerV3.scala | 7 +- .../v3/PropertyDefinitionControllerV3.scala | 20 +- .../v3/VersionedModelControllerV3.scala | 67 ++-- .../EndpointDisabledException.scala | 19 ++ ...ed.scala => EntityDisabledException.scala} | 2 +- .../exceptions/EntityInUseException.scala | 3 +- .../models/rest/DisabledPayload.scala | 4 +- .../repositories/DatasetMongoRepository.scala | 28 +- .../VersionedMongoRepository.scala | 35 +- .../rest_api/services/DatasetService.scala | 9 +- .../services/PropertyDefinitionService.scala | 6 +- .../rest_api/services/StatisticsService.scala | 4 +- .../services/VersionedModelService.scala | 14 +- .../services/v3/DatasetServiceV3.scala | 16 +- .../services/v3/HavingSchemaService.scala | 2 +- .../services/v3/MappingTableServiceV3.scala | 6 +- .../v3/PropertyDefinitionServiceV3.scala | 41 +++ .../services/v3/SchemaServiceV3.scala | 7 + .../rest_api/utils/implicits/package.scala | 2 +- .../controllers/BaseRestApiTest.scala | 8 +- .../DatasetApiIntegrationSuite.scala | 45 ++- ...ropertyDefinitionApiIntegrationSuite.scala | 10 +- .../SchemaApiFeaturesIntegrationSuite.scala | 52 +-- .../DatasetControllerV3IntegrationSuite.scala | 238 +++++++++++--- ...ingTableControllerV3IntegrationSuite.scala | 214 +++++++++++- ...finitionControllerV3IntegrationSuite.scala | 187 ++++++++++- .../SchemaControllerV3IntegrationSuite.scala | 307 +++++++++++++++++- .../DatasetRepositoryIntegrationSuite.scala | 6 +- 38 files changed, 1343 insertions(+), 217 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala create mode 100644 data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala rename rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/{EndpointDisabled.scala => EntityDisabledException.scala} (84%) rename data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala => rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala (84%) create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala index 02e1c6cfd..f963f9c37 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala @@ -15,15 +15,44 @@ package za.co.absa.enceladus.model +import com.fasterxml.jackson.annotation.JsonIgnore import za.co.absa.enceladus.model.menas.MenasReference -case class UsedIn( - datasets: Option[Seq[MenasReference]] = None, - mappingTables: Option[Seq[MenasReference]] = None -) { +case class UsedIn(datasets: Option[Seq[MenasReference]] = None, + mappingTables: Option[Seq[MenasReference]] = None) { - def nonEmpty: Boolean = { - datasets.exists(_.nonEmpty) || mappingTables.exists(_.nonEmpty) + /** + * Should any of the original UsedIn equal to Some(Seq.empty), it will be None after normalization + */ + @JsonIgnore + lazy val normalized: UsedIn = { + def normalizeOne(field: Option[Seq[MenasReference]]) = field match { + case None => None + case Some(x) if x.isEmpty => None + case otherNonEmpty => otherNonEmpty + } + + val normalizedDs = normalizeOne(datasets) + val normalizedMt = normalizeOne(mappingTables) + + (normalizedDs, normalizedMt) match { + case (`datasets`, `mappingTables`) => this // no normalization needed + case _ => UsedIn(normalizedDs, normalizedMt) + } + } + + @JsonIgnore + val isEmpty: Boolean = { + normalized.datasets == None && normalized.mappingTables == None } + @JsonIgnore + val nonEmpty: Boolean = !isEmpty +} + +object UsedIn { + /** + * Normalized + */ + val empty = UsedIn(None, None) } diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala index f81f73248..ebe47f5cc 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala @@ -15,13 +15,13 @@ package za.co.absa.enceladus.model.test.factories -import java.time.ZonedDateTime - import za.co.absa.enceladus.model.Dataset import za.co.absa.enceladus.model.conformanceRule._ import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.versionedModel.VersionedSummary +import java.time.ZonedDateTime + object DatasetFactory extends EntityFactory[Dataset] { override val collectionBaseName: String = "dataset" @@ -128,7 +128,7 @@ object DatasetFactory extends EntityFactory[Dataset] { } def toSummary(dataset: Dataset): VersionedSummary = { - VersionedSummary(dataset.name, dataset.version) + VersionedSummary(dataset.name, dataset.version, Set(dataset.disabled)) } } diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala new file mode 100644 index 000000000..d7348e475 --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.versionedModel + +/** + * V3 Wrapper for [[za.co.absa.enceladus.model.versionedModel.VersionedSummary]] + */ +case class NamedVersion(name: String, version: Int, disabled: Boolean) diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala index 909a193dc..95d395f23 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala @@ -15,8 +15,18 @@ package za.co.absa.enceladus.model.versionedModel -case class VersionedSummary(_id: String, latestVersion: Int) { - def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion) +/** + * V2 Representation of `VersionedSummary` - V2 does not carry disabled information + */ +case class VersionedSummaryV2(_id: String, latestVersion: Int) + +case class VersionedSummary(_id: String, latestVersion: Int, disabledSet: Set[Boolean]) { + def toV2: VersionedSummaryV2 = VersionedSummaryV2(_id, latestVersion) + + def toNamedVersion: NamedVersion = { + val disabled = disabledSet.contains(true) // legacy mixed state reported as disabled for V3 summary + NamedVersion(_id, latestVersion, disabled) + } } diff --git a/data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala b/data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala new file mode 100644 index 000000000..6b4e72551 --- /dev/null +++ b/data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import za.co.absa.enceladus.model.menas.MenasReference + +class UsedInTest extends AnyFlatSpec with Matchers { + + private val exampleRef = MenasReference(Some("collection1"), "entity1", 1) + + "UsedIn" should "correctly evaluate .nonEmpty" in { + UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).nonEmpty shouldBe true + UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).nonEmpty shouldBe true + UsedIn(Some(Seq(exampleRef)), None).nonEmpty shouldBe true + + UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).nonEmpty shouldBe true + UsedIn(None, Some(Seq(exampleRef))).nonEmpty shouldBe true + + UsedIn(Some(Seq.empty), Some(Seq.empty)).nonEmpty shouldBe false + UsedIn(None, Some(Seq.empty)).nonEmpty shouldBe false + UsedIn(Some(Seq.empty), None).nonEmpty shouldBe false + UsedIn(None, None).nonEmpty shouldBe false + } + + it should "correctly evaluate .empty" in { + UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).isEmpty shouldBe false + UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).isEmpty shouldBe false + UsedIn(Some(Seq(exampleRef)), None).isEmpty shouldBe false + + UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).isEmpty shouldBe false + UsedIn(None, Some(Seq(exampleRef))).isEmpty shouldBe false + + UsedIn(Some(Seq.empty), Some(Seq.empty)).isEmpty shouldBe true + UsedIn(None, Some(Seq.empty)).isEmpty shouldBe true + UsedIn(Some(Seq.empty), None).isEmpty shouldBe true + UsedIn(None, None).isEmpty shouldBe true + } + + it should "normalize" in { + UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).normalized shouldBe UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))) + + UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).normalized shouldBe UsedIn(Some(Seq(exampleRef)), None) + UsedIn(Some(Seq(exampleRef)), None).normalized shouldBe UsedIn(Some(Seq(exampleRef)), None) + + UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).normalized shouldBe UsedIn(None, Some(Seq(exampleRef))) + UsedIn(None, Some(Seq(exampleRef))).normalized shouldBe UsedIn(None, Some(Seq(exampleRef))) + + UsedIn(Some(Seq.empty), Some(Seq.empty)).normalized shouldBe UsedIn(None, None) + UsedIn(None, None).normalized shouldBe UsedIn(None, None) + + + + + } + + + +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala index fcdd69e91..bc0ec4f1c 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala @@ -30,7 +30,7 @@ import za.co.absa.enceladus.rest_api.services.DatasetService import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.model.properties.PropertyDefinition -import za.co.absa.enceladus.model.versionedModel.VersionedSummary +import za.co.absa.enceladus.model.versionedModel.VersionedSummaryV2 import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel.Constants.DefaultValidationLevelName @@ -49,9 +49,9 @@ class DatasetController @Autowired()(datasetService: DatasetService) @GetMapping(Array("/latest")) @ResponseStatus(HttpStatus.OK) def getLatestVersions(@RequestParam(value = "missing_property", required = false) - missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummaryV2]] = { datasetService.getLatestVersions(missingProperty.toScalaOption) - .map(datasets => datasets.map(dataset => VersionedSummary(dataset.name, dataset.version))) + .map(datasets => datasets.map(dataset => VersionedSummaryV2(dataset.name, dataset.version))) } @PostMapping(Array("/{datasetName}/rule/create")) @@ -67,7 +67,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) case Some((ds, validation)) => ds // v2 disregarding validation case _ => throw notFound() } - case _ => throw notFound() + case _ => Future.failed(notFound()) } } yield res } @@ -104,7 +104,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) } else { datasetService.filterProperties(dsProperties, DatasetController.paramsToPropertyDefinitionFilter(scalaFilterMap)) } - case None => throw notFound() + case None => Future.failed(notFound()) } } @@ -126,7 +126,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def getPropertiesValidation(@PathVariable datasetName: String, @PathVariable datasetVersion: Int): CompletableFuture[Validation] = { datasetService.getVersion(datasetName, datasetVersion).flatMap { case Some(entity) => datasetService.validateProperties(entity.propertiesAsMap, forRun = false) - case None => throw notFound() + case None => Future.failed(notFound()) } } @@ -135,7 +135,7 @@ class DatasetController @Autowired()(datasetService: DatasetService) def getPropertiesValidationForRun(@PathVariable datasetName: String, @PathVariable datasetVersion: Int): CompletableFuture[Validation] = { datasetService.getVersion(datasetName, datasetVersion).flatMap { case Some(entity) => datasetService.validateProperties(entity.propertiesAsMap, forRun = true) - case None => throw notFound() + case None => Future.failed(notFound()) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala index d5ed48dae..5953c8570 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala @@ -16,7 +16,6 @@ package za.co.absa.enceladus.rest_api.controllers import java.util.concurrent.CompletableFuture - import org.apache.hadoop.fs.Path import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.HttpStatus @@ -24,6 +23,8 @@ import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.menas.HDFSFolder import za.co.absa.enceladus.rest_api.services.HDFSService +import scala.concurrent.Future + @RestController @RequestMapping(Array("/api/hdfs")) class HDFSController @Autowired() (hdfsService: HDFSService) extends BaseController { @@ -41,7 +42,7 @@ class HDFSController @Autowired() (hdfsService: HDFSService) extends BaseControl if (exists) { hdfsService.getFolder(path) } else { - throw notFound() + Future.failed(notFound()) } } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala index 11c26b063..24dc896bc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala @@ -26,7 +26,6 @@ import org.springframework.security.access.prepost.PreAuthorize import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ -import za.co.absa.enceladus.rest_api.exceptions.EndpointDisabled import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService import za.co.absa.enceladus.model.ExportableObject import za.co.absa.enceladus.model.properties.PropertyDefinition @@ -39,15 +38,15 @@ import scala.concurrent.ExecutionContext.Implicits.global */ @RestController @RequestMapping(path = Array("/api/properties/datasets"), produces = Array("application/json")) -class PropertyDefinitionController @Autowired()(propertyDefService: PropertyDefinitionService) - extends VersionedModelController(propertyDefService) { +class PropertyDefinitionController @Autowired()(propertyDefinitionService: PropertyDefinitionService) + extends VersionedModelController(propertyDefinitionService) { import za.co.absa.enceladus.rest_api.utils.implicits._ @GetMapping(Array("")) def getAllDatasetProperties(): CompletableFuture[Seq[PropertyDefinition]] = { logger.info("retrieving all dataset properties in full") - propertyDefService.getLatestVersions() + propertyDefinitionService.getLatestVersions() } @PostMapping(Array("")) @@ -63,8 +62,6 @@ class PropertyDefinitionController @Autowired()(propertyDefService: PropertyDefi val location: URI = new URI(s"/api/properties/datasets/${entity.name}/${entity.version}") ResponseEntity.created(location).body(entity) } - - // TODO: Location header would make sense for the underlying VersionedModelController.create, too. Issue #1611 } @GetMapping(Array("/{propertyName}")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala index 5ef13e650..115c18469 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala @@ -62,11 +62,16 @@ class RestExceptionHandler { ResponseEntity.notFound().build[Any]() } - @ExceptionHandler(value = Array(classOf[EndpointDisabled])) - def handleEndpointDisabled(exception: EndpointDisabled): ResponseEntity[Any] = { + @ExceptionHandler(value = Array(classOf[EndpointDisabledException])) + def handleEndpointDisabled(exception: EndpointDisabledException): ResponseEntity[Any] = { ResponseEntity.status(HttpStatus.I_AM_A_TEAPOT).build[Any]() // Could change for LOCKED but I like this more } + @ExceptionHandler(value = Array(classOf[EntityDisabledException])) + def handleEntityDisabled(exception: EntityDisabledException): ResponseEntity[EntityDisabledException] = { + ResponseEntity.status(HttpStatus.BAD_REQUEST).body(exception) + } + @ExceptionHandler(value = Array(classOf[SchemaParsingException])) def handleBadRequestException(exception: SchemaParsingException): ResponseEntity[Any] = { val response = RestResponse(exception.message, Option(SchemaParsingError.fromException(exception))) @@ -111,8 +116,8 @@ class RestExceptionHandler { } @ExceptionHandler(value = Array(classOf[EntityInUseException])) - def handleValidationException(exception: EntityInUseException): ResponseEntity[UsedIn] = { - ResponseEntity.badRequest().body(exception.usedIn) + def handleValidationException(exception: EntityInUseException): ResponseEntity[EntityInUseException] = { + ResponseEntity.badRequest().body(exception) } @ExceptionHandler(value = Array(classOf[MethodArgumentTypeMismatchException])) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index e129a8b82..af0c844f0 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -23,7 +23,7 @@ import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn} -import za.co.absa.enceladus.model.versionedModel._ +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummaryV2} import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.services.VersionedModelService import za.co.absa.enceladus.model.menas.audit._ @@ -38,8 +38,9 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @GetMapping(Array("/list", "/list/{searchQuery}")) @ResponseStatus(HttpStatus.OK) - def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummaryV2]] = { versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) + .map(_.map(_.toV2)) } @GetMapping(Array("/searchSuggestions")) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala index b56cf937a..6f69ef820 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala @@ -28,6 +28,7 @@ import za.co.absa.enceladus.rest_api.utils.implicits._ import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future @RestController @RequestMapping(path = Array("/api-v3/datasets")) @@ -57,14 +58,12 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) case Some((entity, validation)) => // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties") - .body(validation) // todo include in tests + .body(validation) case None => throw notFound() } } } - // todo putIntoInfoFile switch needed? - @GetMapping(Array("/{name}/{version}/rules")) @ResponseStatus(HttpStatus.OK) def getConformanceRules(@PathVariable name: String, @@ -90,7 +89,7 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3) suffix = s"/rules/$addedRuleOrder").body(validation) case _ => throw notFound() } - case None => throw notFound() + case None => Future.failed(notFound()) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala index e80c86778..1b3063349 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala @@ -15,7 +15,6 @@ package za.co.absa.enceladus.rest_api.controllers.v3 -import com.mongodb.client.result.UpdateResult import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.access.prepost.PreAuthorize @@ -24,16 +23,16 @@ import org.springframework.security.core.userdetails.UserDetails import org.springframework.web.bind.annotation._ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.{ExportableObject, Validation} -import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload +import za.co.absa.enceladus.rest_api.services.v3.PropertyDefinitionServiceV3 -import java.util.Optional import java.util.concurrent.CompletableFuture import javax.servlet.http.HttpServletRequest @RestController @RequestMapping(path = Array("/api-v3/property-definitions/datasets"), produces = Array("application/json")) -class PropertyDefinitionControllerV3 @Autowired()(propertyDefService: PropertyDefinitionService) - extends VersionedModelControllerV3(propertyDefService) { +class PropertyDefinitionControllerV3 @Autowired()(propertyDefinitionService: PropertyDefinitionServiceV3) + extends VersionedModelControllerV3(propertyDefinitionService) { // super-class implementation is sufficient, but the following changing endpoints need admin-auth @@ -69,16 +68,11 @@ class PropertyDefinitionControllerV3 @Autowired()(propertyDefService: PropertyDe super.edit(user, name, version, item, request) } - @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @DeleteMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) @PreAuthorize("@authConstants.hasAdminRole(authentication)") - override def disable(@PathVariable name: String, - @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { - - super.disable(name, version) + override def disable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { + super.disable(name) } - - // todo add "enable" with preAuth check when available, too - } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 578a4a4f1..911604609 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -26,6 +26,8 @@ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation} import za.co.absa.enceladus.rest_api.controllers.BaseController import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey +import za.co.absa.enceladus.rest_api.exceptions.{EntityDisabledException, NotFoundException, ValidationException} +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload import za.co.absa.enceladus.rest_api.services.VersionedModelService import java.net.URI @@ -36,7 +38,7 @@ import scala.concurrent.Future import scala.util.{Failure, Success, Try} object VersionedModelControllerV3 { - val LatestVersionKey = "latest" + final val LatestVersionKey = "latest" } abstract class VersionedModelControllerV3[C <: VersionedModel with Product @@ -46,19 +48,19 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product import scala.concurrent.ExecutionContext.Implicits.global - // todo maybe offset/limit? + // todo maybe offset/limit -> Issue #2060 @GetMapping(Array("")) @ResponseStatus(HttpStatus.OK) - def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedLatestVersion]] = { + def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[NamedVersion]] = { versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption) - .map(_.map(_.toNamedLatestVersion)) + .map(_.map(_.toNamedVersion)) } @GetMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedLatestVersion] = { - versionedModelService.getLatestVersionSummary(name) map { - case Some(entity) => entity.toNamedLatestVersion + def getVersionSummaryForEntity(@PathVariable name: String): CompletableFuture[NamedVersion] = { + versionedModelService.getLatestVersionSummary(name).map { + case Some(entity) => entity.toNamedVersion case None => throw notFound() } } @@ -87,6 +89,12 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product forVersionExpression(name, version) { case (name, versionInt) => versionedModelService.getUsedIn(name, Some(versionInt)) } } + @GetMapping(Array("/{name}/used-in")) + @ResponseStatus(HttpStatus.OK) + def usedIn(@PathVariable name: String): CompletableFuture[UsedIn] = { + versionedModelService.getUsedIn(name, None) + } + @GetMapping(Array("/{name}/{version}/export")) @ResponseStatus(HttpStatus.OK) def exportSingleEntity(@PathVariable name: String, @PathVariable version: String): CompletableFuture[String] = { @@ -125,7 +133,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { versionedModelService.isDisabled(item.name).flatMap { isDisabled => if (isDisabled) { - versionedModelService.recreate(principal.getUsername, item) + Future.failed(EntityDisabledException(s"Entity ${item.name} is disabled. Enable it first (PUT) to push new versions (PUT).")) } else { versionedModelService.create(item, principal.getUsername) } @@ -148,25 +156,42 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } else if (version != item.version) { Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) } else { - versionedModelService.update(user.getUsername, item).map { - case Some((updatedEntity, validation)) => - createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) - case None => throw notFound() + versionedModelService.isDisabled(item.name).flatMap { isDisabled => + if (isDisabled) { + throw EntityDisabledException(s"Entity ${item.name} is disabled. Enable it first to create new versions.") + } else { + versionedModelService.update(user.getUsername, item).map { + case Some((updatedEntity, validation)) => + createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) + case None => throw notFound() + } + } } } } - @DeleteMapping(Array("/{name}", "/{name}/{version}")) + @PutMapping(Array("/{name}")) @ResponseStatus(HttpStatus.OK) - def disable(@PathVariable name: String, - @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = { - val v = if (version.isPresent) { - // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException - Some(version.get.toInt) - } else { - None + def enable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { + versionedModelService.enableEntity(name).map { updateResult => // always enabling all version of the entity + if (updateResult.getMatchedCount > 0) { + DisabledPayload(disabled = false) + } else { + throw NotFoundException(s"No versions for entity $name found to be enabled.") + } + } + } + + @DeleteMapping(Array("/{name}")) + @ResponseStatus(HttpStatus.OK) + def disable(@PathVariable name: String): CompletableFuture[DisabledPayload] = { + versionedModelService.disableVersion(name, None).map { updateResult => // always disabling all version of the entity + if (updateResult.getMatchedCount > 0) { + DisabledPayload(disabled = true) + } else { + throw NotFoundException(s"No versions for entity $name found to be disabled.") + } } - versionedModelService.disableVersion(name, v) } /** diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala new file mode 100644 index 000000000..76b919e1a --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabledException.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.exceptions + +case class EndpointDisabledException(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) + diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabled.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala similarity index 84% rename from rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabled.scala rename to rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala index fb745d5df..54a770d9d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EndpointDisabled.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala @@ -15,5 +15,5 @@ package za.co.absa.enceladus.rest_api.exceptions -case class EndpointDisabled(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) +case class EntityDisabledException(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala index 9661f8767..2235654e0 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityInUseException.scala @@ -17,4 +17,5 @@ package za.co.absa.enceladus.rest_api.exceptions import za.co.absa.enceladus.model.UsedIn -case class EntityInUseException(usedIn: UsedIn) extends Exception() +case class EntityInUseException(message: String = "There are dependencies present preventing the action", + usedIn: UsedIn) extends Exception() diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala similarity index 84% rename from data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala rename to rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala index 8c44eff11..009cefdb5 100644 --- a/data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/models/rest/DisabledPayload.scala @@ -13,6 +13,6 @@ * limitations under the License. */ -package za.co.absa.enceladus.model.versionedModel +package za.co.absa.enceladus.rest_api.models.rest -case class NamedLatestVersion(name: String, version: Int) +case class DisabledPayload(disabled: Boolean) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala index cf8c83a43..240d09752 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/DatasetMongoRepository.scala @@ -15,18 +15,17 @@ package za.co.absa.enceladus.rest_api.repositories +import org.mongodb.scala.model.Projections._ +import org.mongodb.scala.model.{Filters, Sorts} import org.mongodb.scala.{Completed, MongoDatabase} import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Repository -import za.co.absa.enceladus.model.{Dataset, MappingTable, Schema} - -import scala.reflect.ClassTag -import za.co.absa.enceladus.model.menas.MenasReference -import org.mongodb.scala.model.Filters -import org.mongodb.scala.model.Projections._ import za.co.absa.enceladus.model +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.{Dataset, MappingTable, Schema} import scala.concurrent.Future +import scala.reflect.ClassTag object DatasetMongoRepository { val collectionBaseName: String = "dataset" @@ -77,13 +76,24 @@ class DatasetMongoRepository @Autowired()(mongoDb: MongoDatabase) * @return List of Menas references to Datasets, which contain the relevant conformance rules */ def containsMappingRuleRefEqual(refColVal: (String, Any)*): Future[Seq[MenasReference]] = { - - val equals = Filters.and(refColVal.map(col => Filters.eq(col._1, col._2)) :_*) - val filter = Filters.elemMatch("conformance", equals) + // The gist of the find query that this method is based on; testable in a mongo client + // { $and : [ + // {... non disabled filter here...}, + // {"conformance": {"$elemMatch": {$and : [ + // {"mappingTable": "AnotherAwesomeMappingTable"}, // from refColVal + // {"mappingTableVersion": 1} // from refColVal + // ]}}} + // ]} + + val equalConditionsFilter = Filters.and(refColVal.map { + case (key, value) => Filters.eq(key, value) + } :_*) + val filter = Filters.and(getNotDisabledFilter, Filters.elemMatch("conformance", equalConditionsFilter)) collection .find[MenasReference](filter) .projection(fields(include("name", "version"), computed("collection", collectionBaseName))) + .sort(Sorts.ascending("name", "version")) .toFuture() } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala index 90f9dd6e8..d3fe970f3 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/repositories/VersionedMongoRepository.scala @@ -27,7 +27,7 @@ import org.mongodb.scala.model.Updates._ import org.mongodb.scala.model._ import org.mongodb.scala.result.UpdateResult import za.co.absa.enceladus.model.menas._ -import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary, VersionedSummaryV2} import scala.concurrent.Future import scala.reflect.ClassTag @@ -68,10 +68,13 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab } val pipeline = Seq( filter(Filters.and(searchFilter, getNotDisabledFilter)), - Aggregates.group("$name", Accumulators.max("latestVersion", "$version")), + Aggregates.group("$name", + Accumulators.max("latestVersion", "$version") + ), sort(Sorts.ascending("_id")) ) - collection.aggregate[VersionedSummary](pipeline).toFuture() + collection.aggregate[VersionedSummaryV2](pipeline).toFuture() + .map(_.map(summaryV2 => VersionedSummary(summaryV2._id, summaryV2.latestVersion, Set(false)))) // because of the notDisabled filter } def getLatestVersions(missingProperty: Option[String]): Future[Seq[C]] = { @@ -90,7 +93,10 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { val pipeline = Seq( filter(getNameFilter(name)), - Aggregates.group("$name", Accumulators.max("latestVersion", "$version")) + Aggregates.group("$name", + Accumulators.max("latestVersion", "$version"), + Accumulators.addToSet("disabledSet", "$disabled") + ) ) collection.aggregate[VersionedSummary](pipeline).headOption() } @@ -134,6 +140,7 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab } + // for V3 usage: version = None def disableVersion(name: String, version: Option[Int], username: String): Future[UpdateResult] = { collection.updateMany(getNameVersionFilter(name, version), combine( set("disabled", true), @@ -141,6 +148,14 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab set("userDisabled", username))).toFuture() } + // V3 only + def enableAllVersions(name: String, username: String): Future[UpdateResult] = { + collection.updateMany(getNameVersionFilter(name, version = None), combine( + set("disabled", false), + set("dateDisabled", ZonedDateTime.now()), + set("userDisabled", username))).toFuture() + } + def isDisabled(name: String): Future[Boolean] = { val pipeline = Seq(filter(getNameFilter(name)), Aggregates.addFields(Field("enabled", BsonDocument("""{$toInt: {$not: "$disabled"}}"""))), @@ -167,6 +182,18 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab .toFuture() } + def findRefContainedAsKey(refNameCol: String, name: String): Future[Seq[MenasReference]] = { + + // `refNameCol` contains a map where the `name` is the key, so this is e.g. {"properties.keyName" : {$exists : true}} + val filter = Filters.and(getNotDisabledFilter, Filters.exists(s"$refNameCol.$name", true)) + + collection + .find[MenasReference](filter) + .projection(fields(include("name", "version"), computed("collection", collectionBaseName))) + .sort(Sorts.ascending("name", "version")) + .toFuture() + } + private def collectLatestVersions(postAggFilter: Option[Bson]): Future[Seq[C]] = { val pipeline = Seq( filter(Filters.notEqual("disabled", true)), diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index e707c98e1..6a9028895 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -17,8 +17,7 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.rest_api.repositories.DatasetMongoRepository -import za.co.absa.enceladus.rest_api.repositories.OozieRepository +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository, PropertyDefinitionMongoRepository} import za.co.absa.enceladus.rest_api.services.DatasetService.RuleValidationsAndFields import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, _} import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance @@ -39,7 +38,7 @@ import scala.util.{Failure, Success} @Service class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, - datasetPropertyDefinitionService: PropertyDefinitionService) + propertyDefinitionService: PropertyDefinitionService) extends VersionedModelService(datasetMongoRepository) { import scala.concurrent.ExecutionContext.Implicits.global @@ -209,7 +208,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository def validateProperties(properties: Map[String, String], forRun: Boolean = false): Future[Validation] = { - datasetPropertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => + propertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => val propDefsMap = Map(propDefs.map { propDef => (propDef.name, propDef) }: _*) // map(key, propDef) val existingPropsValidation = properties.toSeq.map { case (key, value) => validateExistingProperty(key, value, propDefsMap) } @@ -221,7 +220,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } def filterProperties(properties: Map[String, String], filter: PropertyDefinition => Boolean): Future[Map[String, String]] = { - datasetPropertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => + propertyDefinitionService.getLatestVersions().map { propDefs: Seq[PropertyDefinition] => val filteredPropDefNames = propDefs.filter(filter).map(_.name).toSet properties.filterKeys(filteredPropDefNames.contains) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala index 7b4e5a039..cb5bc9b08 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala @@ -23,13 +23,15 @@ import za.co.absa.enceladus.model.properties.PropertyDefinition import scala.concurrent.Future -@Service +@Service("propertyDefinitionService") // by-name qualifier: V2 implementations use the base implementation, not v3 class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: PropertyDefinitionMongoRepository) extends VersionedModelService(propertyDefMongoRepository) { import scala.concurrent.ExecutionContext.Implicits.global - override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = Future.successful(UsedIn()) + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + Future.successful(UsedIn()) + } override def update(username: String, propertyDef: PropertyDefinition): Future[Option[(PropertyDefinition, Validation)]] = { super.update(username, propertyDef.name, propertyDef.version) { latest => diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala index 2af9a0c1e..878fdb85f 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/StatisticsService.scala @@ -23,10 +23,10 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @Component -class StatisticsService @Autowired() (propertyDefService: PropertyDefinitionService, datasetService: DatasetService){ +class StatisticsService @Autowired() (propertyDefinitionService: PropertyDefinitionService, datasetService: DatasetService){ //#TODO find optimizations #1897 def getPropertiesWithMissingCount(): Future[Seq[PropertyDefinitionStats]] = { - val propertyDefsFuture = propertyDefService.getLatestVersions() + val propertyDefsFuture = propertyDefinitionService.getLatestVersions() propertyDefsFuture .map { (props: Seq[PropertyDefinition]) => val propertiesWithMissingCounts: Seq[Future[PropertyDefinitionStats]] = props.map(propertyDef => diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index a1b1c5236..de26f6e36 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -267,6 +267,17 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit versionedMongoRepository.findRefEqual(refNameCol, refVersionCol, name, version) } + /** + * Enables all versions of the entity by name. + * @param name + */ + def enableEntity(name: String): Future[UpdateResult] = { + val auth = SecurityContextHolder.getContext.getAuthentication + val principal = auth.getPrincipal.asInstanceOf[UserDetails] + + versionedMongoRepository.enableAllVersions(name, principal.getUsername) + } + def disableVersion(name: String, version: Option[Int]): Future[UpdateResult] = { val auth = SecurityContextHolder.getContext.getAuthentication val principal = auth.getPrincipal.asInstanceOf[UserDetails] @@ -278,7 +289,8 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit private def disableVersion(name: String, version: Option[Int], usedIn: UsedIn, principal: UserDetails): Future[UpdateResult] = { if (usedIn.nonEmpty) { - throw EntityInUseException(usedIn) + val entityVersionStr = s"""entity "$name"${ version.map(" v" + _).getOrElse("")}""" // either "entity MyName" or "entity MyName v23" + throw EntityInUseException(s"""Cannot disable $entityVersionStr, because it is used in the following entities""", usedIn) } else { versionedMongoRepository.disableVersion(name, version, principal.getUsername) } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index f5dff3127..afd647010 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -18,11 +18,11 @@ package za.co.absa.enceladus.rest_api.services.v3 import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} -import za.co.absa.enceladus.model.{Dataset, Validation} +import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} import za.co.absa.enceladus.rest_api.exceptions.ValidationException import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, OozieRepository} import za.co.absa.enceladus.rest_api.services.DatasetService._ -import za.co.absa.enceladus.rest_api.services.{DatasetService, MappingTableService, PropertyDefinitionService, SchemaService} +import za.co.absa.enceladus.rest_api.services.DatasetService import scala.concurrent.Future @@ -30,10 +30,10 @@ import scala.concurrent.Future @Service class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, - datasetPropertyDefinitionService: PropertyDefinitionService, - mappingTableService: MappingTableService, - val schemaService: SchemaService) - extends DatasetService(datasetMongoRepository, oozieRepository, datasetPropertyDefinitionService) + propertyDefinitionService: PropertyDefinitionServiceV3, + mappingTableService: MappingTableServiceV3, + val schemaService: SchemaServiceV3) + extends DatasetService(datasetMongoRepository, oozieRepository, propertyDefinitionService) with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global @@ -91,6 +91,10 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito } yield update } + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + super.getUsedIn(name, version).map(_.normalized) + } + } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala index 498721d8b..cb8104e15 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/HavingSchemaService.scala @@ -21,7 +21,7 @@ import za.co.absa.enceladus.rest_api.services.SchemaService import scala.concurrent.{ExecutionContext, Future} trait HavingSchemaService { - protected def schemaService: SchemaService + protected def schemaService: SchemaServiceV3 def validateSchemaExists(schemaName: String, schemaVersion: Int) (implicit executionContext: ExecutionContext): Future[Validation] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala index a97e9e11a..12fb346f8 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala @@ -26,7 +26,7 @@ import scala.concurrent.Future @Service class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, - val schemaService: SchemaService) + val schemaService: SchemaServiceV3) extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) with HavingSchemaService { import scala.concurrent.ExecutionContext.Implicits.global @@ -36,6 +36,10 @@ class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTab originalValidation <- super.validate(item) mtSchemaValidation <- validateSchemaExists(item.schemaName, item.schemaVersion) } yield originalValidation.merge(mtSchemaValidation) + } + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + super.getUsedIn(name, version).map(_.normalized) } + } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala new file mode 100644 index 000000000..bab32b4df --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.{UsedIn, Validation} +import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, PropertyDefinitionMongoRepository} +import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService + +import scala.concurrent.Future + +@Service +class PropertyDefinitionServiceV3 @Autowired()(propertyDefMongoRepository: PropertyDefinitionMongoRepository, + datasetMongoRepository: DatasetMongoRepository) + extends PropertyDefinitionService(propertyDefMongoRepository) { + + import scala.concurrent.ExecutionContext.Implicits.global + + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + for { + usedInD <- datasetMongoRepository.findRefContainedAsKey("properties", name) + optionalUsedInD = if (usedInD.isEmpty) None else Some(usedInD) + } yield UsedIn(optionalUsedInD, None) + } + +} diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala index 1622a7b6b..7878fc7c7 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala @@ -34,6 +34,8 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, sparkMenasConvertor: SparkMenasSchemaConvertor) extends SchemaService(schemaMongoRepository, mappingTableMongoRepository, datasetMongoRepository, sparkMenasConvertor) { + import scala.concurrent.ExecutionContext.Implicits.global + override def validate(item: Schema): Future[Validation] = { if (item.fields.isEmpty) { // V3 disallows empty schema fields - V2 allowed it at first that to get updated by an attachment upload/remote-load @@ -47,4 +49,9 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, override protected def updateFields(current: Schema, update: Schema) : Schema = { current.setDescription(update.description).asInstanceOf[Schema].copy(fields = update.fields) } + + override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { + super.getUsedIn(name, version).map(_.normalized) + } + } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala index 897bc0d17..5b0a4afd1 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/utils/implicits/package.scala @@ -60,7 +60,7 @@ package object implicits { classOf[Run], classOf[Schema], classOf[SchemaField], classOf[SplineReference], classOf[RunSummary], classOf[RunDatasetNameGroupedSummary], classOf[RunDatasetVersionGroupedSummary], classOf[RuntimeConfig], classOf[OozieSchedule], classOf[OozieScheduleInstance], classOf[ScheduleTiming], classOf[DataFormat], - classOf[UserInfo], classOf[VersionedSummary], classOf[MenasAttachment], classOf[MenasReference], + classOf[UserInfo], classOf[VersionedSummary], classOf[VersionedSummaryV2], classOf[MenasAttachment], classOf[MenasReference], classOf[PropertyDefinition], classOf[PropertyType], classOf[Essentiality], classOf[LandingPageInformation], classOf[TodaysRunsStatistics], classOf[DataFrameFilter] diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala index 1670e6ce7..1206b1b15 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/BaseRestApiTest.scala @@ -184,13 +184,13 @@ abstract class BaseRestApiTest(loginPath: String, apiPath: String) extends BaseR sendAsync(HttpMethod.PUT, urlPath, headers, bodyOpt) } - def sendDelete[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), - bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { + def sendDelete[T](urlPath: String, headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { send(HttpMethod.DELETE, urlPath, headers) } - def sendDeleteByAdmin[B, T](urlPath: String, headers: HttpHeaders = new HttpHeaders(), - bodyOpt: Option[B] = None)(implicit ct: ClassTag[T]): ResponseEntity[T] = { + def sendDeleteByAdmin[T](urlPath: String, headers: HttpHeaders = new HttpHeaders()) + (implicit ct: ClassTag[T]): ResponseEntity[T] = { sendByAdmin(HttpMethod.DELETE, urlPath, headers) } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala index 837d7d1c2..4f13250da 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala @@ -23,7 +23,6 @@ import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule import za.co.absa.enceladus.model.dataFrameFilter._ -import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ @@ -437,4 +436,48 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl } } + s"DELETE $apiUrl/disable/{name}/{version}" can { + "return 200" when { + "a Dataset with the given name and version exists" should { + "disable only the dataset with the given name and version" in { + val dsA = DatasetFactory.getDummyDataset(name = "dsA", version = 1) + val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1) + datasetFixture.add(dsA, dsB) + + val response = sendDelete[String](s"$apiUrl/disable/dsA/1") + + assertOk(response) + + val actual = response.getBody + val expected = """{"matchedCount":1,"modifiedCount":1,"upsertedId":null,"modifiedCountAvailable":true}""" + assert(actual == expected) + } + } + "multiple versions of the Dataset with the given name exist" should { + "disable the specified version of the Dataset" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2) + datasetFixture.add(dsA1, dsA2) + + val response = sendDelete[String](s"$apiUrl/disable/dsA/1") + + assertOk(response) + + val actual = response.getBody + val expected = """{"matchedCount":1,"modifiedCount":1,"upsertedId":null,"modifiedCountAvailable":true}""" + assert(actual == expected) + } + } + + "no Dataset with the given name exists" should { + "disable nothing" in { + val response = sendDelete[String](s"$apiUrl/disable/aDataset/1") + + assertNotFound(response) + // Beware that, sadly, V2 Schemas returns 200 on disable of non-existent entity while V2 Datasets returns 404 + // This is due to getUsedIn implementation (non) checking the entity existence. + } + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala index c8b9ad592..7c451b08d 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala @@ -138,7 +138,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "otherPropertyDefinition", version = 1) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition") assertOk(response) @@ -153,7 +153,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition") assertOk(response) @@ -173,7 +173,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "otherPropertyDefinition", version = 1) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition/1") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition/1") assertOk(response) val actual = response.getBody @@ -191,7 +191,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2) propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) - val response = sendDeleteByAdmin[PropertyDefinition, String](deleteUrl) + val response = sendDeleteByAdmin[String](deleteUrl) assertOk(response) val actual = response.getBody @@ -203,7 +203,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTestV2 with Befor "no PropertyDefinition with the given name exists" should { "disable nothing" in { - val response = sendDeleteByAdmin[PropertyDefinition, String](s"$apiUrl/disable/propertyDefinition/1") + val response = sendDeleteByAdmin[String](s"$apiUrl/disable/propertyDefinition/1") assertOk(response) val actual = response.getBody diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala index 6251e5f71..a0a95a40d 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/SchemaApiFeaturesIntegrationSuite.scala @@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.integration.controllers import java.io.File import java.nio.file.{Files, Path} - import com.github.tomakehurst.wiremock.WireMockServer import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder import com.github.tomakehurst.wiremock.core.WireMockConfiguration @@ -40,6 +39,7 @@ import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException import za.co.absa.enceladus.restapi.TestResourcePath import scala.collection.immutable.HashMap @@ -223,7 +223,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "otherSchema", version = 1) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -238,7 +238,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -255,7 +255,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -272,7 +272,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -283,7 +283,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd } "no Schema with the given name exists" should { "disable nothing" in { - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema") + val response = sendDelete[String](s"$apiUrl/disable/schema") assertOk(response) @@ -303,16 +303,18 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq(MenasReference(None, "dataset", 1))), Some(Seq())) + val expected = EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset", 1))), Some(Seq())) + ) assert(actual == expected) } } - "some version of the Schema is used by a enabled MappingTable" should { + "some version of the Schema is used by an enabled MappingTable" should { "return a list of the entities the Schema is used in" in { val mappingTable = MappingTableFactory.getDummyMappingTable(name = "mapping", schemaName = "schema", schemaVersion = 1, disabled = false) mappingTableFixture.add(mappingTable) @@ -320,12 +322,14 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping", 1)))) + val expected = EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping", 1)))) + ) assert(actual == expected) } } @@ -340,7 +344,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "otherSchema", version = 1) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -355,7 +359,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -372,7 +376,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -389,7 +393,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/2") + val response = sendDelete[String](s"$apiUrl/disable/schema/2") assertOk(response) @@ -406,7 +410,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -423,7 +427,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/2") + val response = sendDelete[String](s"$apiUrl/disable/schema/2") assertOk(response) @@ -434,7 +438,7 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd } "no Schema with the given name exists" should { "disable nothing" in { - val response = sendDelete[Schema, String](s"$apiUrl/disable/schema/1") + val response = sendDelete[String](s"$apiUrl/disable/schema/1") assertOk(response) @@ -455,12 +459,14 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema/1") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema/1") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq(MenasReference(None, "dataset1", 1))), Some(Seq())) + val expected = EntityInUseException("""Cannot disable entity "schema" v1, because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1))), Some(Seq())) + ) assert(actual == expected) } } @@ -473,12 +479,14 @@ class SchemaApiFeaturesIntegrationSuite extends BaseRestApiTestV2 with BeforeAnd val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) schemaFixture.add(schema1, schema2) - val response = sendDelete[Schema, UsedIn](s"$apiUrl/disable/schema/1") + val response = sendDelete[EntityInUseException](s"$apiUrl/disable/schema/1") assertBadRequest(response) val actual = response.getBody - val expected = UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping1", 1)))) + val expected = EntityInUseException("""Cannot disable entity "schema" v1, because it is used in the following entities""", + UsedIn(Some(Seq()), Some(Seq(MenasReference(None, "mapping1", 1)))) + ) assert(actual == expected) } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index 5105fff5a..a036eb426 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -28,12 +28,12 @@ import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion +import za.co.absa.enceladus.model.versionedModel.NamedVersion import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.EntityDisabledException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ - -import scala.collection.JavaConverters._ +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -86,29 +86,6 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA assert(actual == expected) } - "create a new version of Dataset" when { - "the dataset is disabled (i.e. all version are disabled)" in { - schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) - val dataset1 = DatasetFactory.getDummyDataset("dummyDs", version = 1, disabled = true) - val dataset2 = DatasetFactory.getDummyDataset("dummyDs", version = 2, disabled = true) - datasetFixture.add(dataset1, dataset2) - - val dataset3 = DatasetFactory.getDummyDataset("dummyDs", version = 7) // version is ignored for create - val response = sendPost[Dataset, String](apiUrl, bodyOpt = Some(dataset3)) - assertCreated(response) - val locationHeaders = response.getHeaders.get("location").asScala - locationHeaders should have size 1 - val relativeLocation = stripBaseUrl(locationHeaders.head) // because locationHeader contains domain, port, etc. - - val response2 = sendGet[Dataset](stripBaseUrl(relativeLocation)) - assertOk(response2) - - val actual = response2.getBody - val expected = toExpected(dataset3.copy(version = 3, parent = Some(DatasetFactory.toParent(dataset2))), actual) - - assert(actual == expected) - } - } } "return 400" when { @@ -134,8 +111,19 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA val responseBody = response.getBody responseBody shouldBe Validation(Map("undefinedProperty1" -> List("There is no property definition for key 'undefinedProperty1'."))) } + "disabled entity with the name already exists" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", disabled = true) + datasetFixture.add(dataset1) + + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("a new version attempt")) + val response = sendPost[Dataset, EntityDisabledException](apiUrl, bodyOpt = Some(dataset2)) + + assertBadRequest(response) + response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") + } } - // todo what to do if "the last dataset version is disabled"? + } s"GET $apiUrl/{name}" should { @@ -148,9 +136,31 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA parent = Some(DatasetFactory.toParent(datasetV1))) datasetFixture.add(datasetV1, datasetV2) - val response = sendGet[NamedLatestVersion](s"$apiUrl/datasetA") + val response = sendGet[NamedVersion](s"$apiUrl/datasetA") assertOk(response) - assert(response.getBody == NamedLatestVersion("datasetA", 2)) + assert(response.getBody == NamedVersion("datasetA", 2, disabled = false)) + } + + "a Dataset with the given name exists - all disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1, disabled = true) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, disabled = true) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[NamedVersion](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == NamedVersion("datasetA", 2, disabled = true)) + } + + "a Dataset with with mixed disabled states" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1) + val datasetV2 = DatasetFactory.getDummyDataset(name = "datasetA", version = 2, disabled = true) + datasetFixture.add(datasetV1, datasetV2) + + val response = sendGet[NamedVersion](s"$apiUrl/datasetA") + assertOk(response) + assert(response.getBody == NamedVersion("datasetA", 2, disabled = true)) // mixed state -> disabled } } @@ -263,9 +273,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA description = Some("second version"), properties = Some(Map("keyA" -> "valA")), version = 2) datasetFixture.add(datasetA1, datasetA2) - Seq("keyA", "keyB", "keyC").foreach {propName => propertyDefinitionFixture.add( - PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) - )} + Seq("keyA", "keyB", "keyC").foreach { propName => + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition(propName, essentiality = Essentiality.Optional) + ) + } // this will cause missing property 'keyD' to issue a warning if not present propertyDefinitionFixture.add( PropertyDefinitionFactory.getDummyPropertyDefinition("keyD", essentiality = Essentiality.Recommended) @@ -335,6 +347,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA response2.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") } } + "entity is disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val dataset1 = DatasetFactory.getDummyDataset("dummyDs", disabled = true) + datasetFixture.add(dataset1) + + val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("ds update")) + val response = sendPut[Dataset, EntityDisabledException](s"$apiUrl/dummyDs/1", bodyOpt = Some(dataset2)) + + assertBadRequest(response) + response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") + } } } @@ -512,25 +535,15 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } - s"GET $apiUrl/{name}/{version}/used-in" should { + s"GET $apiUrl/{name}/used-in" should { "return 404" when { - "when the dataset of latest version does not exist" in { - val response = sendGet[String](s"$apiUrl/notFoundDataset/latest/used-in") - assertNotFound(response) - } - } - - "return 404" when { - "when the dataset of name/version does not exist" in { + "when the dataset of name does not exist" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetA) - val response = sendGet[String](s"$apiUrl/notFoundDataset/1/used-in") + val response = sendGet[String](s"$apiUrl/notFoundDataset/used-in") assertNotFound(response) - - val response2 = sendGet[String](s"$apiUrl/datasetA/7/used-in") - assertNotFound(response2) } } @@ -539,7 +552,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetA = DatasetFactory.getDummyDataset(name = "datasetA") datasetFixture.add(datasetA) - val response = sendGet[UsedIn](s"$apiUrl/datasetA/latest/used-in") + val response = sendGet[UsedIn](s"$apiUrl/datasetA/used-in") assertOk(response) response.getBody shouldBe UsedIn(None, None) @@ -547,11 +560,11 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } "return 200" when { - "for existing name+version for dataset" in { + "for existing name for dataset" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val dataset2 = DatasetFactory.getDummyDataset(name = "dataset", version = 2) datasetFixture.add(dataset2) - val response = sendGet[UsedIn](s"$apiUrl/dataset/2/used-in") + val response = sendGet[UsedIn](s"$apiUrl/dataset/used-in") assertOk(response) response.getBody shouldBe UsedIn(None, None) @@ -827,7 +840,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(datasetV1) val response = sendPost[ConformanceRule, String](s"$apiUrl/notFoundDataset/456/rules", - bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + bodyOpt = Some(LiteralConformanceRule(0, "column1", true, value = "ABC"))) assertNotFound(response) } } @@ -836,12 +849,12 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when the there is a conflicting conf rule #" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( - LiteralConformanceRule(order = 0,"column1", true, "ABC") + LiteralConformanceRule(order = 0, "column1", true, "ABC") )) datasetFixture.add(datasetV1) val response = sendPost[ConformanceRule, String](s"$apiUrl/datasetA/1/rules", - bodyOpt = Some(LiteralConformanceRule(0,"column1", true, value = "ABC"))) + bodyOpt = Some(LiteralConformanceRule(0, "column1", true, value = "ABC"))) assertBadRequest(response) response.getBody should include("Rule with order 0 cannot be added, another rule with this order already exists.") @@ -863,7 +876,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA "when conf rule is added" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List( - LiteralConformanceRule(order = 0,"column1", true, "ABC")) + LiteralConformanceRule(order = 0, "column1", true, "ABC")) ) datasetFixture.add(datasetV1) @@ -916,4 +929,125 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + s"PUT $apiUrl/{name}" can { + "return 200" when { + "a Dataset with the given name exists" should { + "enable the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = true) + val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1, disabled = true) + datasetFixture.add(dsA1, dsA2, dsB) + + val response = sendPut[String, DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions now enabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + + // unrelated dataset unaffected + val responseB = sendGet[Dataset](s"$apiUrl/dsB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe true + } + } + + "a Dataset with the given name exists and there have mixed disabled states (historical)" should { + "enable all versions the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) + datasetFixture.add(dsA1, dsA2) + + val response = sendPut[String, DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions enabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + } + } + } + + "return 404" when { + "no Dataset with the given name exists" should { + "enable nothing" in { + val response = sendPut[String, DisabledPayload](s"$apiUrl/aDataset") + assertNotFound(response) + } + } + } + } + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a Dataset with the given name exists" should { + "disable the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2) + val dsB = DatasetFactory.getDummyDataset(name = "dsB", version = 1) + datasetFixture.add(dsA1, dsA2, dsB) + + val response = sendDelete[DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated dataset unaffected + val responseB = sendGet[Dataset](s"$apiUrl/dsB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a Dataset with the given name exists and there have mixed disabled states (historical)" should { + "disable all versions the dataset with the given name" in { + val dsA1 = DatasetFactory.getDummyDataset(name = "dsA", version = 1, disabled = true) + val dsA2 = DatasetFactory.getDummyDataset(name = "dsA", version = 2, disabled = false) + datasetFixture.add(dsA1, dsA2) + + val response = sendDelete[DisabledPayload](s"$apiUrl/dsA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Dataset](s"$apiUrl/dsA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Dataset](s"$apiUrl/dsA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 404" when { + "no Dataset with the given name exists" should { + "disable nothing" in { + val response = sendDelete[String](s"$apiUrl/aDataset") + assertNotFound(response) + } + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index 7dece37b2..4f975dcc0 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -23,12 +23,16 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.properties.PropertyDefinition -import za.co.absa.enceladus.model.{DefaultValue, MappingTable, Validation} -import za.co.absa.enceladus.model.test.factories.{MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} -import za.co.absa.enceladus.rest_api.integration.controllers.BaseRestApiTestV3 +import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule +import za.co.absa.enceladus.model.dataFrameFilter._ +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} +import za.co.absa.enceladus.model.{DefaultValue, MappingTable, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ -import za.co.absa.enceladus.rest_api.integration.controllers.toExpected +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException + @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -41,10 +45,13 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be @Autowired private val schemaFixture: SchemaFixtureService = null + @Autowired + private val datasetFixture: DatasetFixtureService = null + private val apiUrl = "/mapping-tables" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(mappingTableFixture, schemaFixture) + override def fixtures: List[FixtureService[_]] = List(mappingTableFixture, schemaFixture, datasetFixture) s"POST $apiUrl" should { "return 400" when { @@ -230,4 +237,199 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } } + private def mcr(mtName: String, mtVersion: Int, index: Int = 0) = MappingConformanceRule(index, + controlCheckpoint = true, + mappingTable = mtName, + mappingTableVersion = mtVersion, + attributeMappings = Map("InputValue" -> "STRING_VAL"), + targetAttribute = "CCC", + outputColumn = "ConformedCCC", + isNullSafe = true, + mappingTableFilter = Some( + AndJoinedFilters(Set( + OrJoinedFilters(Set( + EqualsFilter("column1", "soughtAfterValue"), + EqualsFilter("column1", "alternativeSoughtAfterValue") + )), + DiffersFilter("column2", "anotherValue"), + NotFilter(IsNullFilter("col3")) + )) + ), + overrideMappingTableOwnFilter = Some(true) + ) + + s"GET $apiUrl/{name}/used-in" should { + "return 200" when { + "there are used-in records" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable",1))) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable",1)), disabled = true) + val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) + datasetFixture.add(datasetA, datasetB, datasetC) + + val response = sendGet[String](s"$apiUrl/mappingTable/used-in") + assertOk(response) + + // datasetB is disabled -> not reported + // datasetC is reported, because this is a version-less check + // String-typed this time to also check isEmpty/nonEmpty serialization presence + response.getBody shouldBe + """ + |{"datasets":[ + |{"collection":null,"name":"datasetA","version":1},{"collection":null,"name":"datasetC","version":1} + |], + |"mappingTables":null} + |""".stripMargin.replaceAll("[\\r\\n]", "") + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 200" when { + "there are used-in records for particular version" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable",1))) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable",1)), disabled = true) + val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) + datasetFixture.add(datasetA, datasetB, datasetC) + + val response = sendGet[UsedIn](s"$apiUrl/mappingTable/1/used-in") + assertOk(response) + + // datasetB is disabled -> not reported + // datasetC is not reported, because it depends on v2 of the MT + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1))), + mappingTables = None + ) + } + + "there are no used-in records for particular version" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + mappingTableFixture.add(mappingTable1) + + val response = sendGet[UsedIn](s"$apiUrl/mappingTable/1/used-in") + assertOk(response) + + response.getBody shouldBe UsedIn( + datasets = None, + mappingTables = None + ) + } + } + } + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a MappingTable with the given name exists" should { + "disable the mappingTable with the given name" in { + val mtA1 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 1) + val mtA2 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 2) + val mtB = MappingTableFactory.getDummyMappingTable(name = "mtB", version = 1) + mappingTableFixture.add(mtA1, mtA2, mtB) + + val response = sendDelete[DisabledPayload](s"$apiUrl/mtA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[MappingTable](s"$apiUrl/mtA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[MappingTable](s"$apiUrl/mtA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated mappingTable unaffected + val responseB = sendGet[MappingTable](s"$apiUrl/mtB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a MappingTable with the given name exists and there have mixed (historical) disabled states " should { + "disable all versions the mappingTable with the given name" in { + val mtA1 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 1, disabled = true) + val mtA2 = MappingTableFactory.getDummyMappingTable(name = "mtA", version = 2, disabled = false) + mappingTableFixture.add(mtA1, mtA2) + + val response = sendDelete[DisabledPayload](s"$apiUrl/mtA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[MappingTable](s"$apiUrl/mtA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[MappingTable](s"$apiUrl/mtA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the MappingTable is only used in disabled Datasets" should { + "disable the MappingTable" in { + val dataset = DatasetFactory.getDummyDataset(conformance = List(mcr("mappingTable", 1)), disabled = true) + datasetFixture.add(dataset) + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val response = sendDelete[DisabledPayload](s"$apiUrl/mappingTable") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[MappingTable](s"$apiUrl/mappingTable/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[MappingTable](s"$apiUrl/mappingTable/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 400" when { + "the MappingTable is used by an enabled Dataset" should { + "return a list of the entities the MappingTable is used in" in { + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 1) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", conformance = List(mcr("mappingTable", 1))) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, conformance = List(mcr("mappingTable", 2))) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3",conformance = List(mcr("anotherMappingTable", 8))) // moot + val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", conformance = List(mcr("mappingTable", 2)), disabled = true) + datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) + + val response = sendDelete[EntityInUseException](s"$apiUrl/mappingTable") + + assertBadRequest(response) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "mappingTable", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + ) + } + } + } + + "return 404" when { + "no MappingTable with the given name exists" should { + "disable nothing" in { + val response = sendDelete[String](s"$apiUrl/aMappingTable") + assertNotFound(response) + } + } + } + } + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 633666d76..2c325405c 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -23,13 +23,16 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.Validation +import za.co.absa.enceladus.model.{UsedIn, Validation} +import za.co.absa.enceladus.model.menas.MenasReference import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} -import za.co.absa.enceladus.model.test.factories.PropertyDefinitionFactory -import za.co.absa.enceladus.model.versionedModel.NamedLatestVersion +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} +import za.co.absa.enceladus.model.versionedModel.NamedVersion +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ +import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -39,10 +42,13 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + @Autowired + private val datasetFixture: DatasetFixtureService = null + private val apiUrl = "/property-definitions/datasets" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(propertyDefinitionFixture) + override def fixtures: List[FixtureService[_]] = List(propertyDefinitionFixture, datasetFixture) private def minimalPdCreatePayload(name: String, suggestedValue: Option[String]) = { @@ -137,9 +143,9 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w version = 2, parent = Some(PropertyDefinitionFactory.toParent(pdV1))) propertyDefinitionFixture.add(pdV1, pdV2) - val response = sendGet[NamedLatestVersion](s"$apiUrl/pdA") + val response = sendGet[NamedVersion](s"$apiUrl/pdA") assertOk(response) - assert(response.getBody == NamedLatestVersion("pdA", 2)) + assert(response.getBody == NamedVersion("pdA", 2, disabled = false)) } } @@ -370,5 +376,172 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w } } - // todo delete + s"GET $apiUrl/{name}/used-in" should { + "return 200" when { + "there are used-in records" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 1) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 2, description = Some("An update")) + val propDefB = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propB", version = 1) // moot + propertyDefinitionFixture.add(propDefA1, propDefA2, propDefB) + + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("propA" -> "something"))) + val datasetB1 = DatasetFactory.getDummyDataset(name = "datasetB", properties = Some(Map("propA" -> "something")), disabled = true) + val datasetC1 = DatasetFactory.getDummyDataset(name = "datasetC", properties = Some(Map("propA" -> "something else"))) + datasetFixture.add(datasetA1, datasetB1, datasetC1) + + val response = sendGet[UsedIn](s"$apiUrl/propA/used-in") + assertOk(response) + + // propDefB is moot. + // datasetB is not reported, because it is disabled + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), + mappingTables = None + ) + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 200" when { + "there are used-in records for particular version" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 1) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propA", version = 2, description = Some("An update")) + propertyDefinitionFixture.add(propDefA1, propDefA2) + + val datasetA1 = DatasetFactory.getDummyDataset(name = "datasetA", properties = Some(Map("propA" -> "something"))) + val datasetB1 = DatasetFactory.getDummyDataset(name = "datasetB", properties = Some(Map("propA" -> "something")), disabled = true) + val datasetC1 = DatasetFactory.getDummyDataset(name = "datasetC", properties = Some(Map("propA" -> "something else"))) + datasetFixture.add(datasetA1, datasetB1, datasetC1) + + val response = sendGet[UsedIn](s"$apiUrl/propA/1/used-in") + assertOk(response) + + // same outcome as $apiUrl/{name}/used-in above -- because propDefs are not tied by version to datasets + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1), MenasReference(None, "datasetC", 1))), + mappingTables = None + ) + } + } + } + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a PropertyDefinition with the given name exists" should { + "disable the propertyDefinition with the given name" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 1) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 2) + val propDefB = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefB", version = 1) + propertyDefinitionFixture.add(propDefA1, propDefA2, propDefB) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/propDefA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated propDef unaffected + val responseB = sendGet[PropertyDefinition](s"$apiUrl/propDefB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a PropertyDefinition with the given name exists and there have mixed (historical) disabled states " should { + "disable all versions the propertyDefinition with the given name" in { + val propDefA1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 1, disabled = true) + val propDefA2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propDefA", version = 2, disabled = false) + propertyDefinitionFixture.add(propDefA1, propDefA2) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/propDefA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[PropertyDefinition](s"$apiUrl/propDefA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the PropertyDefinition is only used in disabled Datasets" should { + "disable the PropertyDefinition" in { + val propertyDefinition1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 1) + val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinition", version = 2) + propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2) + + val dataset = DatasetFactory.getDummyDataset(disabled = true, properties = Some(Map("propertyDefinition" -> "value xyz"))) + datasetFixture.add(dataset) + + val response = sendDeleteByAdmin[DisabledPayload](s"$apiUrl/propertyDefinition") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinition/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[PropertyDefinition](s"$apiUrl/propertyDefinition/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 400" when { + "the PropertyDefinition is used by an enabled Dataset" should { + "return a list of the entities the PropertyDefinition is used in" in { + val propertyDefinition1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "keyA", version = 1) + val propertyDefinition2 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "keyA", version = 2, propertyType = EnumPropertyType("x", "y", "z")) + val propertyDefinitionAsdf = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "keyASDF", version = 1) // moot support + propertyDefinitionFixture.add(propertyDefinition1, propertyDefinition2, propertyDefinitionAsdf) + + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", properties = Some(Map("keyA" -> "x"))) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, properties = Some(Map("keyA" -> "z"))) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3", properties = Some(Map("keyASDF" -> "ASDF"))) // moot + val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", properties = Some(Map("keyA" -> "x")), disabled = true) + datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) + + val response = sendDeleteByAdmin[EntityInUseException](s"$apiUrl/keyA") + + assertBadRequest(response) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "keyA", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + ) + } + } + } + + "return 404" when { + "no PropertyDefinition with the given name exists" should { + "disable nothing" in { + val response = sendDeleteByAdmin[String](s"$apiUrl/aPropertyDefinition") + assertNotFound(response) + } + } + } + + "return 403" when { + s"admin auth is not used for DELETE" in { + val propertyDefinitionV1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinitionA", version = 1) + propertyDefinitionFixture.add(propertyDefinitionV1) + + val response = sendDelete[Validation](s"$apiUrl/propertyDefinitionA") + response.getStatusCode shouldBe HttpStatus.FORBIDDEN + } + } + } } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index e5f948571..d03194ac5 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -27,15 +27,16 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.{HttpStatus, MediaType} import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner -import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, SchemaFactory} -import za.co.absa.enceladus.model.{Schema, SchemaField, Validation} +import za.co.absa.enceladus.model.menas.MenasReference +import za.co.absa.enceladus.model.test.factories.{AttachmentFactory, DatasetFactory, MappingTableFactory, SchemaFactory} +import za.co.absa.enceladus.model.{Schema, SchemaField, UsedIn, Validation} +import za.co.absa.enceladus.rest_api.exceptions.EntityInUseException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ -import za.co.absa.enceladus.rest_api.models.rest.RestResponse +import za.co.absa.enceladus.rest_api.models.rest.{DisabledPayload, RestResponse} import za.co.absa.enceladus.rest_api.models.rest.errors.{SchemaFormatError, SchemaParsingError} import za.co.absa.enceladus.rest_api.repositories.RefCollection import za.co.absa.enceladus.rest_api.utils.SchemaType -import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import za.co.absa.enceladus.restapi.TestResourcePath import java.io.File @@ -72,9 +73,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn @Autowired private val attachmentFixture: AttachmentFixtureService = null - @Autowired - private val convertor: SparkMenasSchemaConvertor = null - private val apiUrl = "/schemas" private val schemaRefCollection = RefCollection.SCHEMA.name().toLowerCase() @@ -174,9 +172,6 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } - // todo disable dataset - all versions/one version/ check the usage to prevent from disabling - // todo used-in implementation checks - s"GET $apiUrl/{name}/{version}/json" should { "return 404" when { "no schema exists for the specified name" in { @@ -195,7 +190,7 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn assertNotFound(response) } - "the schema has no fields" in { // todo 404 or 400 failed valiadation??? + "the schema has no fields" in { val schema = SchemaFactory.getDummySchema(name = "schemaA", version = 1) schemaFixture.add(schema) @@ -731,4 +726,294 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } + s"GET $apiUrl/{name}/used-in" should { + "return 200" when { + "there are used-in records" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", schemaName = "schema", schemaVersion = 1) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", schemaName = "schema", schemaVersion = 1, disabled = true) + datasetFixture.add(datasetA, datasetB) + + val mappingTableA = MappingTableFactory.getDummyMappingTable(name = "mappingA", schemaName = "schema", schemaVersion = 1) + val mappingTableB = MappingTableFactory.getDummyMappingTable(name = "mappingB", schemaName = "schema", schemaVersion = 1, disabled = true) + val mappingTableC = MappingTableFactory.getDummyMappingTable(name = "mappingC", schemaName = "schema", schemaVersion = 2) + mappingTableFixture.add(mappingTableA, mappingTableB, mappingTableC) + + + val response = sendGet[UsedIn](s"$apiUrl/schema/used-in") + assertOk(response) + + // datasetB and mappingB are disabled -> not reported + // mappingC is reported, even though it schema is tied to schema-v2, because disabling is done on the whole entity in API v3 + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1))), + mappingTables = Some(Seq(MenasReference(None, "mappingA", 1), MenasReference(None, "mappingC", 1))) + ) + } + } + } + + s"GET $apiUrl/{name}/{version}/used-in" should { + "return 200" when { + "there are used-in records for particular version" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", schemaName = "schema", schemaVersion = 1) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", schemaName = "schema", schemaVersion = 1, disabled = true) + datasetFixture.add(datasetA, datasetB) + + val mappingTableA = MappingTableFactory.getDummyMappingTable(name = "mappingA", schemaName = "schema", schemaVersion = 1) + val mappingTableB = MappingTableFactory.getDummyMappingTable(name = "mappingB", schemaName = "schema", schemaVersion = 1, disabled = true) + val mappingTableC = MappingTableFactory.getDummyMappingTable(name = "mappingC", schemaName = "schema", schemaVersion = 2) + mappingTableFixture.add(mappingTableA, mappingTableB, mappingTableC) + + + val response = sendGet[UsedIn](s"$apiUrl/schema/1/used-in") + assertOk(response) + + // datasetB and mappingB are disabled -> not reported + // mappingC is tied to schema v2 -> not reported + response.getBody shouldBe UsedIn( + datasets = Some(Seq(MenasReference(None, "datasetA", 1))), + mappingTables = Some(Seq(MenasReference(None, "mappingA", 1))) + ) + } + } + } + + s"PUT $apiUrl/{name}" can { + "return 200" when { + "a Schema with the given name exists" should { + "enable the Schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = true) + val schB = SchemaFactory.getDummySchema(name = "schB", version = 1, disabled = true) + schemaFixture.add(schA1, schA2, schB) + + val response = sendPut[String, DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions now enabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + + // unrelated schema unaffected + val responseB = sendGet[Schema](s"$apiUrl/schB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe true + } + } + + "a Schema with the given name exists and there have mixed disabled states (historical)" should { + "enable all versions the schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) + schemaFixture.add(schA1, schA2) + + val response = sendPut[String, DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = false) + + // all versions enabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe false + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe false + } + } + } + + "return 404" when { + "no Schema with the given name exists" should { + "enable nothing" in { + val response = sendPut[String, DisabledPayload](s"$apiUrl/aSchema") + assertNotFound(response) + } + } + } + + } + + s"DELETE $apiUrl/{name}" can { + "return 200" when { + "a Schema with the given name exists" should { + "disable the schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2) + val schB = SchemaFactory.getDummySchema(name = "schB", version = 1) + schemaFixture.add(schA1, schA2, schB) + + val response = sendDelete[DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + + // unrelated schema unaffected + val responseB = sendGet[Schema](s"$apiUrl/schB/1") + assertOk(responseB) + responseB.getBody.disabled shouldBe false + } + } + + "a Schema with the given name exists and there have mixed (historical) disabled states " should { + "disable all versions the schema with the given name" in { + val schA1 = SchemaFactory.getDummySchema(name = "schA", version = 1, disabled = true) + val schA2 = SchemaFactory.getDummySchema(name = "schA", version = 2, disabled = false) + schemaFixture.add(schA1, schA2) + + val response = sendDelete[DisabledPayload](s"$apiUrl/schA") + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schA/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schA/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the Schema is only used in disabled Datasets" should { + "disable the Schema" in { + val dataset = DatasetFactory.getDummyDataset(schemaName = "schema", schemaVersion = 1, disabled = true) + datasetFixture.add(dataset) + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val response = sendDelete[DisabledPayload](s"$apiUrl/schema") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schema/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schema/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + "the Schema is only used in disabled MappingTables" should { + "disable the Schema" in { + val mappingTable = MappingTableFactory.getDummyMappingTable(schemaName = "schema", schemaVersion = 1, disabled = true) + mappingTableFixture.add(mappingTable) + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val response = sendDelete[DisabledPayload](s"$apiUrl/schema") + + assertOk(response) + response.getBody shouldBe DisabledPayload(disabled = true) + + // all versions disabled + val responseA1 = sendGet[Schema](s"$apiUrl/schema/1") + assertOk(responseA1) + responseA1.getBody.disabled shouldBe true + + val responseA2 = sendGet[Schema](s"$apiUrl/schema/2") + assertOk(responseA2) + responseA2.getBody.disabled shouldBe true + } + } + } + + "return 400" when { + "the Schema is used by an enabled Dataset" should { + "return a list of the entities the Schema is used in" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", schemaName = "schema", schemaVersion = 1) + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, schemaName = "schema", schemaVersion = 2) + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3", schemaName = "anotherSchema", schemaVersion = 8) // moot + val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", schemaName = "schema", schemaVersion = 2, disabled = true) + datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) + + val response = sendDelete[EntityInUseException](s"$apiUrl/schema") + + assertBadRequest(response) + response.getBody shouldBe EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset1", 1), MenasReference(None, "dataset2", 7))), None) + ) + } + } + "the Schema is used by a enabled MappingTable" should { + "return a list of the entities the Schema is used in" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mapping1", schemaName = "schema", schemaVersion = 1, disabled = false) + val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mapping2", schemaName = "schema", schemaVersion = 2, disabled = false) + mappingTableFixture.add(mappingTable1, mappingTable2) + + val response = sendDelete[EntityInUseException](s"$apiUrl/schema") + assertBadRequest(response) + + response.getBody shouldBe EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(None, Some(Seq(MenasReference(None, "mapping1", 1), MenasReference(None, "mapping2", 1)))) + ) + } + } + "the Schema is used by combination of MT and DS" should { + "return a list of the entities the Schema is used in" in { + val schema1 = SchemaFactory.getDummySchema(name = "schema", version = 1) + val schema2 = SchemaFactory.getDummySchema(name = "schema", version = 2) + schemaFixture.add(schema1, schema2) + + val mappingTable1 = MappingTableFactory.getDummyMappingTable(name = "mapping1", schemaName = "schema", schemaVersion = 1, disabled = false) + mappingTableFixture.add(mappingTable1) + + val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", schemaName = "schema", schemaVersion = 2) + datasetFixture.add(dataset2) + + val response = sendDelete[EntityInUseException](s"$apiUrl/schema") + assertBadRequest(response) + + response.getBody shouldBe EntityInUseException("""Cannot disable entity "schema", because it is used in the following entities""", + UsedIn(Some(Seq(MenasReference(None, "dataset2", 1))), Some(Seq(MenasReference(None, "mapping1", 1)))) + ) + } + } + } + + "return 404" when { + "no Schema with the given name exists" should { + "disable nothing" in { + val response = sendDelete[String](s"$apiUrl/aSchema") + assertNotFound(response) + } + } + } + } + } diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala index 087381db9..c2c535c01 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -579,11 +579,11 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers } "returns even the disabled dataset" when { "only disabled dataset exists" in { - val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = false) val dataset2 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, version = 2) datasetFixture.add(dataset1, dataset2) val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) - actual shouldBe Some(VersionedSummary("datasetA", 2)) // warning: currently, this method reports the disabled, too + actual shouldBe Some(VersionedSummary("datasetA", 2, Set(true, false))) } } @@ -595,7 +595,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest with Matchers datasetFixture.add(dataset1, dataset2, dataset3) val actual = await(datasetMongoRepository.getLatestVersionSummary("datasetA")) - actual shouldBe Some(VersionedSummary("datasetA", 3)) + actual shouldBe Some(VersionedSummary("datasetA", 3, Set(false))) } } }