Feature/1693 api v3 delete recreate (#2055)

* #1693 API v3: VersionedModel v3 disable/enable + usedIn
 - integrationTest cases added
 - 2 levels of .../used-in exist ( /{name}/used-in & /{name}/{version}/used-in). The former is used in disable checking.
 - UsedIn - empty/nonEmpty normalization  V2 alternated between None and Some(Seq.empty) in various places, V3 consistently returns None for no references groups.
 - `NamedLatestVersion` generalized into a multipurpose `NamedVersion`. Small updates, thanks @benedeki
 - API v3 summary (NamedVersion) now contains `disabled` information - mainly on GET ...{/name}
- disable fail due to nonEmpty used in now carries a wrapper with an error message (`UsedIn` wrapped in `EntityInUseException`)
- Future {throw x}` replaced with `Future.failed(x)` in rest_api, cleanup

data-model/src/main/scala/za/co/absa/enceladus/model/UsedIn.scala

@@ -15,15 +15,44 @@
 
 package za.co.absa.enceladus.model
 
+import com.fasterxml.jackson.annotation.JsonIgnore
 import za.co.absa.enceladus.model.menas.MenasReference
 
-case class UsedIn(
-  datasets: Option[Seq[MenasReference]] = None,
-  mappingTables: Option[Seq[MenasReference]] = None
-) {
+case class UsedIn(datasets: Option[Seq[MenasReference]] = None,
+                  mappingTables: Option[Seq[MenasReference]] = None) {
 
-  def nonEmpty: Boolean = {
-    datasets.exists(_.nonEmpty) || mappingTables.exists(_.nonEmpty)
+  /**
+   * Should any of the original UsedIn equal to Some(Seq.empty), it will be None after normalization
+   */
+  @JsonIgnore
+  lazy val normalized: UsedIn = {
+    def normalizeOne(field: Option[Seq[MenasReference]]) = field match {
+      case None => None
+      case Some(x) if x.isEmpty => None
+      case otherNonEmpty => otherNonEmpty
+    }
+
+    val normalizedDs = normalizeOne(datasets)
+    val normalizedMt = normalizeOne(mappingTables)
+
+    (normalizedDs, normalizedMt) match {
+      case (`datasets`, `mappingTables`) => this // no normalization needed
+      case _ => UsedIn(normalizedDs, normalizedMt)
+    }
+  }
+
+  @JsonIgnore
+  val isEmpty: Boolean = {
+    normalized.datasets == None && normalized.mappingTables == None
   }
 
+  @JsonIgnore
+  val nonEmpty: Boolean = !isEmpty
+}
+
+object UsedIn {
+  /**
+   * Normalized
+   */
+  val empty = UsedIn(None, None)
 }

data-model/src/main/scala/za/co/absa/enceladus/model/test/factories/DatasetFactory.scala

@@ -15,13 +15,13 @@
 
 package za.co.absa.enceladus.model.test.factories
 
-import java.time.ZonedDateTime
-
 import za.co.absa.enceladus.model.Dataset
 import za.co.absa.enceladus.model.conformanceRule._
 import za.co.absa.enceladus.model.menas.MenasReference
 import za.co.absa.enceladus.model.versionedModel.VersionedSummary
 
+import java.time.ZonedDateTime
+
 object DatasetFactory extends EntityFactory[Dataset] {
 
   override val collectionBaseName: String = "dataset"
@@ -128,7 +128,7 @@ object DatasetFactory extends EntityFactory[Dataset] {
   }
 
   def toSummary(dataset: Dataset): VersionedSummary = {
-    VersionedSummary(dataset.name, dataset.version)
+    VersionedSummary(dataset.name, dataset.version, Set(dataset.disabled))
   }
 
 }

data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedVersion.scala

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.model.versionedModel
+
+/**
+ * V3 Wrapper for [[za.co.absa.enceladus.model.versionedModel.VersionedSummary]]
+ */
+case class NamedVersion(name: String, version: Int, disabled: Boolean)

data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala

@@ -15,8 +15,18 @@
 
 package za.co.absa.enceladus.model.versionedModel
 
-case class VersionedSummary(_id: String, latestVersion: Int) {
-  def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion)
+/**
+ * V2 Representation of `VersionedSummary` - V2 does not carry disabled information
+ */
+case class VersionedSummaryV2(_id: String, latestVersion: Int)
+
+case class VersionedSummary(_id: String, latestVersion: Int, disabledSet: Set[Boolean]) {
+  def toV2: VersionedSummaryV2 = VersionedSummaryV2(_id, latestVersion)
+
+  def toNamedVersion: NamedVersion = {
+    val disabled = disabledSet.contains(true) // legacy mixed state reported as disabled for V3 summary
+    NamedVersion(_id, latestVersion, disabled)
+  }
 }
 
 

data-model/src/test/scala/za/co/absa/enceladus/model/UsedInTest.scala

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.model
+
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+import za.co.absa.enceladus.model.menas.MenasReference
+
+class UsedInTest extends AnyFlatSpec with Matchers {
+
+  private val exampleRef = MenasReference(Some("collection1"), "entity1", 1)
+
+  "UsedIn" should "correctly evaluate .nonEmpty" in {
+    UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).nonEmpty shouldBe true
+    UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).nonEmpty shouldBe true
+    UsedIn(Some(Seq(exampleRef)), None).nonEmpty shouldBe true
+
+    UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).nonEmpty shouldBe true
+    UsedIn(None, Some(Seq(exampleRef))).nonEmpty shouldBe true
+
+    UsedIn(Some(Seq.empty), Some(Seq.empty)).nonEmpty shouldBe false
+    UsedIn(None, Some(Seq.empty)).nonEmpty shouldBe false
+    UsedIn(Some(Seq.empty), None).nonEmpty shouldBe false
+    UsedIn(None, None).nonEmpty shouldBe false
+  }
+
+  it should "correctly evaluate .empty" in {
+    UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).isEmpty shouldBe false
+    UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).isEmpty shouldBe false
+    UsedIn(Some(Seq(exampleRef)), None).isEmpty shouldBe false
+
+    UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).isEmpty shouldBe false
+    UsedIn(None, Some(Seq(exampleRef))).isEmpty shouldBe false
+
+    UsedIn(Some(Seq.empty), Some(Seq.empty)).isEmpty shouldBe true
+    UsedIn(None, Some(Seq.empty)).isEmpty shouldBe true
+    UsedIn(Some(Seq.empty), None).isEmpty shouldBe true
+    UsedIn(None, None).isEmpty shouldBe true
+  }
+
+  it should "normalize" in {
+    UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef))).normalized shouldBe UsedIn(Some(Seq(exampleRef)), Some(Seq(exampleRef)))
+
+    UsedIn(Some(Seq(exampleRef)), Some(Seq.empty)).normalized shouldBe UsedIn(Some(Seq(exampleRef)), None)
+    UsedIn(Some(Seq(exampleRef)), None).normalized shouldBe UsedIn(Some(Seq(exampleRef)), None)
+
+    UsedIn(Some(Seq.empty), Some(Seq(exampleRef))).normalized shouldBe UsedIn(None, Some(Seq(exampleRef)))
+    UsedIn(None, Some(Seq(exampleRef))).normalized shouldBe UsedIn(None, Some(Seq(exampleRef)))
+
+    UsedIn(Some(Seq.empty), Some(Seq.empty)).normalized shouldBe UsedIn(None, None)
+    UsedIn(None, None).normalized shouldBe UsedIn(None, None)
+
+
+
+
+  }
+
+
+
+}

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala

@@ -30,7 +30,7 @@ import za.co.absa.enceladus.rest_api.services.DatasetService
 import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel
 import za.co.absa.enceladus.model.conformanceRule.ConformanceRule
 import za.co.absa.enceladus.model.properties.PropertyDefinition
-import za.co.absa.enceladus.model.versionedModel.VersionedSummary
+import za.co.absa.enceladus.model.versionedModel.VersionedSummaryV2
 import za.co.absa.enceladus.model.{Dataset, Validation}
 import za.co.absa.enceladus.utils.validation.ValidationLevel.Constants.DefaultValidationLevelName
 
@@ -49,9 +49,9 @@ class DatasetController @Autowired()(datasetService: DatasetService)
   @GetMapping(Array("/latest"))
   @ResponseStatus(HttpStatus.OK)
   def getLatestVersions(@RequestParam(value = "missing_property", required = false)
-                        missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = {
+                        missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummaryV2]] = {
     datasetService.getLatestVersions(missingProperty.toScalaOption)
-      .map(datasets => datasets.map(dataset => VersionedSummary(dataset.name, dataset.version)))
+      .map(datasets => datasets.map(dataset => VersionedSummaryV2(dataset.name, dataset.version)))
   }
 
   @PostMapping(Array("/{datasetName}/rule/create"))
@@ -67,7 +67,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
           case Some((ds, validation)) => ds // v2 disregarding validation
           case _        => throw notFound()
         }
-        case _ => throw notFound()
+        case _ => Future.failed(notFound())
       }
     } yield res
   }
@@ -104,7 +104,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
         } else {
           datasetService.filterProperties(dsProperties, DatasetController.paramsToPropertyDefinitionFilter(scalaFilterMap))
         }
-      case None => throw notFound()
+      case None => Future.failed(notFound())
     }
   }
 
@@ -126,7 +126,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
   def getPropertiesValidation(@PathVariable datasetName: String, @PathVariable datasetVersion: Int): CompletableFuture[Validation] = {
     datasetService.getVersion(datasetName, datasetVersion).flatMap {
       case Some(entity) => datasetService.validateProperties(entity.propertiesAsMap, forRun = false)
-      case None => throw notFound()
+      case None => Future.failed(notFound())
     }
   }
 
@@ -135,7 +135,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
   def getPropertiesValidationForRun(@PathVariable datasetName: String, @PathVariable datasetVersion: Int): CompletableFuture[Validation] = {
     datasetService.getVersion(datasetName, datasetVersion).flatMap {
       case Some(entity) => datasetService.validateProperties(entity.propertiesAsMap, forRun = true)
-      case None => throw notFound()
+      case None => Future.failed(notFound())
     }
   }
 

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/HDFSController.scala

@@ -16,14 +16,15 @@
 package za.co.absa.enceladus.rest_api.controllers
 
 import java.util.concurrent.CompletableFuture
-
 import org.apache.hadoop.fs.Path
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.http.HttpStatus
 import org.springframework.web.bind.annotation._
 import za.co.absa.enceladus.model.menas.HDFSFolder
 import za.co.absa.enceladus.rest_api.services.HDFSService
 
+import scala.concurrent.Future
+
 @RestController
 @RequestMapping(Array("/api/hdfs"))
 class HDFSController @Autowired() (hdfsService: HDFSService) extends BaseController {
@@ -41,7 +42,7 @@ class HDFSController @Autowired() (hdfsService: HDFSService) extends BaseControl
       if (exists) {
         hdfsService.getFolder(path)
       } else {
-        throw notFound()
+        Future.failed(notFound())
       }
     }
   }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/PropertyDefinitionController.scala

@@ -26,7 +26,6 @@ import org.springframework.security.access.prepost.PreAuthorize
 import org.springframework.security.core.annotation.AuthenticationPrincipal
 import org.springframework.security.core.userdetails.UserDetails
 import org.springframework.web.bind.annotation._
-import za.co.absa.enceladus.rest_api.exceptions.EndpointDisabled
 import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService
 import za.co.absa.enceladus.model.ExportableObject
 import za.co.absa.enceladus.model.properties.PropertyDefinition
@@ -39,15 +38,15 @@ import scala.concurrent.ExecutionContext.Implicits.global
  */
 @RestController
 @RequestMapping(path = Array("/api/properties/datasets"), produces = Array("application/json"))
-class PropertyDefinitionController @Autowired()(propertyDefService: PropertyDefinitionService)
-  extends VersionedModelController(propertyDefService) {
+class PropertyDefinitionController @Autowired()(propertyDefinitionService: PropertyDefinitionService)
+  extends VersionedModelController(propertyDefinitionService) {
 
   import za.co.absa.enceladus.rest_api.utils.implicits._
 
   @GetMapping(Array(""))
   def getAllDatasetProperties(): CompletableFuture[Seq[PropertyDefinition]] = {
     logger.info("retrieving all dataset properties in full")
-    propertyDefService.getLatestVersions()
+    propertyDefinitionService.getLatestVersions()
   }
 
   @PostMapping(Array(""))
@@ -63,8 +62,6 @@ class PropertyDefinitionController @Autowired()(propertyDefService: PropertyDefi
       val location: URI = new URI(s"/api/properties/datasets/${entity.name}/${entity.version}")
       ResponseEntity.created(location).body(entity)
     }
-
-    // TODO: Location header would make sense for the underlying VersionedModelController.create, too. Issue #1611
   }
 
   @GetMapping(Array("/{propertyName}"))

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala

@@ -62,11 +62,16 @@ class RestExceptionHandler {
     ResponseEntity.notFound().build[Any]()
   }
 
-  @ExceptionHandler(value = Array(classOf[EndpointDisabled]))
-  def handleEndpointDisabled(exception: EndpointDisabled): ResponseEntity[Any] = {
+  @ExceptionHandler(value = Array(classOf[EndpointDisabledException]))
+  def handleEndpointDisabled(exception: EndpointDisabledException): ResponseEntity[Any] = {
     ResponseEntity.status(HttpStatus.I_AM_A_TEAPOT).build[Any]() // Could change for LOCKED but I like this more
   }
 
+  @ExceptionHandler(value = Array(classOf[EntityDisabledException]))
+  def handleEntityDisabled(exception: EntityDisabledException): ResponseEntity[EntityDisabledException] = {
+    ResponseEntity.status(HttpStatus.BAD_REQUEST).body(exception)
+  }
+
   @ExceptionHandler(value = Array(classOf[SchemaParsingException]))
   def handleBadRequestException(exception: SchemaParsingException): ResponseEntity[Any] = {
     val response = RestResponse(exception.message, Option(SchemaParsingError.fromException(exception)))
@@ -111,8 +116,8 @@ class RestExceptionHandler {
   }
 
   @ExceptionHandler(value = Array(classOf[EntityInUseException]))
-  def handleValidationException(exception: EntityInUseException): ResponseEntity[UsedIn] = {
-    ResponseEntity.badRequest().body(exception.usedIn)
+  def handleValidationException(exception: EntityInUseException): ResponseEntity[EntityInUseException] = {
+    ResponseEntity.badRequest().body(exception)
   }
 
   @ExceptionHandler(value = Array(classOf[MethodArgumentTypeMismatchException]))

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala

@@ -23,7 +23,7 @@ import org.springframework.security.core.annotation.AuthenticationPrincipal
 import org.springframework.security.core.userdetails.UserDetails
 import org.springframework.web.bind.annotation._
 import za.co.absa.enceladus.model.{ExportableObject, UsedIn}
-import za.co.absa.enceladus.model.versionedModel._
+import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummaryV2}
 import za.co.absa.enceladus.rest_api.exceptions.NotFoundException
 import za.co.absa.enceladus.rest_api.services.VersionedModelService
 import za.co.absa.enceladus.model.menas.audit._
@@ -38,8 +38,9 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
 
   @GetMapping(Array("/list", "/list/{searchQuery}"))
   @ResponseStatus(HttpStatus.OK)
-  def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = {
+  def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummaryV2]] = {
     versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption)
+      .map(_.map(_.toV2))
   }
 
   @GetMapping(Array("/searchSuggestions"))

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala

@@ -28,6 +28,7 @@ import za.co.absa.enceladus.rest_api.utils.implicits._
 import java.util.concurrent.CompletableFuture
 import javax.servlet.http.HttpServletRequest
 import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
 
 @RestController
 @RequestMapping(path = Array("/api-v3/datasets"))
@@ -57,14 +58,12 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3)
         case Some((entity, validation)) =>
           // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties
           createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties")
-            .body(validation) // todo include in tests
+            .body(validation)
         case None => throw notFound()
       }
     }
   }
 
-  // todo putIntoInfoFile switch needed?
-
   @GetMapping(Array("/{name}/{version}/rules"))
   @ResponseStatus(HttpStatus.OK)
   def getConformanceRules(@PathVariable name: String,
@@ -90,7 +89,7 @@ class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3)
             suffix = s"/rules/$addedRuleOrder").body(validation)
         case _ => throw notFound()
       }
-      case None => throw notFound()
+      case None => Future.failed(notFound())
     }
   }