Merge pull request #491 from dataswift/dev

Dev to Main

.github/workflows/ci.yml

@@ -40,20 +40,14 @@ jobs:
       - name: Update Submodule for schema
         run: git submodule update
 
-      - name: Compile
-        run: sbt +compile
-
-      - name: Test:Compile
-        run: sbt test:compile
-
       - name: Test
-        run: sbt +test -Dconfig.file=hat/conf/application.test.conf
+        run: sbt clean test -Dconfig.file=hat/conf/application.test.conf
 
       # - name: Coverage - off until we re-add Coverage
       #   run: sbt "project hat" coverage test -Dconfig.file=hat/conf/application.test.conf
 
       - name: Build
-        run: sbt docker:stage && docker build hat/target/docker/stage
+        run: sbt Docker/stage && docker build hat/target/docker/stage
 
       - name: AWS - Configure
         uses: aws-actions/configure-aws-credentials@v1

Dockerfile

@@ -0,0 +1 @@
+FROM ghcr.io/dataswift/base:v0.1.0

build.sbt

@@ -1,5 +1,4 @@
 import Dependencies.Library
-import com.typesafe.sbt.packager.docker._
 import sbt.Keys._
 
 val codeguruURI =
@@ -8,9 +7,12 @@ val codeguruURI =
 // the application
 lazy val hat = project
   .in(file("hat"))
+  .settings(dockerSettings)
   .enablePlugins(PlayScala)
   .enablePlugins(SbtWeb, SbtSassify, SbtGzip, SbtDigest)
   .enablePlugins(BasicSettings)
+  .enablePlugins(SlickCodeGeneratorPlugin)
+  .disablePlugins(PlayLogback)
   .settings(
     libraryDependencies ++= Seq(
           Library.Play.ws,
@@ -28,6 +30,7 @@ lazy val hat = project
           Library.Play.Jwt.atlassianJwtCore,
           Library.Play.Jwt.bouncyCastlePkix,
           Library.Backend.logPlay,
+          Library.Backend.redisCache,
           Library.HATDeX.dexClient,
           Library.HATDeX.codegen,
           Library.Utils.awsJavaS3Sdk,
@@ -36,7 +39,6 @@ lazy val hat = project
           Library.Utils.prettyTime,
           Library.Utils.nbvcxz,
           Library.Utils.alpakkaAwsLambda,
-          Library.Utils.playMemcached % Runtime,
           Library.scalaGuice,
           Library.circeConfig,
           Library.ContractLibrary.adjudicator,
@@ -55,6 +57,7 @@ lazy val hat = project
           case BuildEnv.Stage | BuildEnv.Production =>
             libraryDependencies.value
         }),
+    excludeDependencies := Seq(ExclusionRule("org.slf4j", "slf4j-nop")),
     Test / parallelExecution := false,
     Assets / pipelineStages := Seq(digest),
     Assets / sourceDirectory := baseDirectory.value / "app" / "org" / "hatdex" / "hat" / "phata" / "assets",
@@ -70,47 +73,6 @@ lazy val hat = project
     Compile / packageDoc / publishArtifact := false,
     Compile / packageSrc / publishArtifact := false
   )
-  .enablePlugins(JavaAppPackaging)
-  .enablePlugins(AshScriptPlugin)
-  .settings(
-    // Use the alternative "Ash" script for running compiled project form inside Alpine-derived container
-    // as Bash is incompatible with Alpine
-    Universal / javaOptions ++= Seq(),
-    Docker / packageName := "hat",
-    // add a flag to not run in prod
-    // modify the binary to include "-javaagent:codeguru-profiler-java-agent-standalone-1.1.0.jar="profilingGroupName:HatInDev,heapSummaryEnabled:true"
-    dockerCommands := (buildEnv.value match {
-          case BuildEnv.Developement | BuildEnv.Test =>
-            Seq(
-              Cmd("FROM", "adoptopenjdk/openjdk11:jre-11.0.10_9-alpine"),
-              Cmd("WORKDIR", "/opt/docker/bin"),
-              Cmd("CMD", s"./${packageName.value}"),
-              Cmd("EXPOSE", "9000"),
-              Cmd("RUN", s"apk add --no-cache wget; wget --no-check-certificate ${codeguruURI}"),
-              Cmd("USER", "daemon"),
-              Cmd("COPY", "--chown=daemon:daemon", "1/opt", "opt", "/opt/")
-            )
-          case BuildEnv.Stage | BuildEnv.Production =>
-            Seq(
-              Cmd("FROM", "adoptopenjdk/openjdk11:jre-11.0.10_9-alpine"),
-              Cmd("WORKDIR", "/opt/docker/bin"),
-              Cmd("CMD", s"./${packageName.value}"),
-              Cmd("EXPOSE", "9000"),
-              Cmd("USER", "daemon"),
-              Cmd("COPY", "--chown=daemon:daemon", "1/opt", "opt", "/opt/")
-            )
-        }),
-    Universal / javaOptions ++= (buildEnv.value match {
-          case BuildEnv.Developement | BuildEnv.Test =>
-            Seq(
-              "-javaagent:/opt/docker/bin/codeguru-profiler-java-agent-standalone-1.1.0.jar=\"profilingGroupName:HatInDev,heapSummaryEnabled:true\""
-            )
-          case BuildEnv.Stage | BuildEnv.Production =>
-            Seq(
-            )
-        })
-  )
-  .enablePlugins(SlickCodeGeneratorPlugin)
   .settings(
     gentables / codegenPackageName := "org.hatdex.hat.dal",
     gentables / codegenBaseDir := (baseDirectory.value / "app").getCanonicalPath,
@@ -141,6 +103,7 @@ lazy val hat = project
 inThisBuild(
   List(
     scalaVersion := "2.13.5",
+    scalafixScalaBinaryVersion := "2.13",
     semanticdbEnabled := true,
     semanticdbVersion := scalafixSemanticdb.revision,
     scalafixDependencies += "com.github.liancheng" %% "organize-imports" % "0.5.0"

charts/hat/templates/deployment.yaml

@@ -13,9 +13,9 @@ spec:
       {{- include "hat.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+      {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:

hat/app/org/hatdex/hat/api/controllers/Authentication.scala

@@ -436,7 +436,11 @@ class Authentication @Inject() (
                     val emailVerificationOptions =
                       EmailVerificationOptions(email, language, app.application.id, maybeSetupUrl.getOrElse(""))
                     val verificationLink = emailVerificationLink(request.host, token.id, emailVerificationOptions)
-                    mailer.verifyEmail(email, verificationLink)
+                    mailer.verifyEmail(email,
+                                       app.application.info.name,
+                                       app.application.info.graphics.logo.normal,
+                                       verificationLink
+                    )
 
                     response
                 }

hat/app/org/hatdex/hat/api/controllers/RichData.scala

@@ -38,7 +38,6 @@ import org.hatdex.hat.utils.{ HatBodyParsers, LoggingProvider }
 import org.hatdex.libs.dal.HATPostgresProfile
 import play.api.libs.json.Reads._
 import play.api.libs.json.{ JsArray, JsValue, Json }
-import play.api.libs.ws.WSClient
 import play.api.mvc._
 
 import java.util.UUID

hat/app/org/hatdex/hat/api/controllers/legacyContractRequests.scala

@@ -4,9 +4,9 @@ import eu.timepit.refined.collection.NonEmpty
 import eu.timepit.refined.refineV
 import io.dataswift.adjudicator.Types.{ ContractId, HatName, ShortLivedToken }
 import io.dataswift.models.hat.json.HatJsonFormats
+import io.dataswift.models.hat.json.RichDataJsonFormats._
 import io.dataswift.models.hat.{ ApiHatFile, EndpointData }
 import play.api.libs.json.{ JsValue, Json, Reads }
-import io.dataswift.models.hat.json.RichDataJsonFormats._
 
 import java.util.UUID
 import scala.util.Try

hat/app/org/hatdex/hat/api/service/applications/ApplicationStatusCheckService.scala

@@ -0,0 +1,32 @@
+package org.hatdex.hat.api.service.applications
+
+import io.dataswift.models.hat.applications.ApplicationStatus
+import org.hatdex.hat.api.service.RemoteExecutionContext
+import play.api.libs.ws.WSClient
+
+import javax.inject.Inject
+import scala.concurrent.Future
+import scala.concurrent.duration.DurationInt
+
+class ApplicationStatusCheckService @Inject() (
+    wsClient: WSClient
+  )(implicit val rec: RemoteExecutionContext) {
+
+  def status(
+      statusCheck: ApplicationStatus.Status,
+      token: String): Future[Boolean] =
+    statusCheck match {
+      case _: ApplicationStatus.Internal => Future.successful(true)
+      case s: ApplicationStatus.External => status(s, token)
+    }
+
+  protected def status(
+      statusCheck: ApplicationStatus.External,
+      token: String): Future[Boolean] =
+    wsClient
+      .url(statusCheck.statusUrl)
+      .withHttpHeaders("x-auth-token" -> token)
+      .withRequestTimeout(5000.millis)
+      .get()
+      .map(_.status == statusCheck.expectedStatus)
+}

hat/app/org/hatdex/hat/api/service/applications/ApplicationsService.scala

@@ -26,7 +26,6 @@ package org.hatdex.hat.api.service.applications
 import akka.Done
 import akka.actor.ActorSystem
 import com.mohiva.play.silhouette.api.Silhouette
-import dev.profunktor.auth.jwt.JwtSecretKey
 import io.dataswift.adjudicator.Types.ContractId
 import io.dataswift.models.hat.applications._
 import io.dataswift.models.hat.{ AccessToken, EndpointQuery }
@@ -35,10 +34,10 @@ import org.hatdex.hat.api.service.applications.ApplicationExceptions.{
   HatApplicationSetupException
 }
 import org.hatdex.hat.api.service.richData.{ DataDebitService, RichDataDuplicateDebitException, RichDataService }
-import org.hatdex.hat.api.service.{ DalExecutionContext, RemoteExecutionContext, StatsReporter }
+import org.hatdex.hat.api.service.{ DalExecutionContext, StatsReporter }
 import org.hatdex.hat.authentication.HatApiAuthEnvironment
 import org.hatdex.hat.authentication.models.HatUser
-import org.hatdex.hat.client.AdjudicatorWsClient
+import org.hatdex.hat.client.AdjudicatorClient
 import org.hatdex.hat.dal.Tables
 import org.hatdex.hat.dal.Tables.ApplicationStatusRow
 import org.hatdex.hat.resourceManagement.HatServer
@@ -48,7 +47,6 @@ import org.hatdex.libs.dal.HATPostgresProfile.api._
 import org.joda.time.DateTime
 import play.api.cache.AsyncCacheApi
 import play.api.libs.json.{ JsObject, JsString }
-import play.api.libs.ws.WSClient
 import play.api.mvc.RequestHeader
 import play.api.{ Configuration, Logging }
 
@@ -58,29 +56,6 @@ import scala.concurrent.Future
 import scala.concurrent.duration._
 import scala.util.Success
 
-class ApplicationStatusCheckService @Inject() (
-    wsClient: WSClient
-  )(implicit val rec: RemoteExecutionContext) {
-
-  def status(
-      statusCheck: ApplicationStatus.Status,
-      token: String): Future[Boolean] =
-    statusCheck match {
-      case _: ApplicationStatus.Internal => Future.successful(true)
-      case s: ApplicationStatus.External => status(s, token)
-    }
-
-  protected def status(
-      statusCheck: ApplicationStatus.External,
-      token: String): Future[Boolean] =
-    wsClient
-      .url(statusCheck.statusUrl)
-      .withHttpHeaders("x-auth-token" -> token)
-      .withRequestTimeout(5000.millis)
-      .get()
-      .map(_.status == statusCheck.expectedStatus)
-}
-
 class ApplicationsService @Inject() (
     cache: AsyncCacheApi,
     richDataService: RichDataService,
@@ -92,22 +67,11 @@ class ApplicationsService @Inject() (
     statsReporter: StatsReporter,
     configuration: Configuration,
     system: ActorSystem,
-    wsClient: WSClient
+    adjudicatorClient: AdjudicatorClient
   )(implicit val ec: DalExecutionContext)
     extends Logging {
 
-  private val applicationsCacheDuration = configuration.get[FiniteDuration]("memcached.application-ttl")
-
-  //** Adjudicator
-  private val adjudicatorAddress      = configuration.get[String]("adjudicator.address")
-  private val adjudicatorScheme       = configuration.get[String]("adjudicator.scheme")
-  private val adjudicatorEndpoint     = s"${adjudicatorScheme}${adjudicatorAddress}"
-  private val adjudicatorSharedSecret = configuration.get[String]("adjudicator.sharedSecret")
-  private val adjudicatorClient = new AdjudicatorWsClient(
-    adjudicatorEndpoint,
-    JwtSecretKey(adjudicatorSharedSecret),
-    wsClient
-  )
+  private val applicationsCacheDuration = configuration.get[FiniteDuration]("application-cache-ttl")
 
   def hmiDetails(id: String): Future[Option[Application]] =
     trustedApplicationProvider.application(id) // calls cached by TrustedApplicationProvider
@@ -225,8 +189,7 @@ class ApplicationsService @Inject() (
       hatName: String): Future[Any] =
     application.kind match {
       case _: ApplicationKind.Contract =>
-        adjudicatorClient
-          .joinContract(hatName, ContractId(UUID.fromString(application.id)))
+        adjudicatorClient.joinContract(hatName, ContractId(UUID.fromString(application.id)))
       case _ =>
         Future.successful(Done)
     }
@@ -346,13 +309,11 @@ class ApplicationsService @Inject() (
           .map(_ => throw e)
 
       case _: HatApplicationDependencyException =>
-        applicationStatus(application.application.id, bustCache = true).map { maybeApplication =>
-          maybeApplication match {
-            case Some(application) =>
-              application.copy(dependenciesEnabled = Some(false))
-            case None =>
-              throw new NoSuchElementException("Application not found")
-          }
+        applicationStatus(application.application.id, bustCache = true).map {
+          case Some(application) =>
+            application.copy(dependenciesEnabled = Some(false))
+          case None =>
+            throw new NoSuchElementException("Application not found")
         }
     }
   }
@@ -479,7 +440,7 @@ class ApplicationsService @Inject() (
           )
         for {
           ((status, _), canCacheStatus) <- eventualStatus
-          (mostRecentData, canCacheData) <- eventualMostRecentData
+          (mostRecentDateTime, canCacheData) <- eventualMostRecentData
         } yield {
           logger.debug(
             s"Check compatibility between $version and new ${app.status}: ${Version(version)
@@ -495,7 +456,7 @@ class ApplicationsService @Inject() (
                 app.status.compatibility.greaterThan(Version(version))
               ), // Needs updating if setup version beyond compatible
               dependenciesEnabled = None,
-              mostRecentData
+              mostRecentDateTime
             ),
             canCacheStatus && canCacheData
           )