refactor logging

thenetcircle · Oct 24, 2018 · 96066ca · 96066ca
1 parent d9aeb96
commit 96066ca
Show file tree

Hide file tree

Showing 15 changed files with 175 additions and 101 deletions.
diff --git a/admin/backend/src/main/resources/logback.xml b/admin/backend/src/main/resources/logback.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+	<property name="EB_LOGPREFIX" value="${EB_LOGDIR:-./log}/${EB_APPNAME:-event-bus}.${EB_ENV:-dev}" />
+
+	<!-- system appender -->
+	<appender name="ADMIN" class="ch.qos.logback.core.FileAppender">
+		<file>${EB_LOGPREFIX}.admin.log</file>
+		<append>true</append>
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} %X{sourceThread} %X{akkaSource} - %msg%n</pattern>
+		</encoder>
+	</appender>
+
+	<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} %X{sourceThread} %X{akkaSource} - %msg%n</pattern>
+		</encoder>
+	</appender>
+
+	<!-- loggers -->
+	<root level="${EB_LOGLEVEL:-DEBUG}">
+		<appender-ref ref="${EB_LOGREF:-ADMIN}" />
+	</root>
+
+	<logger name="org.apache.zookeeper" level="INFO" />
+
+</configuration>
diff --git a/core/src/main/resources/logback.xml b/core/src/main/resources/logback.xml
@@ -1,14 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
 
-    <shutdownHook/>
+    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook" />
 
-	<!-- appenders -->
-	<appender name="ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
-		<file>${EB_LOGFILE:-./log/default}.log</file>
+	<property name="EB_LOGPREFIX" value="${EB_LOGDIR:-./log}/${EB_APPNAME:-event-bus}.${EB_ENV:-dev}.${EB_RUNNERNAME:-default-runner}" />
+
+	<!-- system appender -->
+	<appender name="SYSTEM" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>${EB_LOGPREFIX}.system.log</file>
 		<append>true</append>
 		<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-			<fileNamePattern>${EB_LOGFILE:-./log/default}.%d{yyyy-MM-dd}.log</fileNamePattern>
+			<fileNamePattern>${EB_LOGPREFIX}.system.%d{yyyy-MM-dd}.log</fileNamePattern>
 			<maxHistory>30</maxHistory>
 			<totalSizeCap>3GB</totalSizeCap>
 		</rollingPolicy>
@@ -17,28 +19,38 @@
 		</encoder>
 	</appender>
 
-	<appender name="ASYNC_ROLLING" class="ch.qos.logback.classic.AsyncAppender">
-		<appender-ref ref="ROLLING" />
+	<appender name="ASYNC_SYSTEM" class="ch.qos.logback.classic.AsyncAppender">
+		<appender-ref ref="SYSTEM" />
 	</appender>
 
-	<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+	<!-- task appender -->
+	<appender name="TASK" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>${EB_LOGPREFIX}.task.log</file>
+		<append>true</append>
+		<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+			<fileNamePattern>${EB_LOGPREFIX}.task.%d{yyyy-MM-dd}.log</fileNamePattern>
+			<maxHistory>30</maxHistory>
+			<totalSizeCap>3GB</totalSizeCap>
+		</rollingPolicy>
 		<encoder>
 			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} %X{sourceThread} %X{akkaSource} - %msg%n</pattern>
 		</encoder>
 	</appender>
 
-	<appender name="SENTRY" class="io.sentry.logback.SentryAppender">
-		<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-			<level>WARN</level>
-		</filter>
+	<appender name="ASYNC_TASK" class="ch.qos.logback.classic.AsyncAppender">
+		<appender-ref ref="TASK" />
 	</appender>
 
-	<appender name="MISSED" class="ch.qos.logback.core.FileAppender">
-		<file>${EB_MISSED_EVENTS_FILE:-./log/missed_events}</file>
+	<!-- missed appender -->
+	<appender name="MISSED" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>${EB_LOGPREFIX}.missed.log</file>
 		<append>true</append>
 		<immediateFlush>true</immediateFlush>
+		<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+			<fileNamePattern>${EB_LOGPREFIX}.missed.%d{yyyy-MM-dd_HH}.log</fileNamePattern>
+		</rollingPolicy>
 		<encoder>
-		  <pattern>%msg%n</pattern>
+			<pattern>%msg%n</pattern>
 		</encoder>
 	</appender>
 
@@ -51,19 +63,37 @@
 		<neverBlock>true</neverBlock>
 	</appender>
 
+	<!-- other appender -->
+	<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} %X{sourceThread} %X{akkaSource} - %msg%n</pattern>
+		</encoder>
+	</appender>
+
+	<appender name="SENTRY" class="io.sentry.logback.SentryAppender">
+		<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+			<level>WARN</level>
+		</filter>
+	</appender>
+
 
 	<!-- loggers -->
 	<root level="${EB_LOGLEVEL:-DEBUG}">
-		<appender-ref ref="${EB_LOGREF:-CONSOLE}" />
+		<appender-ref ref="${EB_LOGREF:-ASYNC_SYSTEM}" />
 		<appender-ref ref="SENTRY" />
 	</root>
 
 	<logger name="org.apache.kafka" level="INFO" />
 	<logger name="org.apache.zookeeper" level="INFO" />
 	<logger name="com.datastax.driver.core.Connection" level="INFO" />
 
-	<logger name="com.thenetcircle.event_bus.misc.MissedEventHandler" additivity="false">
-		<appender-ref ref="ASYNC_MISSED" />
+	<logger name="com.thenetcircle.event_bus.misc.Logging.task" level="${EB_LOGLEVEL:-DEBUG}" additivity="false">
+		<appender-ref ref="${EB_LOGREF:-ASYNC_TASK}" />
+		<appender-ref ref="SENTRY" />
+	</logger>
+
+	<logger name="com.thenetcircle.event_bus.misc.Logging.missed" level="DEBUG" additivity="false">
+		<appender-ref ref="MISSED" />
 	</logger>
 
 </configuration>
diff --git a/core/src/main/scala/com/thenetcircle/event_bus/AbstractApp.scala b/core/src/main/scala/com/thenetcircle/event_bus/AbstractApp.scala
@@ -19,13 +19,13 @@ package com.thenetcircle.event_bus
 
 import akka.actor.ActorSystem
 import com.thenetcircle.event_bus.context.AppContext
+import com.thenetcircle.event_bus.misc.Logging
 import com.typesafe.config.Config
-import com.typesafe.scalalogging.StrictLogging
 
 import scala.concurrent.Await
 import scala.concurrent.duration._
 
-trait AbstractApp extends StrictLogging {
+trait AbstractApp extends Logging {
 
   logger.info("Application is initializing.")
 

diff --git a/...e/event_bus/misc/MissedEventHandler.scala → ...thenetcircle/event_bus/misc/Logging.scala b/...e/event_bus/misc/MissedEventHandler.scala → ...thenetcircle/event_bus/misc/Logging.scala
@@ -17,12 +17,21 @@
 
 package com.thenetcircle.event_bus.misc
 
-import com.thenetcircle.event_bus.interfaces.Event
-import com.typesafe.scalalogging.StrictLogging
+import com.typesafe.scalalogging.{LazyLogging, Logger}
+import org.slf4j.LoggerFactory
 
-object MissedEventHandler extends StrictLogging {
+trait Logging extends LazyLogging {
 
-  def handle(event: Event): Unit =
-    logger.warn(event.body.data)
+  protected lazy val taskLogger: Logger =
+    Logger(LoggerFactory.getLogger(Logging.taskLoggerPrefix + "." + getClass.getName.split('.').last))
+
+}
+
+object Logging {
+
+  val missedLogger: Logger =
+    Logger(LoggerFactory.getLogger(s"com.thenetcircle.event_bus.misc.Logging.missed"))
+
+  val taskLoggerPrefix = s"com.thenetcircle.event_bus.misc.Logging.task"
 
 }
diff --git a/core/src/main/scala/com/thenetcircle/event_bus/misc/ZooKeeperManager.scala b/core/src/main/scala/com/thenetcircle/event_bus/misc/ZooKeeperManager.scala
@@ -18,7 +18,6 @@
 package com.thenetcircle.event_bus.misc
 
 import com.thenetcircle.event_bus.context.AppContext
-import com.typesafe.scalalogging.StrictLogging
 import org.apache.curator.framework.imps.CuratorFrameworkState
 import org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode
 import org.apache.curator.framework.recipes.cache._
@@ -28,7 +27,7 @@ import org.apache.curator.retry.ExponentialBackoffRetry
 import scala.collection.JavaConverters._
 
 class ZooKeeperManager private (client: CuratorFramework, rootPath: String)(implicit appContext: AppContext)
-    extends StrictLogging {
+    extends Logging {
 
   val appRootPath: String =
     appContext.getSystemConfig().getString("app.zookeeper.rootpath") + s"/${appContext.getAppName()}"

diff --git a/core/src/main/scala/com/thenetcircle/event_bus/story/Story.scala b/core/src/main/scala/com/thenetcircle/event_bus/story/Story.scala
@@ -23,13 +23,12 @@ import akka.{Done, NotUsed}
 import com.thenetcircle.event_bus.context.TaskRunningContext
 import com.thenetcircle.event_bus.interfaces.EventStatus.{Norm, ToFB}
 import com.thenetcircle.event_bus.interfaces.{Event, _}
-import com.thenetcircle.event_bus.misc.MonitoringHelp
+import com.thenetcircle.event_bus.misc.{Logging, MonitoringHelp}
 import com.thenetcircle.event_bus.story.StoryStatus.StoryStatus
-import com.typesafe.scalalogging.StrictLogging
 
 import scala.concurrent.Future
-import scala.util.{Failure, Success}
 import scala.util.control.NonFatal
+import scala.util.{Failure, Success}
 
 case class StorySettings(name: String, status: StoryStatus = StoryStatus.INIT)
 
@@ -39,7 +38,7 @@ class Story(
     val sinkTask: SinkTask,
     val transformTasks: Option[List[TransformTask]] = None,
     val fallbackTask: Option[FallbackTask] = None
-) extends StrictLogging
+) extends Logging
     with MonitoringHelp {
 
   type Payload = (EventStatus, Event) // middle result type

diff --git a/core/src/main/scala/com/thenetcircle/event_bus/story/StoryZooKeeperListener.scala b/core/src/main/scala/com/thenetcircle/event_bus/story/StoryZooKeeperListener.scala
@@ -20,15 +20,14 @@ package com.thenetcircle.event_bus.story
 import akka.actor.{ActorRef, ActorSystem, Cancellable}
 import com.thenetcircle.event_bus.BuildInfo
 import com.thenetcircle.event_bus.context.AppContext
-import com.thenetcircle.event_bus.misc.{Util, ZooKeeperManager}
-import com.typesafe.scalalogging.StrictLogging
+import com.thenetcircle.event_bus.misc.{Logging, Util, ZooKeeperManager}
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type._
 import org.apache.curator.framework.recipes.cache.{PathChildrenCache, PathChildrenCacheEvent}
 import org.apache.curator.framework.recipes.leader.LeaderLatch
 
+import scala.concurrent.duration._
 import scala.util.Random
 import scala.util.control.NonFatal
-import scala.concurrent.duration._
 
 object StoryZooKeeperListener {
   def apply(runnerName: String, storyRunner: ActorRef, storyBuilder: StoryBuilder)(
@@ -41,7 +40,7 @@ object StoryZooKeeperListener {
 class StoryZooKeeperListener(runnerName: String, storyRunner: ActorRef, storyBuilder: StoryBuilder)(
     implicit appContext: AppContext,
     system: ActorSystem
-) extends StrictLogging {
+) extends Logging {
 
   require(
     appContext.getZooKeeperManager().isDefined,

diff --git a/core/src/main/scala/com/thenetcircle/event_bus/tasks/cassandra/CassandraFallback.scala b/core/src/main/scala/com/thenetcircle/event_bus/tasks/cassandra/CassandraFallback.scala
@@ -28,16 +28,15 @@ import com.google.common.util.concurrent.{FutureCallback, Futures, ListenableFut
 import com.thenetcircle.event_bus.context.{TaskBuildingContext, TaskRunningContext}
 import com.thenetcircle.event_bus.interfaces.EventStatus.{Fail, InFB, ToFB}
 import com.thenetcircle.event_bus.interfaces.{Event, EventStatus, FallbackTask, FallbackTaskBuilder}
-import com.thenetcircle.event_bus.misc.Util
-import com.typesafe.scalalogging.StrictLogging
+import com.thenetcircle.event_bus.misc.{Logging, Util}
 import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.util.control.NonFatal
 
 case class CassandraSettings(contactPoints: List[String], port: Int = 9042, parallelism: Int = 2)
 
-class CassandraFallback(val settings: CassandraSettings) extends FallbackTask with StrictLogging {
+class CassandraFallback(val settings: CassandraSettings) extends FallbackTask with Logging {
 
   private var clusterOption: Option[Cluster]             = None
   private var sessionOption: Option[Session]             = None
@@ -115,12 +114,16 @@ class CassandraFallback(val settings: CassandraSettings) extends FallbackTask wi
               .map[(EventStatus, Event)](result => (InFB, event))
               .recover {
                 case NonFatal(ex) =>
-                  logger.warn(s"sending to cassandra[1] fallback was failed with error $ex")
+                  taskLogger.warn(
+                    s"sending to cassandra[1] fallback was failed with error $ex"
+                  )
                   (Fail(ex), event)
               }
           } catch {
             case NonFatal(ex) =>
-              logger.debug(s"sending to cassandra[2] fallback failed with error $ex")
+              taskLogger.debug(
+                s"sending to cassandra[2] fallback failed with error $ex"
+              )
               Future.successful((Fail(ex), event))
           }
       }