Skip to content

Commit

Permalink
🔖 Release v1.1.18
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidBakerEffendi
DavidBakerEffendi committed Mar 30, 2022
1 parent c72f8a4 commit 2fbccdf
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 56 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).

## [1.1.18] - 2022-03-30

### Added

- New `PlumeStastistics` entries related to measuring result re-use and fetching speeds.

### Changed

- Data flow cache now writes to `.cbor` instead of `.json` for improved I/O performance.

## [1.1.17] - 2022-03-29

### Changed
Expand Down
51 changes: 26 additions & 25 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name := "Plume"
inThisBuild(
List(
organization := "com.github.plume-oss",
version := "1.1.15-SNAP",
version := "1.1.18",
scalaVersion := "2.13.8",
crossScalaVersions := Seq("2.13.8", "3.1.1"),
resolvers ++= Seq(
Expand Down Expand Up @@ -42,30 +42,31 @@ Test / fork := true
Test / parallelExecution := false

libraryDependencies ++= Seq(
"io.shiftleft" %% "codepropertygraph" % cpgVersion,
"io.shiftleft" %% "semanticcpg" % cpgVersion,
"io.joern" %% "dataflowengineoss" % joernVersion,
"io.joern" %% "x2cpg" % joernVersion,
"io.joern" %% "jimple2cpg" % joernVersion,
"io.joern" %% "x2cpg" % joernVersion % Test classifier "tests",
"org.soot-oss" % "soot" % sootVersion,
"org.apache.tinkerpop" % "tinkergraph-gremlin" % tinkerGraphVersion,
"org.apache.tinkerpop" % "gremlin-driver" % tinkerGraphVersion,
"org.neo4j.driver" % "neo4j-java-driver" % neo4jVersion,
"commons-codec" % "commons-codec" % apacheCodecVersion,
"commons-io" % "commons-io" % apacheIoVersion,
"org.apache.commons" % "commons-lang3" % apacheLangVersion,
"com.softwaremill.sttp.client3" %% "core" % sttpVersion,
"com.softwaremill.sttp.client3" %% "circe" % sttpVersion,
"com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
"com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion,
"org.scalaj" % "scalaj-http_2.13" % scalajHttpVersion,
"org.lz4" % "lz4-java" % lz4Version,
"org.slf4j" % "slf4j-api" % slf4jVersion,
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
"org.apache.logging.log4j" % "log4j-core" % log4jVersion % Test,
"org.apache.logging.log4j" % "log4j-slf4j-impl" % log4jVersion % Test,
"org.scalatest" %% "scalatest" % scalatestVersion % Test
"io.shiftleft" %% "codepropertygraph" % cpgVersion,
"io.shiftleft" %% "semanticcpg" % cpgVersion,
"io.joern" %% "dataflowengineoss" % joernVersion,
"io.joern" %% "x2cpg" % joernVersion,
"io.joern" %% "jimple2cpg" % joernVersion,
"io.joern" %% "x2cpg" % joernVersion % Test classifier "tests",
"org.soot-oss" % "soot" % sootVersion,
"org.apache.tinkerpop" % "tinkergraph-gremlin" % tinkerGraphVersion,
"org.apache.tinkerpop" % "gremlin-driver" % tinkerGraphVersion,
"org.neo4j.driver" % "neo4j-java-driver" % neo4jVersion,
"commons-codec" % "commons-codec" % apacheCodecVersion,
"commons-io" % "commons-io" % apacheIoVersion,
"org.apache.commons" % "commons-lang3" % apacheLangVersion,
"com.softwaremill.sttp.client3" %% "core" % sttpVersion,
"com.softwaremill.sttp.client3" %% "circe" % sttpVersion,
"com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
"com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion,
"com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % jacksonVersion,
"org.scalaj" % "scalaj-http_2.13" % scalajHttpVersion,
"org.lz4" % "lz4-java" % lz4Version,
"org.slf4j" % "slf4j-api" % slf4jVersion,
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
"org.apache.logging.log4j" % "log4j-core" % log4jVersion % Test,
"org.apache.logging.log4j" % "log4j-slf4j-impl" % log4jVersion % Test,
"org.scalatest" %% "scalatest" % scalatestVersion % Test
) ++ Seq(
"io.circe" %% "circe-core",
"io.circe" %% "circe-generic",
Expand Down
3 changes: 2 additions & 1 deletion src/main/scala/com/github/plume/oss/PlumeStatistics.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ object PlumeStatistics extends Enumeration {
type PlumeStatistic = Value

val TIME_OPEN_DRIVER, TIME_CLOSE_DRIVER, TIME_EXTRACTION, TIME_REACHABLE_BY_QUERYING,
TIME_REMOVING_OUTDATED_GRAPH, PROGRAM_CLASSES, PROGRAM_METHODS = Value
TIME_REMOVING_OUTDATED_GRAPH, TIME_REMOVING_OUTDATED_CACHE, TIME_RETRIEVING_CACHE,
TIME_STORING_CACHE, PROGRAM_CLASSES, PROGRAM_METHODS = Value

private val statistics: mutable.Map[PlumeStatistic, Long] =
PlumeStatistics.values.map((_, 0L)).to(collection.mutable.Map)
Expand Down
8 changes: 4 additions & 4 deletions src/main/scala/com/github/plume/oss/domain/package.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ package com.github.plume.oss

import com.fasterxml.jackson.core.util.DefaultPrettyPrinter
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.fasterxml.jackson.databind.json.JsonMapper
import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule}
import com.fasterxml.jackson.dataformat.cbor.databind.CBORMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.joern.dataflowengineoss.queryengine.{PathElement, ReachableByResult, ResultTable}
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.codepropertygraph.generated.nodes.{Call, CfgNode, StoredNode}
Expand All @@ -23,10 +23,10 @@ import scala.util.Using
package object domain {

private val logger = LoggerFactory.getLogger("com.github.plume.oss.domain")
private val mapper = JsonMapper
private val mapper = CBORMapper
.builder()
.addModule(DefaultScalaModule)
.build() :: ClassTagExtensions
.build()

/** Given an object and a path, will serialize the object to the given path.
* @param o object to serialize.
Expand Down
70 changes: 44 additions & 26 deletions src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ final case class OverflowDbDriver(
),
heapPercentageThreshold: Int = 80,
serializationStatsEnabled: Boolean = false,
dataFlowCacheFile: Option[Path] = Some(Paths.get("dataFlowCache.json")),
dataFlowCacheFile: Option[Path] = Some(Paths.get("dataFlowCache.cbor")),
compressDataFlowCache: Boolean = true
) extends IDriver {

Expand Down Expand Up @@ -77,7 +77,12 @@ final case class OverflowDbDriver(
dataFlowCacheFile match {
case Some(filePath) =>
if (Files.isRegularFile(filePath))
Some(deserializeCache(filePath, compressDataFlowCache))
Some(
PlumeStatistics.time(
PlumeStatistics.TIME_RETRIEVING_CACHE,
{ deserializeCache(filePath, compressDataFlowCache) }
)
)
else
Some(new ConcurrentHashMap[Long, Vector[SerialReachableByResult]]())
case None => None
Expand All @@ -86,12 +91,21 @@ final case class OverflowDbDriver(
private implicit var context: EngineContext =
EngineContext(
Semantics.fromList(List()),
EngineConfig(initialTable = deserializeResultTable(table, cpg))
EngineConfig(initialTable =
PlumeStatistics.time(
PlumeStatistics.TIME_RETRIEVING_CACHE,
{ deserializeResultTable(table, cpg) }
)
)
)

private def saveDataflowCache(): Unit = dataFlowCacheFile match {
case Some(filePath) if table.isDefined && !table.get.isEmpty =>
serializeCache(table.get, filePath, compressDataFlowCache)
PlumeStatistics.time(
PlumeStatistics.TIME_STORING_CACHE, {
serializeCache(table.get, filePath, compressDataFlowCache)
}
)
case _ => // Do nothing
}

Expand Down Expand Up @@ -423,29 +437,33 @@ final case class OverflowDbDriver(

table match {
case Some(oldTab) =>
val startPSize = oldTab.asScala.flatMap(_._2).size

val newTab = oldTab.asScala
.filter { case (k: Long, _) => isNodeUnderTypes(k, unchangedTypes) }
.map { case (k: Long, v: Vector[SerialReachableByResult]) =>
val filteredPaths = v.filterNot(isResultExpired)
(k, filteredPaths)
PlumeStatistics.time(
PlumeStatistics.TIME_REMOVING_OUTDATED_CACHE, {
val startPSize = oldTab.asScala.flatMap(_._2).size

val newTab = oldTab.asScala
.filter { case (k: Long, _) => isNodeUnderTypes(k, unchangedTypes) }
.map { case (k: Long, v: Vector[SerialReachableByResult]) =>
val filteredPaths = v.filterNot(isResultExpired)
(k, filteredPaths)
}
.toMap
// Refresh old table and add new entries
oldTab.clear()
newTab.foreach { case (k, v) => oldTab.put(k, v) }

val leftOverPSize = newTab.flatMap(_._2).size
if (startPSize > 0)
logger.info(
s"Able to re-use ${(leftOverPSize.toDouble / startPSize) * 100.0}% of the saved paths. " +
s"Removed ${startPSize - leftOverPSize} expired paths from $startPSize saved paths."
)
setDataflowContext(
context.config.maxCallDepth,
context.semantics,
deserializeResultTable(Some(oldTab), cpg)
)
}
.toMap
// Refresh old table and add new entries
oldTab.clear()
newTab.foreach { case (k, v) => oldTab.put(k, v) }

val leftOverPSize = newTab.flatMap(_._2).size
if (startPSize > 0)
logger.info(
s"Able to re-use ${(leftOverPSize.toDouble / startPSize) * 100.0}% of the saved paths. " +
s"Removed ${startPSize - leftOverPSize} expired paths from $startPSize saved paths."
)
setDataflowContext(
context.config.maxCallDepth,
context.semantics,
deserializeResultTable(Some(oldTab), cpg)
)
case None => // Do nothing
}
Expand Down

0 comments on commit 2fbccdf

Please sign in to comment.