Streamlined project a bit

.github/workflows/pr.yaml

@@ -1,7 +1,7 @@
 name: pr
 on: pull_request
 jobs:
-  test:
+  compile:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -18,8 +18,8 @@ jobs:
             ~/.sbt
             ~/.coursier
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
-      - name: Compile and run tests
-        run: sbt clean test
+      - name: Ensure successful compilation
+        run: sbt compile
   formatting:
     runs-on: ubuntu-latest
     steps:
@@ -38,6 +38,6 @@ jobs:
             ~/.coursier
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
       - name: Check formatting
-        run: sbt scalafmtCheck Test/scalafmtCheck
-      - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt Test/scalafmt'"
+        run: sbt scalafmtCheck
+      - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt'"
         if: ${{ failure() }}

.github/workflows/release.yaml

@@ -44,7 +44,9 @@ jobs:
           mv workspace/ichnaea.zip ichnaea.zip
           mv workspace/securibench-micro-JAVA.zip securibench-micro-JAVA.zip
           mv workspace/securibench-micro-JAVASRC.zip securibench-micro-JAVASRC.zip
-          mv workspace/THORAT.zip THORAT.zip
+          mv workspace/thorat.zip thorat.zip
+          mv workspace/defects4j.zip defects4j.zip
+          mv workspace/bugs_in_py.zip bugs_in_py.zip
       - name: Set next release version
         id: taggerFinal
         uses: anothrNick/[email protected]
@@ -59,4 +61,4 @@ jobs:
             ichnaea.zip
             securibench-micro-JAVA.zip
             securibench-micro-JAVASRC.zip
-            THORAT.zip
+            thorat.zip

README.md

@@ -1,7 +1,7 @@
 Joern Benchmarks
 ================
 
-A repository for running Joern against known benchmarks.
+A repository for building snapshots of datasets used by `joern-benchmarks`.
 
 ## Usage
 
@@ -11,21 +11,10 @@ sbt stage
 joern-benchmark v0.0.1
 Usage: joern-benchmark [options] benchmark
 
-A benchmarking suite for Joern
+A benchmark downloader tool for Joern benchmarks
   -h, --help
   --version                Prints the version
-  benchmark                The benchmark to run. Available [ALL,OWASP_JAVASRC,OWASP_JAVA,SECURIBENCH_MICRO_JAVASRC,SECURIBENCH_MICRO_JAVA]
+  benchmark                The benchmark to download. Available [ALL,SECURIBENCH_MICRO_SRC,SECURIBENCH_MICRO_JAVA,ICHNAEA_JSSRC,THORAT_PYSRC,BUGS_IN_PY,DEFECTS4J]
   -d, --dataset-dir <value>
-                           The dataset directory where benchmarks will be downloaded to. Default is `./workspace`.
+                           The dataset directory where benchmarks will be downloaded to. Default is `./workspace`
 ```
-
-## Benchmarks
-
-The benchmark naming convention of `<BENCHMARK>_<FRONTEND>`, e.g. `OWASP_JAVA` runs `OWASP` using the `jimple2cpg`
-frontend (JVM bytecode).
-
-| Benchmark                                                             | Status | Enabled Frontends |
-|-----------------------------------------------------------------------|--------|-------------------|
-| [`OWASP`](https://owasp.org/www-project-benchmark/)                   | WIP    | `JAVASRC`         |
-| [`SECURIBENCH_MICRO`](https://github.com/too4words/securibench-micro) | WIP    | `JAVASRC` `JAVA`  |
-| [`ICHNAEA`](https://www.franktip.org/pubs/tse2020.pdf)                | WIP    | `JSSRC`           |

build.sbt

@@ -1,32 +1,21 @@
-name                     := "joern-benchmarks-datasets-datasets"
+name := "joern-benchmarks-datasets"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.4.1"
 
-// parsed by project/Versions.scala, updated by updateDependencies.sh
-val cpgVersion = "1.6.11"
-val joernVersion      = "2.0.348"
-val overflowdbVersion = "1.192"
-
-
 libraryDependencies ++= Seq(
-  "com.github.pathikrit"    %% "better-files"      % Versions.betterFiles,
-  "com.github.scopt"        %% "scopt"             % Versions.scopt,
+  "com.github.pathikrit" %% "better-files" % Versions.betterFiles,
+  "com.github.scopt" %% "scopt" % Versions.scopt,
   "org.apache.logging.log4j" % "log4j-slf4j2-impl" % Versions.log4j % Optional,
-  "com.lihaoyi"             %% "requests"          % Versions.requests,
-  "com.lihaoyi"             %% "upickle"           % Versions.upickle,
-  "io.joern"                %% "joern-cli"         % Versions.joern,
-  "io.joern" %% "x2cpg" % Versions.joern
+  "com.lihaoyi" %% "requests" % Versions.requests,
+  "com.lihaoyi" %% "upickle" % Versions.upickle,
+  "com.github.sh4869" %% "semver-parser-scala" % Versions.semver,
+  "org.apache.commons" % "commons-compress" % Versions.commonsCompress
 )
 
-// mostly so that `sbt assembly` works, but also to ensure that we don't end up
-// with unexpected shadowing in jar hell
-excludeDependencies ++= Seq(ExclusionRule("io.shiftleft", "codepropertygraph-domain-classes_3"))
-
 assembly / assemblyMergeStrategy := {
-  case "log4j2.xml"                                             => MergeStrategy.first
-  case "module-info.class"                                      => MergeStrategy.first
-  case "META-INF/versions/9/module-info.class"                  => MergeStrategy.first
-  case "io/github/retronym/java9rtexport/Export.class"          => MergeStrategy.first
+  case "log4j2.xml" => MergeStrategy.first
+  case "module-info.class" => MergeStrategy.first
+  case "META-INF/versions/9/module-info.class" => MergeStrategy.first
   case PathList("scala", "collection", "internal", "pprint", _) => MergeStrategy.first
   case x =>
     val oldStrategy = (ThisBuild / assemblyMergeStrategy).value
@@ -48,5 +37,5 @@ ThisBuild / resolvers ++= Seq(
   "Gradle Releases" at "https://repo.gradle.org/gradle/libs-releases/"
 )
 
-Compile / doc / sources                := Seq.empty
+Compile / doc / sources := Seq.empty
 Compile / packageDoc / publishArtifact := false

joern-benchmarks-datasets

@@ -1 +1 @@
-target/universal/stage/bin/main
+target/universal/stage/bin/joern-benchmarks-datasets

project/Versions.scala

@@ -1,28 +1,9 @@
 object Versions {
-  val cpg        = parseVersion("cpgVersion")
-  val joern      = parseVersion("joernVersion")
-  val overflowdb = parseVersion("overflowdbVersion")
-
   val betterFiles = "3.9.2"
-  val log4j       = "2.20.0"
-  val requests    = "0.8.0"
-  val scopt       = "4.1.0"
-  val upickle     = "3.3.0"
-
-  val jsAstGen = "3.14.0"
-
-  private def parseVersion(key: String): String = {
-    val versionRegexp = s""".*val $key[ ]+=[ ]?"(.*?)"""".r
-    val versions: List[String] = scala.io.Source
-      .fromFile("build.sbt")
-      .getLines
-      .filter(_.contains(s"val $key"))
-      .collect { case versionRegexp(version) => version }
-      .toList
-    assert(
-      versions.size == 1,
-      s"""unable to extract $key from build.sbt, expected exactly one line like `val $key= "0.0.0-SNAPSHOT"`."""
-    )
-    versions.head
-  }
+  val commonsCompress = "1.27.1"
+  val log4j = "2.20.0"
+  val requests = "0.8.0"
+  val scopt = "4.1.0"
+  val upickle = "3.3.0"
+  val semver ="0.0.6"
 }

project/plugins.sbt

@@ -2,5 +2,4 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.3")
 addSbtPlugin("com.github.sbt" % "sbt-findbugs" % "2.0.0")
 addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1")
 addSbtPlugin("com.github.sbt"  % "sbt-native-packager"   % "1.9.7")
-addSbtPlugin("io.shiftleft" % "sbt-overflowdb" % "2.104")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1")

src/main/scala/io/joern/benchmarks/datasets/BenchmarkDataset.scala

@@ -5,26 +5,22 @@ import io.joern.benchmarks.datasets.AvailableBenchmarks
 import io.joern.benchmarks.datasets.runner.{
   BugsInPyDownloader,
   DatasetDownloader,
+  Defects4jDownloader,
   IchnaeaDownloader,
   SecuribenchMicroDownloader,
   ThoratDownloader
 }
 import org.slf4j.LoggerFactory
 import upickle.default.*
 
-/** The main benchmarking process.
-  */
 class BenchmarkDataset(config: BenchmarkDatasetConfig) {
   private val logger = LoggerFactory.getLogger(getClass)
 
   def evaluate(): Unit = {
-    logger.info("Beginning evaluation")
+    logger.info("Beginning downloads")
 
     def runBenchmark(benchmarkRunnerCreator: BenchmarkDatasetConfig => DatasetDownloader): Unit = {
-      val benchmarkRunner = benchmarkRunnerCreator(config)
-      val benchmarkName   = benchmarkRunner.benchmarkName
-      logger.info(s"Running $benchmarkName")
-      benchmarkRunner.run()
+      benchmarkRunnerCreator(config).run()
     }
 
     if (config.benchmark == AvailableBenchmarks.ALL) {
@@ -42,9 +38,10 @@ object BenchmarkDataset {
       x => new SecuribenchMicroDownloader(x.datasetDir, JavaCpgTypes.JAVASRC)
     ),
     (AvailableBenchmarks.SECURIBENCH_MICRO_JAVA, x => new SecuribenchMicroDownloader(x.datasetDir, JavaCpgTypes.JAVA)),
-    (AvailableBenchmarks.ICHNAEA_JSSRC, x => new IchnaeaDownloader(x.datasetDir)),
-    (AvailableBenchmarks.THORAT_PYSRC, x => new ThoratDownloader(x.datasetDir)),
-    (AvailableBenchmarks.BUGS_IN_PY, x => new BugsInPyDownloader(x.datasetDir))
+    (AvailableBenchmarks.ICHNAEA, x => new IchnaeaDownloader(x.datasetDir)),
+    (AvailableBenchmarks.THORAT, x => new ThoratDownloader(x.datasetDir)),
+    (AvailableBenchmarks.BUGS_IN_PY, x => new BugsInPyDownloader(x.datasetDir)),
+    (AvailableBenchmarks.DEFECTS4J, x => new Defects4jDownloader(x.datasetDir))
   )
 
 }

src/main/scala/io/joern/benchmarks/datasets/BenchmarkDatasetConfig.scala

@@ -13,19 +13,13 @@ object AvailableBenchmarks extends Enumeration {
   // Joern
   val SECURIBENCH_MICRO_SRC  = Value
   val SECURIBENCH_MICRO_JAVA = Value
-  val ICHNAEA_JSSRC          = Value
-  val THORAT_PYSRC           = Value
+  val ICHNAEA                = Value
+  val THORAT                 = Value
   val BUGS_IN_PY             = Value
+  val DEFECTS4J              = Value
 }
 
 object JavaCpgTypes extends Enumeration {
   val JAVASRC = Value
   val JAVA    = Value
-  val SEMGREP = Value
-}
-
-object OutputFormat extends Enumeration {
-  val JSON = Value
-  val CSV  = Value
-  val MD   = Value
 }