Merge pull request #19 from scala/benchmark

Add micro-benchmarks

README.md

@@ -1,4 +1,4 @@
-## Collection-Strawman
+# Collection-Strawman
 
 [![Join the chat at https://gitter.im/scala/collection-strawman](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/scala/collection-strawman?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
@@ -7,3 +7,19 @@ Prototype improvements for Scala collections.
 - [Gitter Discussion](https://gitter.im/scala/collection-strawman)
 - [Dotty Issue](https://github.com/lampepfl/dotty/issues/818)
 - [Scala Center Proposal](https://github.com/scalacenter/advisoryboard/blob/master/proposals/007-collections.md)
+
+## Build
+
+- Compile the collections and run the
+  tests:
+  ~~~
+  >; compile; test
+  ~~~
+- Run the memory benchmark:
+  ~~~
+  > memoryBenchmark/run
+  ~~~
+- Run the execution time benchmark:
+  ~~~
+  > timeBenchmark/jmh:run
+  ~~~

benchmarks/memory/src/main/scala/strawman/collection/MemoryFootprint.scala

@@ -0,0 +1,50 @@
+package bench
+
+import strawman.collection.immutable.{LazyList, List}
+
+import scala.{Any, AnyRef, App, Int, Long}
+import scala.Predef.{println, ArrowAssoc}
+import scala.compat.Platform
+import java.lang.Runtime
+
+import strawman.collection.mutable.{ArrayBuffer, ListBuffer}
+
+object MemoryFootprint extends App {
+
+  val sizes = scala.List(8, 64, 512, 4096, 32768, 262144, 2097152)
+
+  val runtime = Runtime.getRuntime
+  val obj: AnyRef = null
+  var placeholder: Any = _
+
+  def benchmark[A](gen: Int => A): scala.List[(Int, Long)] = (
+    // We run 5 iterations and pick the last result only
+    for (_ <- scala.Range(0, 5)) yield {
+      for (size <- sizes) yield {
+        placeholder = null
+        Platform.collectGarbage()
+        val memBefore = runtime.totalMemory() - runtime.freeMemory()
+        placeholder = gen(size)
+        Platform.collectGarbage()
+        val memAfter = runtime.totalMemory() - runtime.freeMemory()
+        size -> (memAfter - memBefore)
+      }
+    }
+  ).last
+
+  val memories =
+    scala.Predef.Map(
+      "scala.List"  -> benchmark(scala.List.fill(_)(obj)),
+      "List"        -> benchmark(List.fill(_)(obj)),
+      "LazyList"    -> benchmark(LazyList.fill(_)(obj)),
+      "ArrayBuffer" -> benchmark(ArrayBuffer.fill(_)(obj)),
+      "ListBuffer"  -> benchmark(ListBuffer.fill(_)(obj))
+    )
+
+  // Print the results as a CSV document
+  println("Collection;" + sizes.mkString(";"))
+  for ((name, values) <- memories) {
+    println(name + ";" + values.map(_._2).mkString(";"))
+  }
+
+}

benchmarks/time/src/main/scala/strawman/collection/immutable/LazyListBenchmark.scala

@@ -0,0 +1,58 @@
+package strawman.collection.immutable
+
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+import scala.{Any, AnyRef, Int, Unit}
+
+@BenchmarkMode(scala.Array(Mode.AverageTime))
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(1)
+@Warmup(iterations = 12)
+@Measurement(iterations = 12)
+@State(Scope.Benchmark)
+class LazyListBenchmark {
+
+  @Param(scala.Array("8", "64", "512", "4096", "32768", "262144"/*, "2097152"*/))
+  var size: Int = _
+
+  var xs: LazyList[AnyRef] = _
+  var obj: Any = _
+
+  @Setup(Level.Trial)
+  def initData(): Unit = {
+    xs = LazyList.fill(size)("")
+    obj = ""
+  }
+
+  @Benchmark
+  def cons(): Any = {
+    var ys = LazyList.empty[Any]
+    var i = 0
+    while (i < size) {
+      ys = obj #:: ys
+      i += 1
+    }
+    ys
+  }
+
+  @Benchmark
+  def uncons(): Any = xs.tail
+
+  @Benchmark
+  def concat(): Any = xs ++ xs
+
+  @Benchmark
+  def foreach(): Any = {
+    var n = 0
+    xs.foreach(x => if (x eq null) n += 1)
+    n
+  }
+
+  @Benchmark
+  def lookup(): Any = xs(size - 1)
+
+  @Benchmark
+  def map(): Any = xs.map(x => if (x eq null) "foo" else "bar")
+
+}

benchmarks/time/src/main/scala/strawman/collection/immutable/ListBenchmark.scala

@@ -0,0 +1,70 @@
+package strawman.collection.immutable
+
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+
+import scala.{Any, AnyRef, Int, Unit}
+
+@BenchmarkMode(scala.Array(Mode.AverageTime))
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(1)
+@Warmup(iterations = 12)
+@Measurement(iterations = 12)
+@State(Scope.Benchmark)
+class ListBenchmark {
+
+  @Param(scala.Array("8", "64", "512", "4096", "32768", "262144"/*, "2097152"*/))
+  var size: Int = _
+
+  var xs: List[AnyRef] = _
+  var obj: Any = _
+
+  @Setup(Level.Trial)
+  def initData(): Unit = {
+    xs = List.fill(size)("")
+    obj = ""
+  }
+
+  @Benchmark
+  def cons(): Any = {
+    var ys = List.empty[Any]
+    var i = 0
+    while (i < size) {
+      ys = obj :: ys
+      i += 1
+    }
+    ys
+  }
+
+  @Benchmark
+  def uncons(): Any = xs.tail
+
+  @Benchmark
+  def concat(): Any = xs ++ xs
+
+  @Benchmark
+  def foreach(): Any = {
+    var n = 0
+    xs.foreach(x => if (x eq null) n += 1)
+    n
+  }
+
+  @Benchmark
+  def foreach_while(): Any = {
+    var n = 0
+    var ys = xs
+    while (ys.nonEmpty) {
+      if (ys.head eq null) n += 1
+      ys = ys.tail
+    }
+    n
+  }
+
+  @Benchmark
+  def lookup(): Any = xs(size - 1)
+
+  @Benchmark
+  def map(): Any = xs.map(x => if (x eq null) "foo" else "bar")
+
+}

benchmarks/time/src/main/scala/strawman/collection/immutable/ScalaListBenchmark.scala

@@ -0,0 +1,58 @@
+package strawman.collection.immutable
+
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+import scala.{Any, AnyRef, Int, Unit}
+
+@BenchmarkMode(scala.Array(Mode.AverageTime))
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(1)
+@Warmup(iterations = 12)
+@Measurement(iterations = 12)
+@State(Scope.Benchmark)
+class ScalaListBenchmark {
+
+  @Param(scala.Array("8", "64", "512", "4096", "32768", "262144"/*, "2097152"*/))
+  var size: Int = _
+
+  var xs: scala.List[AnyRef] = _
+  var obj: Any = _
+
+  @Setup(Level.Trial)
+  def initData(): Unit = {
+    xs = scala.List.fill(size)("")
+    obj = ""
+  }
+
+  @Benchmark
+  def cons(): Any = {
+    var ys = scala.List.empty[Any]
+    var i = 0
+    while (i < size) {
+      ys = obj :: ys
+      i += 1
+    }
+    ys
+  }
+
+  @Benchmark
+  def uncons(): Any = xs.tail
+
+  @Benchmark
+  def concat(): Any = xs ++ xs
+
+  @Benchmark
+  def foreach(): Any = {
+    var n = 0
+    xs.foreach(x => if (x eq null) n += 1)
+    n
+  }
+
+  @Benchmark
+  def lookup(): Any = xs(size - 1)
+
+  @Benchmark
+  def map(): Any = xs.map(x => if (x eq null) "foo" else "bar")
+
+}

benchmarks/time/src/main/scala/strawman/collection/mutable/ArrayBufferBenchmark.scala

@@ -0,0 +1,58 @@
+package strawman.collection.mutable
+
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+import scala.{Any, AnyRef, Int, Unit}
+
+@BenchmarkMode(scala.Array(Mode.AverageTime))
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(1)
+@Warmup(iterations = 12)
+@Measurement(iterations = 12)
+@State(Scope.Benchmark)
+class ArrayBufferBenchmark {
+
+  @Param(scala.Array("8", "64", "512", "4096", "32768", "262144"/*, "2097152"*/))
+  var size: Int = _
+
+  var xs: ArrayBuffer[AnyRef] = _
+  var obj: Any = _
+
+  @Setup(Level.Trial)
+  def initData(): Unit = {
+    xs = ArrayBuffer.fill(size)("")
+    obj = ""
+  }
+
+  @Benchmark
+  def cons(): Any = {
+    var ys = ArrayBuffer.empty[Any]
+    var i = 0
+    while (i < size) {
+      ys += obj
+      i += 1
+    }
+    ys
+  }
+
+  @Benchmark
+  def uncons(): Any = xs.tail
+
+  @Benchmark
+  def concat(): Any = xs ++ xs
+
+  @Benchmark
+  def foreach(): Any = {
+    var n = 0
+    xs.foreach(x => if (x eq null) n += 1)
+    n
+  }
+
+  @Benchmark
+  def lookup(): Any = xs(size - 1)
+
+  @Benchmark
+  def map(): Any = xs.map(x => if (x eq null) "foo" else "bar")
+
+}

benchmarks/time/src/main/scala/strawman/collection/mutable/ListBufferBenchmark.scala

@@ -0,0 +1,58 @@
+package strawman.collection.mutable
+
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+import scala.{Any, AnyRef, Int, Unit}
+
+@BenchmarkMode(scala.Array(Mode.AverageTime))
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(1)
+@Warmup(iterations = 12)
+@Measurement(iterations = 12)
+@State(Scope.Benchmark)
+class ListBufferBenchmark {
+
+  @Param(scala.Array("8", "64", "512", "4096", "32768", "262144"/*, "2097152"*/))
+  var size: Int = _
+
+  var xs: ListBuffer[AnyRef] = _
+  var obj: Any = _
+
+  @Setup(Level.Trial)
+  def initData(): Unit = {
+    xs = ListBuffer.fill(size)("")
+    obj = ""
+  }
+
+  @Benchmark
+  def cons(): Any = {
+    var ys = ListBuffer.empty[Any]
+    var i = 0
+    while (i < size) {
+      ys += obj
+      i += 1
+    }
+    ys
+  }
+
+  @Benchmark
+  def uncons(): Any = xs.tail
+
+  @Benchmark
+  def concat(): Any = xs ++ xs
+
+  @Benchmark
+  def foreach(): Any = {
+    var n = 0
+    xs.foreach(x => if (x eq null) n += 1)
+    n
+  }
+
+  @Benchmark
+  def lookup(): Any = xs(size - 1)
+
+  @Benchmark
+  def map(): Any = xs.map(x => if (x eq null) "foo" else "bar")
+
+}