From c9aa09d5a182acce649dd1e41c65950258713efa Mon Sep 17 00:00:00 2001
From: Samuel Herman <samuelherman@macbook-pro.lan>
Date: Fri, 21 Feb 2025 22:01:28 -0800
Subject: [PATCH 1/6] add jmh skeleton

Signed-off-by: Samuel Herman <samuelherman@macbook-pro.lan>
---
 benchmarks-jmh/README.md                      |  37 +++++
 benchmarks-jmh/pom.xml                        | 127 ++++++++++++++++++
 .../jvector/bench/RandomVectorsBenchmark.java |  34 +++++
 pom.xml                                       |   1 +
 4 files changed, 199 insertions(+)
 create mode 100644 benchmarks-jmh/README.md
 create mode 100644 benchmarks-jmh/pom.xml
 create mode 100644 benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java

diff --git a/benchmarks-jmh/README.md b/benchmarks-jmh/README.md
new file mode 100644
index 00000000..dbd0b31f
--- /dev/null
+++ b/benchmarks-jmh/README.md
@@ -0,0 +1,37 @@
+# JMH Benchmarks
+Micro benchmarks for jVector. While {@link Bench.java} is about recall, the JMH benchmarks
+are mostly targeting scalability and latency aspects.
+
+## Building and running the benchmark
+
+1. You can build and then run
+```shell
+mvn clean install
+java --enable-native-access=ALL-UNNAMED \
+  --add-modules=jdk.incubator.vector \
+  -XX:+HeapDumpOnOutOfMemoryError \
+  -Xmx14G -Djvector.experimental.enable_native_vectorization=true \
+  -jar benchmarks-jmh/target/benchmarks-jmh-4.0.0-beta.2-SNAPSHOT.jar 
+```
+2. Build and run the benchmarks directly
+```shell
+mvn clean package -pl :benchmarks-jmh -am -DskipTests=true exec:exec@benchmark -Pjmh
+```
+3. Build and run a specific benchmark or use command line arguments
+```shell
+# Run only benchmarks containing "Sample" in their name
+mvn clean package -pl :benchmarks-jmh -am -DskipTests=true exec:exec@benchmark -Pjmh -DargLine=".*Sample.*"
+
+# Run with different parameters
+mvn clean package -pl :benchmarks-jmh -am -DskipTests=true exec:exec@benchmark -Pjmh -DargLine="-f 1 -wi 5 -i 3 .*Sample.*"
+```
+
+Common JMH command line options you can use in the configuration or command line:
+- `-f <num>` - Number of forks
+- `-wi <num>` - Number of warmup iterations
+- `-i <num>` - Number of measurement iterations
+- `-w <time>` - Warmup time per iteration
+- `-r <time>` - Measurement time per iteration
+- `-t <num>` - Number of threads
+- `-p <param>=<value>` - Benchmark parameters
+- `-prof <profiler>` - Add profiler
diff --git a/benchmarks-jmh/pom.xml b/benchmarks-jmh/pom.xml
new file mode 100644
index 00000000..3a59bc94
--- /dev/null
+++ b/benchmarks-jmh/pom.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.github.jbellis</groupId>
+        <artifactId>jvector-parent</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>benchmarks-jmh</artifactId>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.release>22</maven.compiler.release>
+        <jmh.version>1.37</jmh.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>jvector-base</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.github.jbellis</groupId>
+            <artifactId>jvector-twenty</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.github.jbellis</groupId>
+            <artifactId>jvector-native</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!--Ensures that annotation processor is running during compilation-->
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.13.0</version>
+                <configuration>
+                    <annotationProcessorPaths>
+                        <path>
+                            <groupId>org.openjdk.jmh</groupId>
+                            <artifactId>jmh-generator-annprocess</artifactId>
+                            <version>${jmh.version}</version>
+                        </path>
+                    </annotationProcessorPaths>
+                </configuration>
+            </plugin>
+
+            <!-- Shade this so we can run as a standalone jar -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.3.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.openjdk.jmh.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+    <profiles>
+        <profile>
+            <id>jmh</id>
+            <activation>
+                <activeByDefault>false</activeByDefault>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <configuration>
+                            <skip>false</skip>
+                        </configuration>
+                        <executions>
+                            <execution>
+                                <id>benchmark</id>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>java</executable>
+                                    <arguments>
+                                        <argument>--enable-native-access=ALL-UNNAMED</argument>
+                                        <argument>--add-modules=jdk.incubator.vector</argument>
+                                        <argument>-XX:+HeapDumpOnOutOfMemoryError</argument>
+                                        <argument>-Xmx14G</argument>
+                                        <argument>-Djvector.experimental.enable_native_vectorization=true</argument>
+                                        <argument>-jar</argument>
+                                        <argument>${project.build.directory}/benchmarks-jmh-${project.version}.jar</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>
\ No newline at end of file
diff --git a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
new file mode 100644
index 00000000..40e2abb6
--- /dev/null
+++ b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.github.jbellis.jvector.bench;
+
+import org.openjdk.jmh.annotations.*;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class RandomVectorsBenchmark {
+
+    @Benchmark
+    public void measureSomething() {
+        // Your benchmark code here
+        System.out.println("Hello World!");
+    }
+}
diff --git a/pom.xml b/pom.xml
index c26373b0..ae795e55 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,6 +50,7 @@
         <module>jvector-tests</module>
         <module>jvector-multirelease</module>
         <module>jvector-examples</module>
+        <module>benchmarks-jmh</module>
     </modules>
     <build>
         <resources>

From 1fc9fea0400a13b90c412c2c87f6402481f9c44d Mon Sep 17 00:00:00 2001
From: Samuel Herman <sherman8915@gmail.com>
Date: Fri, 21 Feb 2025 22:47:06 -0800
Subject: [PATCH 2/6] add random query test

Signed-off-by: Samuel Herman <sherman8915@gmail.com>
---
 .gitignore                                    |  3 +
 benchmarks-jmh/pom.xml                        | 11 +++
 .../jvector/bench/RandomVectorsBenchmark.java | 71 +++++++++++++++++--
 benchmarks-jmh/src/main/resources/log4j2.xml  | 15 ++++
 4 files changed, 96 insertions(+), 4 deletions(-)
 create mode 100644 benchmarks-jmh/src/main/resources/log4j2.xml

diff --git a/.gitignore b/.gitignore
index 9646802a..bfcc2652 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,6 @@ hdf5/
 
 ### aider
 .aider*
+
+# JMH generated files
+dependency-reduced-pom.xml
diff --git a/benchmarks-jmh/pom.xml b/benchmarks-jmh/pom.xml
index 3a59bc94..430f00f2 100644
--- a/benchmarks-jmh/pom.xml
+++ b/benchmarks-jmh/pom.xml
@@ -33,6 +33,11 @@
             <artifactId>jvector-native</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.github.jbellis</groupId>
+            <artifactId>jvector-examples</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.openjdk.jmh</groupId>
             <artifactId>jmh-core</artifactId>
@@ -43,6 +48,12 @@
             <artifactId>jmh-generator-annprocess</artifactId>
             <version>${jmh.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <version>2.24.3</version>
+        </dependency>
+
     </dependencies>
 
     <build>
diff --git a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
index 40e2abb6..b2d54093 100644
--- a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
+++ b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
@@ -15,20 +15,83 @@
  */
 package io.github.jbellis.jvector.bench;
 
+import io.github.jbellis.jvector.example.SiftSmall;
+import io.github.jbellis.jvector.example.util.SiftLoader;
+import io.github.jbellis.jvector.graph.*;
+import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
 import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 @BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Thread)
 @Fork(1)
-@Warmup(iterations = 3)
+@Warmup(iterations = 2)
 @Measurement(iterations = 5)
+@Threads(1)
 public class RandomVectorsBenchmark {
+    private static final Logger log = LoggerFactory.getLogger(RandomVectorsBenchmark.class);
+    private RandomAccessVectorValues ravv;
+    private ArrayList<VectorFloat<?>> baseVectors;
+    private ArrayList<VectorFloat<?>> queryVectors;
+    private ArrayList<Set<Integer>> groundTruth;
+    private GraphIndexBuilder graphIndexBuilder;
+    private GraphIndex graphIndex;
+    int originalDimension;
+
+    @Setup
+    public void setup() throws IOException {
+        var siftPath = "siftsmall";
+        baseVectors = SiftLoader.readFvecs(String.format("%s/siftsmall_base.fvecs", siftPath));
+        queryVectors = SiftLoader.readFvecs(String.format("%s/siftsmall_query.fvecs", siftPath));
+        groundTruth = SiftLoader.readIvecs(String.format("%s/siftsmall_groundtruth.ivecs", siftPath));
+        log.info("base vectors size: {}, query vectors size: {}, loaded, dimensions {}",
+                baseVectors.size(), queryVectors.size(), baseVectors.get(0).length());
+        originalDimension = baseVectors.get(0).length();
+        // wrap the raw vectors in a RandomAccessVectorValues
+        ravv = new ListRandomAccessVectorValues(baseVectors, originalDimension);
+
+        // score provider using the raw, in-memory vectors
+        BuildScoreProvider bsp = BuildScoreProvider.randomAccessScoreProvider(ravv, VectorSimilarityFunction.EUCLIDEAN);
+
+        graphIndexBuilder = new GraphIndexBuilder(bsp,
+                ravv.dimension(),
+                16, // graph degree
+                100, // construction search depth
+                1.2f, // allow degree overflow during construction by this factor
+                1.2f); // relax neighbor diversity requirement by this factor
+        graphIndex = graphIndexBuilder.build(ravv);
+        queryVector = SiftSmall.randomVector(originalDimension);
+    }
+
+    @TearDown
+    public void tearDown() throws IOException {
+        baseVectors.clear();
+        queryVectors.clear();
+        groundTruth.clear();
+        graphIndexBuilder.close();
+    }
 
     @Benchmark
-    public void measureSomething() {
+    public void testOnHeapWithRandomQueryVectors(Blackhole blackhole) throws IOException {
+        var queryVector = SiftSmall.randomVector(originalDimension);
         // Your benchmark code here
-        System.out.println("Hello World!");
+        var searchResult = GraphSearcher.search(queryVector,
+                10, // number of results
+                ravv, // vectors we're searching, used for scoring
+                VectorSimilarityFunction.EUCLIDEAN, // how to score
+                graphIndex,
+                Bits.ALL); // valid ordinals to consider
+        blackhole.consume(searchResult);
     }
 }
diff --git a/benchmarks-jmh/src/main/resources/log4j2.xml b/benchmarks-jmh/src/main/resources/log4j2.xml
new file mode 100644
index 00000000..82378826
--- /dev/null
+++ b/benchmarks-jmh/src/main/resources/log4j2.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+    <Appenders>
+        <!-- Console Appender -->
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <!-- Root Logger -->
+        <Root level="INFO">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>
\ No newline at end of file

From 8403c3d9e025c0dec7f71369d87719f1ad5252cd Mon Sep 17 00:00:00 2001
From: Samuel Herman <sherman8915@gmail.com>
Date: Fri, 21 Feb 2025 22:48:27 -0800
Subject: [PATCH 3/6] minor fix

Signed-off-by: Samuel Herman <sherman8915@gmail.com>
---
 .../io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
index b2d54093..e6b34901 100644
--- a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
+++ b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
@@ -71,7 +71,6 @@ public void setup() throws IOException {
                 1.2f, // allow degree overflow during construction by this factor
                 1.2f); // relax neighbor diversity requirement by this factor
         graphIndex = graphIndexBuilder.build(ravv);
-        queryVector = SiftSmall.randomVector(originalDimension);
     }
 
     @TearDown

From 4100bf870cc6394d1fb23f50706f7d583f560c8a Mon Sep 17 00:00:00 2001
From: Samuel Herman <sherman8915@gmail.com>
Date: Fri, 21 Feb 2025 23:07:52 -0800
Subject: [PATCH 4/6] added a random benchamrk with parameter grid

Signed-off-by: Samuel Herman <sherman8915@gmail.com>
---
 .../jvector/bench/RandomVectorsBenchmark.java | 58 +++++------
 .../bench/StaticSetVectorsBenchmark.java      | 96 +++++++++++++++++++
 2 files changed, 127 insertions(+), 27 deletions(-)
 create mode 100644 benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/StaticSetVectorsBenchmark.java

diff --git a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
index e6b34901..8b0360f2 100644
--- a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
+++ b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
@@ -1,27 +1,13 @@
-/*
- * Copyright DataStax, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package io.github.jbellis.jvector.bench;
 
 import io.github.jbellis.jvector.example.SiftSmall;
-import io.github.jbellis.jvector.example.util.SiftLoader;
 import io.github.jbellis.jvector.graph.*;
 import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider;
 import io.github.jbellis.jvector.util.Bits;
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.VectorizationProvider;
 import io.github.jbellis.jvector.vector.types.VectorFloat;
+import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.infra.Blackhole;
 import org.slf4j.Logger;
@@ -29,7 +15,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 @BenchmarkMode(Mode.AverageTime)
@@ -41,23 +26,35 @@
 @Threads(1)
 public class RandomVectorsBenchmark {
     private static final Logger log = LoggerFactory.getLogger(RandomVectorsBenchmark.class);
+    private static final VectorTypeSupport VECTOR_TYPE_SUPPORT = VectorizationProvider.getInstance().getVectorTypeSupport();
     private RandomAccessVectorValues ravv;
     private ArrayList<VectorFloat<?>> baseVectors;
     private ArrayList<VectorFloat<?>> queryVectors;
-    private ArrayList<Set<Integer>> groundTruth;
     private GraphIndexBuilder graphIndexBuilder;
     private GraphIndex graphIndex;
     int originalDimension;
+    @Param({"1000", "10000", "100000", "1000000"})
+    int numBaseVectors;
+    @Param({"10"})
+    int numQueryVectors;
 
     @Setup
     public void setup() throws IOException {
-        var siftPath = "siftsmall";
-        baseVectors = SiftLoader.readFvecs(String.format("%s/siftsmall_base.fvecs", siftPath));
-        queryVectors = SiftLoader.readFvecs(String.format("%s/siftsmall_query.fvecs", siftPath));
-        groundTruth = SiftLoader.readIvecs(String.format("%s/siftsmall_groundtruth.ivecs", siftPath));
-        log.info("base vectors size: {}, query vectors size: {}, loaded, dimensions {}",
-                baseVectors.size(), queryVectors.size(), baseVectors.get(0).length());
-        originalDimension = baseVectors.get(0).length();
+        originalDimension = 128; // Example dimension, can be adjusted
+
+        baseVectors = new ArrayList<>(numBaseVectors);
+        queryVectors = new ArrayList<>(numQueryVectors);
+
+        for (int i = 0; i < numBaseVectors; i++) {
+            VectorFloat<?> vector = createRandomVector(originalDimension);
+            baseVectors.add(vector);
+        }
+
+        for (int i = 0; i < numQueryVectors; i++) {
+            VectorFloat<?> vector = createRandomVector(originalDimension);
+            queryVectors.add(vector);
+        }
+
         // wrap the raw vectors in a RandomAccessVectorValues
         ravv = new ListRandomAccessVectorValues(baseVectors, originalDimension);
 
@@ -73,16 +70,23 @@ public void setup() throws IOException {
         graphIndex = graphIndexBuilder.build(ravv);
     }
 
+    private VectorFloat<?> createRandomVector(int dimension) {
+        VectorFloat<?> vector = VECTOR_TYPE_SUPPORT.createFloatVector(dimension);
+        for (int i = 0; i < dimension; i++) {
+            vector.set(i, (float) Math.random());
+        }
+        return vector;
+    }
+
     @TearDown
     public void tearDown() throws IOException {
         baseVectors.clear();
         queryVectors.clear();
-        groundTruth.clear();
         graphIndexBuilder.close();
     }
 
     @Benchmark
-    public void testOnHeapWithRandomQueryVectors(Blackhole blackhole) throws IOException {
+    public void testOnHeapRandomVectors(Blackhole blackhole) {
         var queryVector = SiftSmall.randomVector(originalDimension);
         // Your benchmark code here
         var searchResult = GraphSearcher.search(queryVector,
diff --git a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/StaticSetVectorsBenchmark.java b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/StaticSetVectorsBenchmark.java
new file mode 100644
index 00000000..184aa214
--- /dev/null
+++ b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/StaticSetVectorsBenchmark.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.github.jbellis.jvector.bench;
+
+import io.github.jbellis.jvector.example.SiftSmall;
+import io.github.jbellis.jvector.example.util.SiftLoader;
+import io.github.jbellis.jvector.graph.*;
+import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider;
+import io.github.jbellis.jvector.util.Bits;
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
+import io.github.jbellis.jvector.vector.types.VectorFloat;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+@Fork(1)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Threads(1)
+public class StaticSetVectorsBenchmark {
+    private static final Logger log = LoggerFactory.getLogger(StaticSetVectorsBenchmark.class);
+    private RandomAccessVectorValues ravv;
+    private ArrayList<VectorFloat<?>> baseVectors;
+    private ArrayList<VectorFloat<?>> queryVectors;
+    private ArrayList<Set<Integer>> groundTruth;
+    private GraphIndexBuilder graphIndexBuilder;
+    private GraphIndex graphIndex;
+    int originalDimension;
+
+    @Setup
+    public void setup() throws IOException {
+        var siftPath = "siftsmall";
+        baseVectors = SiftLoader.readFvecs(String.format("%s/siftsmall_base.fvecs", siftPath));
+        queryVectors = SiftLoader.readFvecs(String.format("%s/siftsmall_query.fvecs", siftPath));
+        groundTruth = SiftLoader.readIvecs(String.format("%s/siftsmall_groundtruth.ivecs", siftPath));
+        log.info("base vectors size: {}, query vectors size: {}, loaded, dimensions {}",
+                baseVectors.size(), queryVectors.size(), baseVectors.get(0).length());
+        originalDimension = baseVectors.get(0).length();
+        // wrap the raw vectors in a RandomAccessVectorValues
+        ravv = new ListRandomAccessVectorValues(baseVectors, originalDimension);
+
+        // score provider using the raw, in-memory vectors
+        BuildScoreProvider bsp = BuildScoreProvider.randomAccessScoreProvider(ravv, VectorSimilarityFunction.EUCLIDEAN);
+
+        graphIndexBuilder = new GraphIndexBuilder(bsp,
+                ravv.dimension(),
+                16, // graph degree
+                100, // construction search depth
+                1.2f, // allow degree overflow during construction by this factor
+                1.2f); // relax neighbor diversity requirement by this factor
+        graphIndex = graphIndexBuilder.build(ravv);
+    }
+
+    @TearDown
+    public void tearDown() throws IOException {
+        baseVectors.clear();
+        queryVectors.clear();
+        groundTruth.clear();
+        graphIndexBuilder.close();
+    }
+
+    @Benchmark
+    public void testOnHeapWithRandomQueryVectors(Blackhole blackhole) throws IOException {
+        var queryVector = SiftSmall.randomVector(originalDimension);
+        // Your benchmark code here
+        var searchResult = GraphSearcher.search(queryVector,
+                10, // number of results
+                ravv, // vectors we're searching, used for scoring
+                VectorSimilarityFunction.EUCLIDEAN, // how to score
+                graphIndex,
+                Bits.ALL); // valid ordinals to consider
+        blackhole.consume(searchResult);
+    }
+}

From eae0f1e4f37c9229bc677277bdbd28a134c4db73 Mon Sep 17 00:00:00 2001
From: Samuel Herman <sherman8915@gmail.com>
Date: Tue, 25 Feb 2025 07:16:15 -0800
Subject: [PATCH 5/6] fix rat excludes

Signed-off-by: Samuel Herman <sherman8915@gmail.com>
---
 .../jvector/bench/RandomVectorsBenchmark.java     | 15 +++++++++++++++
 rat-excludes.txt                                  |  3 ++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
index 8b0360f2..33e7d9f7 100644
--- a/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
+++ b/benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/RandomVectorsBenchmark.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package io.github.jbellis.jvector.bench;
 
 import io.github.jbellis.jvector.example.SiftSmall;
diff --git a/rat-excludes.txt b/rat-excludes.txt
index 9569e501..35d2b6ce 100644
--- a/rat-excludes.txt
+++ b/rat-excludes.txt
@@ -8,4 +8,5 @@ pom.xml
 src/main/assembly/test-jar-with-dependencies.xml
 src/assembly/mrjar.xml
 src/assembly/sourcesjar.xml
-src/main/java/io/github/jbellis/jvector/vector/cnative/*
\ No newline at end of file
+src/main/java/io/github/jbellis/jvector/vector/cnative/*
+src/main/resources/log4j2.xml
\ No newline at end of file

From 6b289f0b8e95612c1fa84d5dccb7a1f050a9eb3f Mon Sep 17 00:00:00 2001
From: Samuel Herman <sherman8915@gmail.com>
Date: Tue, 25 Feb 2025 12:45:24 -0800
Subject: [PATCH 6/6] fix instructions in readme, remove unused exec profile

Signed-off-by: Samuel Herman <sherman8915@gmail.com>
---
 benchmarks-jmh/README.md | 20 +++++++++-----------
 benchmarks-jmh/pom.xml   | 39 ---------------------------------------
 2 files changed, 9 insertions(+), 50 deletions(-)

diff --git a/benchmarks-jmh/README.md b/benchmarks-jmh/README.md
index dbd0b31f..6068e817 100644
--- a/benchmarks-jmh/README.md
+++ b/benchmarks-jmh/README.md
@@ -6,24 +6,22 @@ are mostly targeting scalability and latency aspects.
 
 1. You can build and then run
 ```shell
-mvn clean install
+mvn clean install -DskipTests=true
 java --enable-native-access=ALL-UNNAMED \
   --add-modules=jdk.incubator.vector \
   -XX:+HeapDumpOnOutOfMemoryError \
   -Xmx14G -Djvector.experimental.enable_native_vectorization=true \
   -jar benchmarks-jmh/target/benchmarks-jmh-4.0.0-beta.2-SNAPSHOT.jar 
 ```
-2. Build and run the benchmarks directly
-```shell
-mvn clean package -pl :benchmarks-jmh -am -DskipTests=true exec:exec@benchmark -Pjmh
-```
-3. Build and run a specific benchmark or use command line arguments
-```shell
-# Run only benchmarks containing "Sample" in their name
-mvn clean package -pl :benchmarks-jmh -am -DskipTests=true exec:exec@benchmark -Pjmh -DargLine=".*Sample.*"
 
-# Run with different parameters
-mvn clean package -pl :benchmarks-jmh -am -DskipTests=true exec:exec@benchmark -Pjmh -DargLine="-f 1 -wi 5 -i 3 .*Sample.*"
+You can add additional optional JMH arguments dynamically from command line. For example, to run the benchmarks with 4 forks, 5 warmup iterations, 5 measurement iterations, 2 threads, and 10 seconds warmup time per iteration, use the following command:
+```shell
+java --enable-native-access=ALL-UNNAMED \
+  --add-modules=jdk.incubator.vector \
+  -XX:+HeapDumpOnOutOfMemoryError \
+  -Xmx14G -Djvector.experimental.enable_native_vectorization=true \
+  -jar benchmarks-jmh/target/benchmarks-jmh-4.0.0-beta.2-SNAPSHOT.jar \
+  -f 4 -wi 5 -i 5 -t 2 -w 10s
 ```
 
 Common JMH command line options you can use in the configuration or command line:
diff --git a/benchmarks-jmh/pom.xml b/benchmarks-jmh/pom.xml
index 430f00f2..c82ee270 100644
--- a/benchmarks-jmh/pom.xml
+++ b/benchmarks-jmh/pom.xml
@@ -96,43 +96,4 @@
             </plugin>
         </plugins>
     </build>
-    <profiles>
-        <profile>
-            <id>jmh</id>
-            <activation>
-                <activeByDefault>false</activeByDefault>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.codehaus.mojo</groupId>
-                        <artifactId>exec-maven-plugin</artifactId>
-                        <configuration>
-                            <skip>false</skip>
-                        </configuration>
-                        <executions>
-                            <execution>
-                                <id>benchmark</id>
-                                <goals>
-                                    <goal>exec</goal>
-                                </goals>
-                                <configuration>
-                                    <executable>java</executable>
-                                    <arguments>
-                                        <argument>--enable-native-access=ALL-UNNAMED</argument>
-                                        <argument>--add-modules=jdk.incubator.vector</argument>
-                                        <argument>-XX:+HeapDumpOnOutOfMemoryError</argument>
-                                        <argument>-Xmx14G</argument>
-                                        <argument>-Djvector.experimental.enable_native_vectorization=true</argument>
-                                        <argument>-jar</argument>
-                                        <argument>${project.build.directory}/benchmarks-jmh-${project.version}.jar</argument>
-                                    </arguments>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-    </profiles>
 </project>
\ No newline at end of file