diff --git a/.github/workflows/php.yml b/.github/workflows/php.yml
index 5050f096..96ba17e1 100755
--- a/.github/workflows/php.yml
+++ b/.github/workflows/php.yml
@@ -19,4 +19,4 @@ jobs:
       - name: Run codestyle sniffer
         run: composer run-script codestyle
       - name: Run benchmark
-        run: php tests/benchmark.php
+        run: php tests/benchmarks/run.php
diff --git a/tests/benchmark.php b/tests/benchmark.php
deleted file mode 100644
index 054b6a81..00000000
--- a/tests/benchmark.php
+++ /dev/null
@@ -1,23 +0,0 @@
-<?php
-
-require __DIR__ . '/benchmark-mocks.php';
-
-// make sure we're not running through all migrations
-update_option('mc4wp_version', '999.1.1');
-
-$memory = memory_get_usage();
-$time_start = microtime(true);
-
-require dirname(__DIR__, 1) . '/mailchimp-for-wp.php';
-
-do_action('plugins_loaded');
-do_action('setup_theme');
-do_action('after_setup_theme');
-do_action('init');
-do_action('wp_loaded');
-
-$time = round((microtime(true) - $time_start) * 1000, 2);
-$memory_used = (memory_get_usage() - $memory) >> 10;
-
-echo "Memory: $memory_used KB\n";
-echo "Time: $time ms\n";
diff --git a/tests/benchmark-mocks.php b/tests/benchmarks/mocks.php
similarity index 89%
rename from tests/benchmark-mocks.php
rename to tests/benchmarks/mocks.php
index e025143c..fbd423c2 100644
--- a/tests/benchmark-mocks.php
+++ b/tests/benchmarks/mocks.php
@@ -5,7 +5,7 @@
  * phpcs:disable PSR1.Classes.ClassDeclaration.MissingNamespace
 */
 
-define('ABSPATH', dirname(__DIR__, 1));
+define('ABSPATH', dirname(__DIR__, 2));
 define('HOUR_IN_SECONDS', 3600);
 
 $options = [];
@@ -147,24 +147,29 @@ function is_wp_error($thing)
     return false;
 }
 
-function __($string, $text_domain)
+function __($text, $domain)
 {
-    return $string;
+    return $text;
 }
 
-function register_post_type($name, $args)
+function _e($text, $domain)
 {
+    echo $text;
 }
 
-function wp_register_script($handle, $src, $deps = [], $version = '')
+function register_post_type()
 {
 }
 
-function plugins_url($file, $path)
+function wp_register_script($handle, $src, $deps = [], $in_footer = true)
 {
-    return $path;
 }
 
+function plugins_url($path, $plugin = '')
+{
+}
+
+
 class wpdb_mock
 {
     public $prefix = '';
diff --git a/tests/benchmarks/plugin.php b/tests/benchmarks/plugin.php
new file mode 100644
index 00000000..0482e2d2
--- /dev/null
+++ b/tests/benchmarks/plugin.php
@@ -0,0 +1,33 @@
+<?php
+
+// phpcs:disable PSR1.Files.SideEffects
+
+// if query parameter is set, disable opcache until end of request
+if (isset($_GET['disable-opcache']) && (int) $_GET['disable-opcache'] === 1) {
+    ini_set('opcache.enable', 0);
+    ini_set('opcache.enable_cli', 0);
+}
+
+require __DIR__ . '/mocks.php';
+
+// make sure we're not running through all migrations
+update_option('mc4wp_version', '999.1.1');
+
+$memory = memory_get_usage();
+$time_start = microtime(true);
+
+// require main plugin file
+require dirname(__DIR__, 2) . '/mailchimp-for-wp.php';
+
+do_action('plugins_loaded');
+do_action('setup_theme');
+do_action('after_setup_theme');
+do_action('init');
+do_action('wp_loaded');
+do_action('parse_request');
+
+$time = (microtime(true) - $time_start) * 1000.0;
+$memory_used = (memory_get_usage() - $memory) >> 10;
+
+header("Content-Type: application/json");
+echo json_encode(['memory' => $memory_used, 'time' => $time]);
diff --git a/tests/benchmarks/run.php b/tests/benchmarks/run.php
new file mode 100755
index 00000000..ff369f13
--- /dev/null
+++ b/tests/benchmarks/run.php
@@ -0,0 +1,59 @@
+<?php
+
+// phpcs:disable PSR1.Files.SideEffects
+
+function analyze(array $values): array
+{
+    $sum = 0.0;
+    $min = PHP_FLOAT_MAX;
+    $max = PHP_FLOAT_MIN;
+    foreach ($values as $v) {
+        $sum += $v;
+        $min =  min($min, $v);
+        $max = max($max, $v);
+    }
+    $mean = $sum / count($values);
+    return [round($min, 3), round($mean, 3), round($max, 3)];
+}
+
+function bench(int $n = 100, bool $opcache = true): void
+{
+    // run benchmark
+    $memories = [];
+    $times = [];
+    $url = $opcache ? 'http://localhost:8080/plugin.php' : 'http://localhost:8080/plugin.php?disable-opcache=1';
+
+    // make a single request to test whether HTTP server is up
+    // but also to warm the opcache
+    $body = file_get_contents($url);
+    if ($body === false) {
+        throw new Exception("Error making HTTP request. Is the HTTP server running?");
+    }
+
+    // run for 5 seconds
+    $time_until = time() + 5;
+    while (time() < $time_until) {
+        $body = file_get_contents($url);
+        $data = json_decode($body);
+        $memories[] = $data->memory;
+        $times[] = $data->time;
+    }
+
+    [$time_min, $time_mean, $time_max] = analyze($times);
+
+    echo $opcache ? "with opcache:     " : "without opcache:  ";
+    echo "min: $time_min\tmean: $time_mean\tmax: $time_max\n";
+}
+
+$root = __DIR__;
+$ph = popen("php -S localhost:8080 -q -t {$root} &2>/dev/null", "r");
+if (!$ph) {
+    echo "Error starting HTTP server\n";
+    exit(1);
+}
+sleep(2);
+
+bench(100, true);
+bench(100, false);
+
+pclose($ph);