Add js integration test for solidity test runner

.github/workflows/edr-benchmark-feat-solidity-tests.yml

@@ -50,13 +50,13 @@ jobs:
         run: pnpm test
 
       - name: Run benchmark
-        run: pnpm run -s benchmark
+        run: pnpm run benchmark
 
       - name: Validate regressions
-        run: pnpm run -s verify
+        run: pnpm run verify
 
       - name: Generate report for github-action-benchmark
-        run: pnpm run -s report | tee report.json
+        run: pnpm run --silent report | tee report.json
 
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1

.github/workflows/edr-benchmark.yml

@@ -37,13 +37,13 @@ jobs:
         run: pnpm test
 
       - name: Run benchmark
-        run: pnpm run -s benchmark
+        run: pnpm run benchmark
 
       - name: Validate regressions
-        run: pnpm run -s verify
+        run: pnpm run verify
 
       - name: Generate report for github-action-benchmark
-        run: pnpm run -s report | tee report.json
+        run: pnpm run --silent report | tee report.json
 
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1

.github/workflows/edr-ci.yml

@@ -144,15 +144,17 @@ jobs:
           ./.github/scripts/cargo-doc.sh
         shell: bash
 
-  lint:
-    name: Run Prettier
+  build-and-lint:
+    name: Build and lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-node
 
       - name: Install package
         run: pnpm install --frozen-lockfile --prefer-offline
+      - name: Run build script
+        run: pnpm run build:dev
       - name: Run lint script
         run: pnpm run lint
 
@@ -167,7 +169,20 @@ jobs:
         run: pnpm install --frozen-lockfile --prefer-offline
 
       - name: Build edr_napi
-        run: cd crates/edr_napi && pnpm build
+        run: cd crates/edr_napi && pnpm build:dev
 
       - name: Check that there are no uncommitted changes
         run: git diff --exit-code
+
+  edr-integration-tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-node
+
+      - name: Install package
+        run: pnpm install --frozen-lockfile --prefer-offline
+
+      - name: Run integration tests
+        run: pnpm run --recursive --filter './js/integration-tests/*' test

.github/workflows/edr-npm-release.yml

@@ -33,10 +33,10 @@ jobs:
           - host: macos-12
             target: x86_64-apple-darwin
             build: |
-              pnpm build
+              pnpm run build
               strip -x *.node
           - host: windows-2022
-            build: pnpm build
+            build: pnpm run build
             target: x86_64-pc-windows-msvc
           - host: ubuntu-22.04
             target: x86_64-unknown-linux-gnu
@@ -56,7 +56,7 @@ jobs:
           - host: macos-12
             target: aarch64-apple-darwin
             build: |
-              pnpm build --target aarch64-apple-darwin
+              pnpm run build --target aarch64-apple-darwin
               strip -x *.node
           - host: ubuntu-22.04
             target: aarch64-unknown-linux-gnu

.syncpackrc.js

@@ -12,7 +12,7 @@ const config = {
     },
     {
       packages: ["**"],
-      dependencies: ["@nomicfoundation/edr", "hardhat-solidity-tests"],
+      dependencies: ["@nomicfoundation/edr", "@nomicfoundation/edr-helpers", "hardhat-solidity-tests"],
       dependencyTypes: ["local"]
     },
   ],

crates/edr_napi/package.json

@@ -52,17 +52,18 @@
   "repository": "NomicFoundation/edr.git",
   "scripts": {
     "artifacts": "napi artifacts",
-    "build": "napi build --platform --profile napi-publish",
+    "build": "pnpm run build:publish",
     "build:debug": "napi build --platform",
+    "build:dev": "napi build --platform --release",
+    "build:publish": "napi build --platform --profile napi-publish",
     "build:scenarios": "napi build --platform --release --features scenarios",
-    "build:test": "napi build --platform --release",
     "build:tracing": "napi build --platform --release --features tracing",
     "clean": "rm -rf @nomicfoundation/edr.node",
     "eslint": "eslint 'test/**/*.ts'",
     "lint": "pnpm run prettier && pnpm run eslint",
     "lint:fix": "pnpm run prettier --write",
     "prepublishOnly": "bash scripts/prepublish.sh",
-    "pretest": "pnpm build:test",
+    "pretest": "cd ../.. && pnpm build:dev",
     "prettier": "prettier --check \"test/**.ts\"",
     "test": "pnpm tsc && node --max-old-space-size=8192 node_modules/mocha/bin/_mocha --recursive \"test/**/*.ts\"",
     "testNoBuild": "pnpm tsc && node --max-old-space-size=8192 node_modules/mocha/bin/_mocha --recursive \"test/**/*.ts\"",

crates/edr_napi/test/solidity-tests.ts

@@ -3,36 +3,12 @@ import { assert } from "chai";
 import {
   ArtifactId,
   ContractData,
-  SuiteResult,
-  runSolidityTests,
   Artifact,
+  runSolidityTests,
   SolidityTestRunnerConfigArgs,
+  SuiteResult,
 } from "..";
 
-// This throws an error if the tests fail
-async function executeSolidityTests(
-  artifacts: Artifact[],
-  testSuites: ArtifactId[],
-  configArgs: SolidityTestRunnerConfigArgs
-): Promise<SuiteResult[]> {
-  return new Promise((resolve, reject) => {
-    const resultsFromCallback: SuiteResult[] = [];
-
-    runSolidityTests(
-      artifacts,
-      testSuites,
-      configArgs,
-      (result: SuiteResult) => {
-        resultsFromCallback.push(result);
-        if (resultsFromCallback.length === artifacts.length) {
-          resolve(resultsFromCallback);
-        }
-      },
-      reject
-    );
-  });
-}
-
 describe("Solidity Tests", () => {
   it("executes basic tests", async function () {
     const artifacts = [
@@ -45,7 +21,7 @@ describe("Solidity Tests", () => {
       projectRoot: __dirname,
     };
 
-    const results = await executeSolidityTests(artifacts, testSuites, config);
+    const results = await runAllSolidityTests(artifacts, testSuites, config);
 
     assert.equal(results.length, artifacts.length);
 
@@ -77,7 +53,7 @@ describe("Solidity Tests", () => {
     };
 
     await assert.isRejected(
-      executeSolidityTests(artifacts, testSuites, config),
+      runAllSolidityTests(artifacts, testSuites, config),
       Error
     );
   });
@@ -103,3 +79,26 @@ function loadContract(artifactPath: string): Artifact {
     contract,
   };
 }
+
+async function runAllSolidityTests(
+  artifacts: Artifact[],
+  testSuites: ArtifactId[],
+  configArgs: SolidityTestRunnerConfigArgs
+): Promise<SuiteResult[]> {
+  return new Promise((resolve, reject) => {
+    const resultsFromCallback: SuiteResult[] = [];
+
+    runSolidityTests(
+      artifacts,
+      testSuites,
+      configArgs,
+      (suiteResult: SuiteResult) => {
+        resultsFromCallback.push(suiteResult);
+        if (resultsFromCallback.length === artifacts.length) {
+          resolve(resultsFromCallback);
+        }
+      },
+      reject
+    );
+  });
+}

crates/tools/js/benchmark/.gitignore

@@ -1,2 +1,3 @@
 node_modules
 forge-std
+dist/

crates/tools/js/benchmark/package.json

@@ -5,6 +5,7 @@
   "author": "",
   "devDependencies": {
     "@nomicfoundation/edr": "workspace:*",
+    "@nomicfoundation/edr-helpers": "workspace:*",
     "@types/argparse": "^2.0.16",
     "@types/chai": "^4.2.0",
     "@types/lodash": "^4.14.123",
@@ -23,24 +24,26 @@
     "mocha": "^10.0.0",
     "prettier": "^3.2.5",
     "simple-git": "^3.25.0",
-    "tsx": "^4.7.1"
+    "tsx": "^4.7.1",
+    "typescript": "~5.5.3"
   },
   "keywords": [],
   "license": "ISC",
   "private": true,
   "scripts": {
     "benchmark": "node --noconcurrent_sweeping --noconcurrent_recompilation --max-old-space-size=28000 --import tsx src/index.ts benchmark",
+    "build": "pnpm run build:dev",
+    "build:dev": "tsc --build --incremental .",
     "eslint": "eslint \"**/*.ts\"",
     "help": "tsx src/index.ts -h",
-    "lint": "pnpm run typecheck && pnpm run prettier && pnpm run eslint",
+    "lint": "pnpm run prettier && pnpm run eslint",
     "lint:fix": "pnpm run prettier --write",
-    "prebenchmark": "cd ../../../edr_napi/ && pnpm build",
-    "presoltests": "cd ../../../edr_napi/ && pnpm build",
+    "prebenchmark": "cd ../../../.. && pnpm build",
+    "presoltests": "pnpm run prebenchmark",
     "prettier": "prettier --check \"**/*.{js,ts}\"",
     "report": "tsx src/index.ts report",
     "soltests": "node --noconcurrent_sweeping --noconcurrent_recompilation --max-old-space-size=28000 --import tsx src/index.ts solidity-tests",
     "test": "mocha --recursive --require tsx/cjs \"test/**/*.ts\"",
-    "typecheck": "tsc --noEmit",
     "verify": "tsx src/index.ts verify"
   }
 }

crates/tools/js/benchmark/src/solidity-tests.ts

@@ -6,7 +6,7 @@ import fs from "fs";
 import { execSync } from "child_process";
 import path from "path";
 import simpleGit from "simple-git";
-import { SuiteResult, runSolidityTests } from "@nomicfoundation/edr";
+import { runAllSolidityTests } from "@nomicfoundation/edr-helpers";
 
 const EXCLUDED_TEST_SUITES = new Set([
   "StdChainsTest",
@@ -64,32 +64,14 @@ export async function runForgeStdTests(forgeStdRepoPath: string) {
     )
     .map((a) => a.id);
 
-  const results = await new Promise<any>((resolve, reject) => {
-    const resultsFromCallback: SuiteResult[] = [];
-    const configs = {
-      projectRoot: forgeStdRepoPath,
-      fuzz: {
-        failurePersistDir: path.join(forgeStdRepoPath, "failures"),
-      },
-    };
-
-    runSolidityTests(
-      artifacts,
-      testSuiteIds,
-      configs,
-      (result) => {
-        console.error(
-          `${result.id.name} took ${computeElapsedSec(start)} seconds`
-        );
-
-        resultsFromCallback.push(result);
-        if (resultsFromCallback.length === artifacts.length) {
-          resolve(resultsFromCallback);
-        }
-      },
-      reject
-    );
-  });
+  const configs = {
+    projectRoot: forgeStdRepoPath,
+    fuzz: {
+      failurePersistDir: path.join(forgeStdRepoPath, "failures"),
+    },
+  };
+  const results = await runAllSolidityTests(artifacts, testSuiteIds, configs);
+
   console.error("elapsed (s)", computeElapsedSec(start));
 
   if (results.length !== EXPECTED_RESULTS) {
@@ -103,7 +85,7 @@ export async function runForgeStdTests(forgeStdRepoPath: string) {
   for (const res of results) {
     for (const r of res.testResults) {
       if (r.status !== "Success") {
-        failed.add(`${res.name} ${r.name} ${r.status}`);
+        failed.add(`${res.id.name} ${r.name} ${r.status}`);
       }
     }
   }

crates/tools/js/benchmark/tsconfig.json

@@ -4,6 +4,7 @@
     "module": "commonjs",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
+    "outDir": "dist",
     "strict": true,
     "skipLibCheck": true,
     "resolveJsonModule": true

hardhat-solidity-tests/.eslintrc.js

@@ -0,0 +1,10 @@
+module.exports = {
+  extends: [`${__dirname}/../config/eslint/eslintrc.js`],
+  parserOptions: {
+    project: `${__dirname}/tsconfig.json`,
+    sourceType: "module",
+  },
+  rules: {
+    "prefer-template": "off",
+  },
+};

hardhat-solidity-tests/example/hardhat.config.js

@@ -1,4 +1,4 @@
-require("hardhat-solidity-tests")
+require("hardhat-solidity-tests");
 
 /** @type import('hardhat/config').HardhatUserConfig */
 module.exports = {

hardhat-solidity-tests/example/package.json

@@ -2,9 +2,10 @@
   "name": "hardhat-solidity-tests-example",
   "version": "0.0.1",
   "author": "Nomic Foundation",
-  "dependencies": {
+  "devDependencies": {
     "hardhat": "2.22.9",
-    "hardhat-solidity-tests": "workspace:*"
+    "hardhat-solidity-tests": "workspace:*",
+    "ts-node": "^10.8.0"
   },
   "license": "MIT",
   "private": true,

hardhat-solidity-tests/package.json

@@ -2,11 +2,21 @@
   "name": "hardhat-solidity-tests",
   "version": "0.0.1",
   "author": "Nomic Foundation",
+  "dependencies": {
+    "@nomicfoundation/edr-helpers": "workspace:*"
+  },
   "devDependencies": {
-    "@nomicfoundation/edr": "workspace:*",
-    "@tsconfig/node18": "^18.2.4",
+    "@tsconfig/node20": "^20.1.4",
     "@types/node": "^20.0.0",
+    "@typescript-eslint/eslint-plugin": "5.61.0",
+    "@typescript-eslint/parser": "5.61.0",
+    "eslint": "^8.44.0",
+    "eslint-config-prettier": "9.1.0",
+    "eslint-plugin-import": "2.27.5",
+    "eslint-plugin-mocha": "10.4.1",
+    "eslint-plugin-prettier": "5.2.1",
     "hardhat": "2.22.9",
+    "prettier": "^3.2.5",
     "typescript": "~5.5.3"
   },
   "license": "MIT",
@@ -16,6 +26,10 @@
   },
   "private": true,
   "scripts": {
-    "build": "tsc"
+    "build": "pnpm run build:dev",
+    "build:dev": "tsc --build --incremental .",
+    "eslint": "eslint src/**/*.ts",
+    "lint": "pnpm prettier && pnpm eslint",
+    "prettier": "prettier --check \"**/*.{js,ts,md,json}\""
   }
 }