diff --git a/test/Makefile b/test/Makefile index 0f23d6c..48c6ee9 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,8 @@ default: run-tests WASM_OPT := ../build/binaryen/bin/wasm-opt +WASI_SDK_PATH := ../deps/wasi-sdk-13.0 +WASI_SYSROOT := $(abspath ${WASI_SDK_PATH}/share/wasi-sysroot) OUTPUT_DIR := ../build/test # https://github.com/caspervonb/wasi-test-suite @@ -21,6 +23,11 @@ WASMTIME_DST := $(OUTPUT_DIR)/wasmtime WASMTIME_SRC_TESTS := $(shell ls $(WASI_TESTS_CRATE_PATH)/src/bin/*.rs) WASMTIME_DST_TESTS := $(WASMTIME_SRC_TESTS:$(WASMTIME_SRC)/%.rs=$(WASMTIME_DST)/%.wasm) +BENCHMARK_SRC := ./subjects +BENCHMARK_DST := $(OUTPUT_DIR)/benchmark +BENCHMARK_SRC_TESTS := $(shell ls ./subjects/*.c) +BENCHMARK_DST_TESTS := $(BENCHMARK_SRC_TESTS:$(BENCHMARK_SRC)/%.c=$(BENCHMARK_DST)/%.wasm) + $(WASMTIME_BIN)/%.wasm: $(wildcard $(WASI_TESTS_CRATE_PATH)/src/**) cargo build --bin $* --target wasm32-wasi --manifest-path $(WASI_TESTS_CRATE_PATH)/Cargo.toml @@ -36,9 +43,12 @@ node_modules: ./package.json ./package-lock.json ../package.json npm install --no-audit --no-fund --no-progress --quiet touch $@ -run-tests: $(BUNDLE) $(WASI_TEST_SUITE_DST_TESTS) $(WASMTIME_DST_TESTS) $(OUTPUT_DIR)/wasm-table.ts node_modules +$(OUTPUT_DIR)/standalone.mjs: $(BENCHMARK_DST_TESTS) node_modules ./driver/*.ts + node benchmark-build.mjs + +run-tests: $(BUNDLE) $(OUTPUT_DIR)/wasm-table.ts node_modules $(OUTPUT_DIR)/standalone.mjs $(shell npm bin)/tsc -p ./tsconfig.json - JEST_JUNIT_OUTPUT_DIR=$(OUTPUT_DIR) NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 \ + JEST_JUNIT_OUTPUT_DIR=$(OUTPUT_DIR) OUTPUT_DIR=$(OUTPUT_DIR) NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 \ $(shell npm bin)/jest --detectOpenHandles -i $(OUTPUT_DIR)/wasm-table.ts: $(WASI_TEST_SUITE_DST_TESTS) $(WASMTIME_DST_TESTS) @@ -51,7 +61,7 @@ $(MEMFS_DST): $(MEMFS_SRC) $(BUNDLE): $(wildcard ../dist/**) $(wildcard ./driver/**) $(MEMFS_DST) $(OUTPUT_DIR)/wasm-table.ts node_modules mkdir -p $(@D) - $(shell npm bin)/esbuild --bundle ./driver/index.ts --outfile=$@ --format=esm --log-level=warning --external:*.wasm + $(shell npm bin)/esbuild --bundle ./driver/worker.ts --outfile=$@ --format=esm --log-level=warning --external:*.wasm $(WASM_OPT): @$(call color,"downloading binaryen") @@ -60,3 +70,12 @@ $(WASM_OPT): echo '9057c8f3f0bbfec47a95985c8f0faad8cc2aa3932e94a7d6b705e245ed140e19 binaryen.tar.gz' | sha256sum -c tar zxvf binaryen.tar.gz --strip-components=1 --touch -C ../build/binaryen rm binaryen.tar.gz + +export WASI_CC := $(abspath ${WASI_SDK_PATH}/bin/clang) -target wasm32-wasi --sysroot=${WASI_SYSROOT} +export WASI_CFLAGS := -Oz -flto +export WASI_LDFLAGS := -flto -Wl,--allow-undefined + +$(BENCHMARK_DST)/%.wasm: $(WASI_SDK_PATH) $(WASM_OPT) $(BENCHMARK_SRC)/%.c + mkdir -p $(BENCHMARK_DST) + $(WASI_CC) $(WASI_CFLAGS) $(WASI_LDFLAGS) subjects/$*.c -o $(BENCHMARK_DST)/$*.wasm + $(WASM_OPT) -g -O --asyncify $(BENCHMARK_DST)/$*.wasm -o $(BENCHMARK_DST)/$*.asyncify.wasm diff --git a/test/benchmark-build.mjs b/test/benchmark-build.mjs new file mode 100644 index 0000000..80f40fc --- /dev/null +++ b/test/benchmark-build.mjs @@ -0,0 +1,43 @@ +// @ts-check +import * as esbuild from 'esbuild' +import * as path from 'path' + +const OUT_DIR = '../build/test/' + +/** + * Node currently can't import wasm files in the way that we do with Workers, it either throws an + * error if you try to import a wasm file or imports an instantiated wasm instance. Whereas in + * Workers we get a WebAssembly.Module as the default export if we import a wasm file, so this + * plugin is to replicate that behavior in the bundle. + * @type {import('esbuild').Plugin} + */ +const wasmLoaderPlugin = { + name: 'wasm-module-loader', + setup: (build) => { + build.onResolve({ filter: /\.wasm$/ }, (args) => ({ + path: args.path, + namespace: 'wasm-module', + })) + + build.onLoad({ filter: /.*/, namespace: 'wasm-module' }, (args) => ({ + contents: ` + import * as fs from 'fs'; + import * as path from 'path'; + import * as url from 'url'; + + const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + export default new WebAssembly.Module(fs.readFileSync(path.resolve(__dirname, '${args.path}'))); + `, + })) + }, +} + +esbuild.build({ + bundle: true, + outfile: path.join(OUT_DIR, 'standalone.mjs'), + format: 'esm', + logLevel: 'warning', + entryPoints: ['./driver/standalone.ts'], + plugins: [wasmLoaderPlugin], + platform: 'node', +}) diff --git a/test/benchmark.test.ts b/test/benchmark.test.ts new file mode 100644 index 0000000..a9e4056 --- /dev/null +++ b/test/benchmark.test.ts @@ -0,0 +1,53 @@ +import * as child from 'node:child_process' +import * as fs from 'node:fs' +import { cwd } from 'node:process' +import path from 'path/posix' +import type { ExecOptions } from './driver/common' + +const { OUTPUT_DIR } = process.env +const moduleNames = fs + .readdirSync(`${OUTPUT_DIR}/benchmark`) + .map((dirent) => `benchmark/${dirent}`) + +for (const modulePath of moduleNames) { + const prettyName = modulePath.split('/').pop() + if (!prettyName) throw new Error('unreachable') + + test(`${prettyName}`, async () => { + const execOptions: ExecOptions = { + moduleName: prettyName, + asyncify: prettyName.endsWith('.asyncify.wasm'), + fs: {}, + preopens: [], + returnOnExit: false, + } + + // Spawns a child process that runs the wasm so we can isolate the profiling to just that + // specific test case. + const proc = child.execFile( + `node`, + [ + '--experimental-vm-modules', + '--cpu-prof', + '--cpu-prof-dir=./prof', + `--cpu-prof-name=${prettyName}.${Date.now()}.cpuprofile`, + 'standalone.mjs', + modulePath, + JSON.stringify(execOptions), + ], + { + encoding: 'utf8', + cwd: OUTPUT_DIR, + } + ) + + let stderr = '' + proc.stderr?.on('data', (data) => (stderr += data)) + + const exitCode = await new Promise((resolve) => proc.once('exit', resolve)) + + if (exitCode !== 0) { + console.error(`Child process exited with code ${exitCode}:\n${stderr}`) + } + }) +} diff --git a/test/driver/common.ts b/test/driver/common.ts index 80fe9a1..c67699e 100644 --- a/test/driver/common.ts +++ b/test/driver/common.ts @@ -1,4 +1,4 @@ -import { Environment, _FS } from '@cloudflare/workers-wasi' +import { Environment, WASI, _FS } from '@cloudflare/workers-wasi' export interface ExecOptions { args?: string[] @@ -13,5 +13,78 @@ export interface ExecOptions { export interface ExecResult { stdout: string + stderr: string status?: number } + +export const exec = async ( + options: ExecOptions, + wasm: WebAssembly.Module, + body?: ReadableStream +): Promise => { + let TransformStream = global.TransformStream + + if (TransformStream === undefined) { + const streams = await import('node:stream/web').catch(() => { + throw new Error('unreachable') + }) + + TransformStream = streams.TransformStream as typeof TransformStream + } + + const stdout = new TransformStream() + const stderr = new TransformStream() + + const wasi = new WASI({ + args: options.args, + env: options.env, + fs: options.fs, + preopens: options.preopens, + returnOnExit: options.returnOnExit, + stderr: stderr.writable, + stdin: body, + stdout: stdout.writable, + streamStdio: options.asyncify, + }) + const instance = new WebAssembly.Instance(wasm, { + wasi_snapshot_preview1: wasi.wasiImport, + }) + const promise = wasi.start(instance) + + const streams = await Promise.all([ + collectStream(stdout.readable), + collectStream(stderr.readable), + ]) + + try { + const result = { + stdout: streams[0], + stderr: streams[1], + status: await promise, + } + return result + } catch (e: any) { + e.message = `${e}\n\nstdout:\n${streams[0]}\n\nstderr:\n${streams[1]}\n\n` + throw e + } +} + +const collectStream = async (stream: ReadableStream): Promise => { + const chunks: Uint8Array[] = [] + + // @ts-ignore + for await (const chunk of stream) { + chunks.push(chunk) + } + + const size = chunks.reduce((acc, chunk) => acc + chunk.byteLength, 0) + const buffer = new Uint8Array(size) + let offset = 0 + + chunks.forEach((chunk) => { + buffer.set(chunk, offset) + offset += chunk.byteLength + }) + + return new TextDecoder().decode(buffer) +} diff --git a/test/driver/index.ts b/test/driver/index.ts deleted file mode 100644 index 43bd4e2..0000000 --- a/test/driver/index.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { WASI, traceImportsToConsole } from '@cloudflare/workers-wasi' -import { ModuleTable } from '../../build/test/wasm-table' -import { ExecOptions, ExecResult } from './common' - -const exec = async ( - options: ExecOptions, - wasm: WebAssembly.Module, - body?: ReadableStream -): Promise => { - const stdout = new TransformStream() - const stderr = new TransformStream() - - const wasi = new WASI({ - args: options.args, - env: options.env, - fs: options.fs, - preopens: options.preopens, - returnOnExit: options.returnOnExit, - stderr: stderr.writable, - stdin: body, - stdout: stdout.writable, - streamStdio: options.asyncify, - }) - const instance = new WebAssembly.Instance(wasm, { - wasi_snapshot_preview1: wasi.wasiImport, - }) - const promise = wasi.start(instance) - - const streams = await Promise.all([ - new Response(stdout.readable).text(), - new Response(stderr.readable).text(), - ]) - - try { - const result = { - stdout: streams[0], - stderr: streams[1], - status: await promise, - } - return result - } catch (e: any) { - e.message = `${e}\n\nstdout:\n${streams[0]}\n\nstderr:\n${streams[1]}\n\n` - throw e - } -} - -export default { - async fetch(request: Request) { - const options: ExecOptions = JSON.parse( - atob(request.headers.get('EXEC_OPTIONS')!) - ) - - const result = await exec( - options, - ModuleTable[options.moduleName], - request.body ?? undefined - ) - return new Response(JSON.stringify(result)) - }, -} diff --git a/test/driver/standalone.ts b/test/driver/standalone.ts new file mode 100644 index 0000000..e8a1af0 --- /dev/null +++ b/test/driver/standalone.ts @@ -0,0 +1,29 @@ +import * as fs from 'node:fs/promises' +import { ReadableStream } from 'node:stream/web' +import { exec } from './common' + +const [modulePath, rawOptions] = process.argv.slice(2) +const options = JSON.parse(rawOptions) + +const nulls = new Uint8Array(4096).fill(0) +let written = 0 + +const stdinStream = new ReadableStream({ + pull: (controller) => { + if (written > 1_000_000) { + controller.close() + } else { + controller.enqueue(nulls) + written += nulls.byteLength + } + }, +}) + +fs.readFile(modulePath) + .then((wasmBytes) => new WebAssembly.Module(wasmBytes)) + .then((wasmModule) => exec(options, wasmModule, stdinStream as any)) + .then((result) => { + console.log(result.stdout) + console.error(result.stderr) + process.exit(result.status) + }) diff --git a/test/driver/worker.ts b/test/driver/worker.ts new file mode 100644 index 0000000..4aa7703 --- /dev/null +++ b/test/driver/worker.ts @@ -0,0 +1,17 @@ +import { ModuleTable } from '../../build/test/wasm-table' +import { ExecOptions, exec } from './common' + +export default { + async fetch(request: Request) { + const options: ExecOptions = JSON.parse( + atob(request.headers.get('EXEC_OPTIONS')!) + ) + + const result = await exec( + options, + ModuleTable[options.moduleName], + request.body ?? undefined + ) + return new Response(JSON.stringify(result)) + }, +} diff --git a/test/driver/wrangler.toml b/test/driver/wrangler.toml index 8a75bc0..3cc90c1 100644 --- a/test/driver/wrangler.toml +++ b/test/driver/wrangler.toml @@ -17,4 +17,3 @@ dir = "../../build/test/" [[build.upload.rules]] type = "CompiledWasm" globs = ["**/*.wasm"] - diff --git a/test/generate-wasm-table.mjs b/test/generate-wasm-table.mjs index c0eb514..e841592 100644 --- a/test/generate-wasm-table.mjs +++ b/test/generate-wasm-table.mjs @@ -1,7 +1,5 @@ -// generate a table with explicit imports since we can't dynamically load -// WebAssembly modules in our worker -import path from 'path' -import fs from 'fs/promises' +import * as path from 'path' +import * as fs from 'fs/promises' const recurseFiles = async (dir) => { const entries = await fs.readdir(dir, { withFileTypes: true }) @@ -21,19 +19,24 @@ const recurseFiles = async (dir) => { ).flat() } - const dir = path.resolve(process.argv[2]) -const files = (await recurseFiles(dir)).map(f => path.join('./', f.substr(dir.length))) +const files = (await recurseFiles(dir)).map((f) => + path.join('./', f.substr(dir.length)) +) + for (const file of files) { console.log('// @ts-ignore') - const identifier = file.replace(/[-\/\.]/g, "_") + const identifier = file.replace(/[-\/\.]/g, '_') console.log(`import ${identifier} from '${file}'`) } console.log() -console.log('export const ModuleTable: { [key: string]: WebAssembly.Module } = {') +console.log( + 'export const ModuleTable: { [key: string]: WebAssembly.Module } = {' +) for (const file of files) { - const identifier = file.replace(/[-\/\.]/g, "_") + const identifier = file.replace(/[-\/\.]/g, '_') console.log(` '${file}': ${identifier},`) } + console.log('}') diff --git a/test/package-lock.json b/test/package-lock.json index 3044298..4d8ad71 100644 --- a/test/package-lock.json +++ b/test/package-lock.json @@ -11,6 +11,7 @@ "devDependencies": { "@cloudflare/workers-types": "^3.2.0", "@types/jest": "^27.0.2", + "@types/node": "^17.0.16", "esbuild": "^0.13.13", "jest": "^27.3.1", "jest-junit": "^13.0.0", @@ -1111,9 +1112,9 @@ } }, "node_modules/@types/node": { - "version": "16.11.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.7.tgz", - "integrity": "sha512-QB5D2sqfSjCmTuWcBWyJ+/44bcjO7VbjSbOE0ucoVbAsSNQc4Lt6QkgkVXkTDwkL4z/beecZNDvVX15D4P8Jbw==", + "version": "17.0.16", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.16.tgz", + "integrity": "sha512-ydLaGVfQOQ6hI1xK2A5nVh8bl0OGoIfYMxPWHqqYe9bTkWCfqiVvZoh2I/QF2sNSkZzZyROBoTefIEI+PB6iIA==", "dev": true }, "node_modules/@types/prettier": { @@ -6005,9 +6006,9 @@ } }, "@types/node": { - "version": "16.11.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.7.tgz", - "integrity": "sha512-QB5D2sqfSjCmTuWcBWyJ+/44bcjO7VbjSbOE0ucoVbAsSNQc4Lt6QkgkVXkTDwkL4z/beecZNDvVX15D4P8Jbw==", + "version": "17.0.16", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.16.tgz", + "integrity": "sha512-ydLaGVfQOQ6hI1xK2A5nVh8bl0OGoIfYMxPWHqqYe9bTkWCfqiVvZoh2I/QF2sNSkZzZyROBoTefIEI+PB6iIA==", "dev": true }, "@types/prettier": { diff --git a/test/package.json b/test/package.json index 9a4d85e..e590d0b 100644 --- a/test/package.json +++ b/test/package.json @@ -4,6 +4,7 @@ "devDependencies": { "@cloudflare/workers-types": "^3.2.0", "@types/jest": "^27.0.2", + "@types/node": "^17.0.16", "esbuild": "^0.13.13", "jest": "^27.3.1", "jest-junit": "^13.0.0", @@ -19,7 +20,10 @@ "extensionsToTreatAsEsm": [ ".ts" ], - "reporters": [ "default", "jest-junit" ], + "reporters": [ + "default", + "jest-junit" + ], "verbose": true, "testRegex": "/.*\\.test\\.ts$", "transform": { diff --git a/test/subjects/read.c b/test/subjects/read.c new file mode 100644 index 0000000..ff966d5 --- /dev/null +++ b/test/subjects/read.c @@ -0,0 +1,15 @@ +#include "assert.h" +#include "stdint.h" +#include "stdio.h" +#include "string.h" + +#define CHUNK_SIZE 4096 +#define ITERATIONS 1024 + +int main() { + const char *chunk_buf[CHUNK_SIZE] = {0}; + + for (int iterations = 0; iterations < ITERATIONS; iterations++) { + fread(chunk_buf, 1, CHUNK_SIZE, stdin); + } +} diff --git a/test/subjects/write.c b/test/subjects/write.c new file mode 100644 index 0000000..3d69152 --- /dev/null +++ b/test/subjects/write.c @@ -0,0 +1,17 @@ +#include "assert.h" +#include "stdint.h" +#include "stdio.h" +#include "string.h" + +#define CHUNK_SIZE 4096 +#define ITERATIONS 1024 + +int main() { + const char *chunk_buf[CHUNK_SIZE] = {0}; + + for (int iterations = 0; iterations < ITERATIONS; iterations++) { + fwrite(chunk_buf, 1, CHUNK_SIZE, stdout); + } + + fflush(stdout); +} diff --git a/test/tsconfig.json b/test/tsconfig.json index ecc1dcb..9ecaf94 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -2,17 +2,10 @@ "extends": "../tsconfig.json", "compilerOptions": { "noEmit": true, - "emitDeclarationOnly":false, + "emitDeclarationOnly": false, "skipLibCheck": true, "lib": ["ESNext", "DOM"], - "types": [ - "@cloudflare/workers-types", - "@types/node", - "@types/jest" - ] + "types": ["@cloudflare/workers-types", "@types/node", "@types/jest"] }, - "include": [ - "**/*.ts" - ] + "include": ["**/*.ts"] } -