chore: add transports benchmark (#521)

Adds a benchmark that measures how long it takes to transfer 100M-1G of data between node, firefox and chrome using WebRTC, WebSockets, WebTransport and TCP. --------- Co-authored-by: Russell Dempsey <[email protected]>
ipfs · May 1, 2024 · 55b9650 · 55b9650
1 parent 251414d
commit 55b9650
Show file tree

Hide file tree

Showing 17 changed files with 890 additions and 0 deletions.
diff --git a/benchmarks/transports/README.md b/benchmarks/transports/README.md
@@ -0,0 +1,59 @@
+# Transport Benchmark
+
+Benchmarks Helia transport performance against each other
+
+To run:
+
+1. Add `benchmarks/*` to the `workspaces` entry in the root `package.json` of this repo
+2. Run
+    ```console
+    $ npm run reset
+    $ npm i
+    $ npm run build
+    $ cd benchmarks/transports
+    $ npm start
+
+    > [email protected] start
+    > npm run build && node dist/src/index.js
+
+
+    > [email protected] build
+    > aegir build --bundle false
+
+    [14:51:28] tsc [started]
+    [14:51:33] tsc [completed]
+
+    Implementation, 105 MB, 210 MB, 315 MB, 419 MB, 524 MB, 629 MB, 734 MB, 839 MB, 944 MB, 1.05 GB
+    TCP (node.js -> node.js) filecoin defaults, 775, 1763, 2104, 3254, 3881, 4384, 5904, 5161, 6382, 6856
+    WebSockets (node.js -> node.js) filecoin defaults, 1068, 1642, 2092, 2812, 4117, 4423, 6117, 7820, 7182, 7816
+    //... results here
+    ```
+3. Graph the CSV data with your favourite graphing tool
+
+## Debugging
+
+To get debug output, run with the `DEBUG` env var set to `test*` to see all output, `recipient*` to just see the recipient's log, `sender=*` to see the sender's log, etc.
+
+Eg.
+
+```console
+$ DEBUG=test* npm start
+```
+
+or
+
+```console
+$ DEBUG='test*,sender*' npm start
+```
+
+## Results
+
+Recently generated graph:
+
+- Lower numbers are better
+- The legend arrow indicates direction of transfer
+    - e.g. `helia -> kubo` is the equivalent of
+      1. `ipfs.add` executed on Helia
+      2. `ipfs.pin` executed on Kubo which pulls the data from Helia
+
+<img width="1042" alt="image" src="https://github.com/ipfs/helia/assets/665810/d0d16ed0-d764-42ee-be73-ac7bbb938103">
diff --git a/benchmarks/transports/package.json b/benchmarks/transports/package.json
@@ -0,0 +1,56 @@
+{
+  "name": "benchmarks-transports",
+  "version": "1.0.0",
+  "main": "index.js",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "clean": "aegir clean",
+    "build": "aegir build --bundle false",
+    "lint": "aegir lint",
+    "dep-check": "aegir dep-check -i playwright-test",
+    "start": "npm run build && node dist/src/index.js"
+  },
+  "dependencies": {
+    "@chainsafe/libp2p-noise": "^15.0.0",
+    "@chainsafe/libp2p-yamux": "^6.0.2",
+    "@helia/block-brokers": "^2.1.1",
+    "@helia/routers": "^1.0.3",
+    "@helia/unixfs": "^3.0.3",
+    "@ipld/dag-pb": "^4.1.0",
+    "@libp2p/circuit-relay-v2": "^1.0.21",
+    "@libp2p/identify": "^1.0.20",
+    "@libp2p/interface": "^1.3.0",
+    "@libp2p/logger": "^4.0.11",
+    "@libp2p/tcp": "^9.0.23",
+    "@libp2p/webrtc": "^4.0.29",
+    "@libp2p/websockets": "^8.0.19",
+    "@libp2p/webtransport": "^4.0.29",
+    "@multiformats/multiaddr": "^12.2.1",
+    "aegir": "^42.2.5",
+    "blockstore-fs": "^1.1.10",
+    "blockstore-idb": "^1.1.8",
+    "datastore-idb": "^2.1.9",
+    "datastore-level": "^10.1.8",
+    "debug": "^4.3.4",
+    "execa": "^8.0.1",
+    "helia": "^4.1.0",
+    "interface-blockstore": "^5.2.10",
+    "interface-datastore": "^8.2.11",
+    "ipfs-unixfs-importer": "^15.1.1",
+    "ipfsd-ctl": "^14.1.0",
+    "it-buffer-stream": "^3.0.2",
+    "it-drain": "^3.0.7",
+    "kubo": "^0.28.0",
+    "kubo-rpc-client": "^4.1.1",
+    "libp2p": "^1.4.0",
+    "multiformats": "^13.1.0",
+    "playwright-test": "^14.1.1",
+    "pretty-bytes": "^6.1.0",
+    "uint8arrays": "^5.0.3"
+  },
+  "browser": {
+    "./dist/src/runner/helia/stores.js": "./dist/src/runner/helia/stores.browser.js",
+    "./dist/src/runner/helia/transports.js": "./dist/src/runner/helia/transports.browser.js"
+  }
+}
diff --git a/benchmarks/transports/src/index.ts b/benchmarks/transports/src/index.ts
@@ -0,0 +1,102 @@
+/* eslint-disable no-console */
+
+import { createServer } from 'ipfsd-ctl'
+import { path as kuboPath } from 'kubo'
+import { create as kuboRpcClient } from 'kubo-rpc-client'
+import prettyBytes from 'pretty-bytes'
+import { createRelay } from './relay.js'
+import { Test } from './test.js'
+import { createTests } from './tests.js'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import type { CID } from 'multiformats/cid'
+
+const ONE_MEG = 1024 * 1024
+const relay = await createRelay()
+
+export interface TransferBenchmark {
+  teardown(): Promise<void>
+  addrs(): Promise<Multiaddr[]>
+  dial(multiaddrs: Multiaddr[]): Promise<void>
+  add(content: AsyncIterable<Uint8Array>, options: ImportOptions): Promise<CID>
+  get(cid: CID): Promise<void>
+}
+
+export interface ImportOptions {
+  cidVersion?: 0 | 1
+  rawLeaves?: boolean
+  chunkSize?: number
+  maxChildrenPerNode?: number
+}
+
+export interface File {
+  name: string
+  options: ImportOptions
+  size: number
+}
+
+const opts: Record<string, ImportOptions> = {
+  'filecoin defaults': {
+    chunkSize: 1024 * 1024,
+    rawLeaves: true,
+    cidVersion: 1,
+    maxChildrenPerNode: 1024
+  }
+}
+
+const tests: Record<string, File[]> = {}
+
+for (const [name, options] of Object.entries(opts)) {
+  tests[name] = []
+
+  for (let i = 100; i < 1100; i += 100) {
+    tests[name].push({
+      name: `${i}`,
+      options,
+      size: ONE_MEG * i
+    })
+  }
+}
+
+console.info(
+  'Implementation,',
+  tests[Object.keys(opts)[0]]
+    .map(file => prettyBytes(ONE_MEG * Number(file.name)))
+    .join(', ')
+)
+
+const server = createServer(29834, {
+  type: 'kubo',
+  test: true,
+  bin: kuboPath(),
+  rpc: kuboRpcClient,
+  init: {
+    emptyRepo: true
+  }
+})
+
+async function main (): Promise<void> {
+  const impls = createTests(relay.libp2p.getMultiaddrs()[0]).map(test => {
+    return new Test(test)
+  })
+
+  for (const [name, files] of Object.entries(tests)) {
+    for (const impl of impls) {
+      process.stdout.write(`${impl.name} ${name}`)
+
+      for (const file of files) {
+        const time = await impl.runTest(file)
+        process.stdout.write(`, ${time}`)
+        await server.clean()
+      }
+
+      process.stdout.write('\n')
+    }
+  }
+
+  await relay.stop()
+}
+
+main().catch(err => {
+  console.error(err) // eslint-disable-line no-console
+  process.exit(1)
+})
diff --git a/benchmarks/transports/src/relay.ts b/benchmarks/transports/src/relay.ts
@@ -0,0 +1,51 @@
+import { noise } from '@chainsafe/libp2p-noise'
+import { yamux } from '@chainsafe/libp2p-yamux'
+import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
+import { identify } from '@libp2p/identify'
+import { prefixLogger } from '@libp2p/logger'
+import { webSockets } from '@libp2p/websockets'
+import { createHelia, type HeliaLibp2p } from 'helia'
+import { createLibp2p } from 'libp2p'
+import type { Libp2p } from '@libp2p/interface'
+
+export async function createRelay (): Promise<HeliaLibp2p<Libp2p<any>>> {
+  const logger = prefixLogger('relay')
+
+  return createHelia({
+    logger,
+    blockBrokers: [],
+    routers: [],
+    libp2p: await createLibp2p({
+      logger,
+      addresses: {
+        listen: [
+          '/ip4/127.0.0.1/tcp/0/ws'
+        ]
+      },
+      transports: [
+        webSockets()
+      ],
+      connectionEncryption: [
+        noise()
+      ],
+      streamMuxers: [
+        yamux()
+      ],
+      services: {
+        identify: identify(),
+        relay: circuitRelayServer({
+          reservations: {
+            maxReservations: Infinity,
+            applyDefaultLimit: false
+          }
+        })
+      },
+      connectionManager: {
+        minConnections: 0
+      },
+      connectionGater: {
+        denyDialMultiaddr: async () => false
+      }
+    })
+  })
+}
diff --git a/benchmarks/transports/src/runner/helia/get-helia.ts b/benchmarks/transports/src/runner/helia/get-helia.ts
@@ -0,0 +1,54 @@
+import { noise } from '@chainsafe/libp2p-noise'
+import { yamux } from '@chainsafe/libp2p-yamux'
+import { bitswap } from '@helia/block-brokers'
+import { libp2pRouting } from '@helia/routers'
+import { identify } from '@libp2p/identify'
+import { prefixLogger } from '@libp2p/logger'
+import { createHelia, type HeliaLibp2p } from 'helia'
+import { createLibp2p } from 'libp2p'
+import { getStores } from './stores.js'
+import { getTransports } from './transports.js'
+import type { Libp2p } from '@libp2p/interface'
+
+export async function getHelia (): Promise<HeliaLibp2p<Libp2p<any>>> {
+  const listen = `${process.env.HELIA_LISTEN ?? ''}`.split(',').filter(Boolean)
+  const { datastore, blockstore } = await getStores()
+  const logger = prefixLogger(`${process.env.HELIA_TYPE}`)
+
+  const libp2p = await createLibp2p({
+    logger,
+    addresses: {
+      listen
+    },
+    transports: getTransports(),
+    connectionEncryption: [
+      noise()
+    ],
+    streamMuxers: [
+      yamux()
+    ],
+    services: {
+      identify: identify()
+    },
+    connectionManager: {
+      minConnections: 0
+    },
+    connectionGater: {
+      denyDialMultiaddr: async () => false
+    },
+    datastore
+  })
+
+  return createHelia({
+    logger,
+    blockstore,
+    datastore,
+    blockBrokers: [
+      bitswap()
+    ],
+    routers: [
+      libp2pRouting(libp2p)
+    ],
+    libp2p
+  })
+}
diff --git a/benchmarks/transports/src/runner/helia/recipient.ts b/benchmarks/transports/src/runner/helia/recipient.ts
@@ -0,0 +1,34 @@
+/* eslint-disable no-console */
+
+import { unixfs } from '@helia/unixfs'
+import { multiaddr } from '@multiformats/multiaddr'
+import drain from 'it-drain'
+import { CID } from 'multiformats'
+import { getHelia } from './get-helia.js'
+
+process.title = `helia transport benchmark ${process.env.HELIA_TYPE}`
+
+const cid = CID.parse(`${process.env.HELIA_CID}`)
+const mas = `${process.env.HELIA_MULTIADDRS}`.split(',').map(str => multiaddr(str))
+const signal = AbortSignal.timeout(parseInt(process.env.HELIA_TIMEOUT ?? '60000'))
+
+const helia = await getHelia()
+
+try {
+  await helia.libp2p.dial(mas, {
+    signal
+  })
+
+  const fs = unixfs(helia)
+  const start = Date.now()
+
+  await drain(fs.cat(cid, {
+    signal
+  }))
+
+  console.info(`TEST-OUTPUT:${Date.now() - start}`)
+} catch {
+  console.info('TEST-OUTPUT:?')
+}
+
+console.info('TEST-OUTPUT:done')
diff --git a/benchmarks/transports/src/runner/helia/sender.ts b/benchmarks/transports/src/runner/helia/sender.ts
@@ -0,0 +1,33 @@
+/* eslint-disable no-console */
+
+import { unixfs } from '@helia/unixfs'
+import { fixedSize } from 'ipfs-unixfs-importer/chunker'
+import { balanced } from 'ipfs-unixfs-importer/layout'
+import bufferStream from 'it-buffer-stream'
+import { getHelia } from './get-helia.js'
+
+process.title = `helia transport benchmark ${process.env.HELIA_TYPE}`
+
+interface ImportOptions {
+  cidVersion?: 0 | 1
+  rawLeaves?: boolean
+  chunkSize?: number
+  maxChildrenPerNode?: number
+}
+
+const options: ImportOptions = JSON.parse(`${process.env.HELIA_IMPORT_OPTIONS}`)
+const size = Number(`${process.env.HELIA_FILE_SIZE}`)
+
+const helia = await getHelia()
+const fs = unixfs(helia)
+
+const cid = await fs.addByteStream(bufferStream(size), {
+  ...options,
+  chunker: options.chunkSize != null ? fixedSize({ chunkSize: options.chunkSize }) : undefined,
+  layout: options.maxChildrenPerNode != null ? balanced({ maxChildrenPerNode: options.maxChildrenPerNode }) : undefined
+})
+
+console.info(`TEST-OUTPUT:${JSON.stringify({
+  cid: cid.toString(),
+  multiaddrs: helia.libp2p.getMultiaddrs().map(ma => ma.toString()).join(',')
+})}`)