diff --git a/README.md b/README.md index 6204f84bc9..41529bc6c5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Code Coverage][coverage-badge]][coverage-link] [![Discord][discord-badge]][discord-link] -This was originally the EthereumJS VM repository. On Q1 2020 we brought some of its building blocks together to simplify development. Below you can find the packages included in this repository. +This was originally the EthereumJS VM repository. In Q1 2020 we brought some of its building blocks together to simplify development. Below you can find the packages included in this repository. 🚧 Please note that the `master` branch is updated on a daily basis, and to inspect code related to a specific package version, refer to the [tags](https://github.com/ethereumjs/ethereumjs-monorepo/tags). diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index 9e1ed19e18..d910d85001 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -9,21 +9,23 @@ import { Server as RPCServer } from 'jayson/promise' import Common, { Chain, Hardfork } from '@ethereumjs/common' import { _getInitializedChains } from '@ethereumjs/common/dist/chains' import { Address, toBuffer } from 'ethereumjs-util' -import { version as packageVersion } from '../package.json' import { parseMultiaddrs, parseGenesisState, parseCustomParams, inspectParams } from '../lib/util' import EthereumClient from '../lib/client' import { Config, DataDirectory } from '../lib/config' import { Logger, getLogger } from '../lib/logging' import { RPCManager } from '../lib/rpc' import * as modules from '../lib/rpc/modules' -import { Event } from '../lib/types' import type { Chain as IChain, GenesisState } from '@ethereumjs/common/dist/types' const level = require('level') const yargs = require('yargs/yargs') const { hideBin } = require('yargs/helpers') +type Account = [address: Address, privateKey: Buffer] + const networks = Object.entries(_getInitializedChains().names) +let logger: Logger + const args = yargs(hideBin(process.argv)) .option('network', { describe: 'Network', @@ -35,11 +37,6 @@ const args = yargs(hideBin(process.argv)) choices: networks.map((n) => parseInt(n[0])), default: undefined, }) - .option('network-id', { - describe: `Network ID`, - choices: networks.map((n) => parseInt(n[0])), - default: undefined, - }) .option('syncmode', { describe: 'Blockchain sync mode (light sync experimental)', choices: ['light', 'full'], @@ -206,6 +203,7 @@ const args = yargs(hideBin(process.argv)) .option('unlock', { describe: 'Comma separated list of accounts to unlock - currently only the first account is used (for sealing PoA blocks and as the default coinbase). Beta, you will be promped for a 0x-prefixed private key until keystore functionality is added - FOR YOUR SAFETY PLEASE DO NOT USE ANY ACCOUNTS HOLDING SUBSTANTIAL AMOUNTS OF ETH', + string: true, array: true, }) .option('dev', { @@ -229,13 +227,10 @@ const args = yargs(hideBin(process.argv)) default: 2350000, }).argv -let logger: Logger - /** * Initializes and returns the databases needed for the client - * @param config */ -function initDBs(config: Config, args: any) { +function initDBs(config: Config) { // Chain DB const chainDataDir = config.getDataDirectory(DataDirectory.Chain) ensureDirSync(chainDataDir) @@ -258,58 +253,55 @@ function initDBs(config: Config, args: any) { } /** - * Starts the client and reacts on the main lifecycle events - * @param config + * Special block execution debug mode (does not change any state) + */ +async function executeBlocks(client: EthereumClient) { + let first = 0 + let last = 0 + let txHashes = [] + try { + const blockRange = (args.executeBlocks as string).split('-').map((val) => { + const reNum = /([0-9]+)/.exec(val) + const num = reNum ? parseInt(reNum[1]) : 0 + const reTxs = /[0-9]+\[(.*)\]/.exec(val) + const txs = reTxs ? reTxs[1].split(',') : [] + return [num, txs] + }) + first = blockRange[0][0] as number + last = blockRange.length === 2 ? (blockRange[1][0] as number) : first + txHashes = blockRange[0][1] as string[] + + if ((blockRange[0][1] as string[]).length > 0 && blockRange.length === 2) { + throw new Error('wrong input') + } + } catch (e: any) { + client.config.logger.error( + 'Wrong input format for block execution, allowed format types: 5, 5-10, 5[0xba4b5fd92a26badad3cad22eb6f7c7e745053739b5f5d1e8a3afb00f8fb2a280,[TX_HASH_2],...], 5[*] (all txs in verbose mode)' + ) + process.exit() + } + await client.executeBlocks(first, last, txHashes) +} + +/** + * Starts and returns the {@link EthereumClient} */ -async function runNode(config: Config) { - config.logger.info( - `Initializing Ethereumjs client version=v${packageVersion} network=${config.chainCommon.chainName()}` - ) +async function startClient(config: Config) { config.logger.info(`Data directory: ${config.datadir}`) if (config.lightserv) { config.logger.info(`Serving light peer requests`) } - const dbs = initDBs(config, args) + + const dbs = initDBs(config) const client = new EthereumClient({ config, ...dbs, }) - client.config.events.on(Event.SERVER_ERROR, (err) => config.logger.error(err)) - client.config.events.on(Event.SERVER_LISTENING, (details) => { - config.logger.info(`Listener up transport=${details.transport} url=${details.url}`) - }) - config.events.on(Event.SYNC_SYNCHRONIZED, (height) => { - client.config.logger.info(`Synchronized blockchain at height ${height}`) - }) await client.open() if (args.executeBlocks) { - // Special block execution debug mode (not changing any state) - let first = 0 - let last = 0 - let txHashes = [] - try { - const blockRange = (args.executeBlocks as string).split('-').map((val) => { - const reNum = /([0-9]+)/.exec(val) - const num = reNum ? parseInt(reNum[1]) : 0 - const reTxs = /[0-9]+\[(.*)\]/.exec(val) - const txs = reTxs ? reTxs[1].split(',') : [] - return [num, txs] - }) - first = blockRange[0][0] as number - last = blockRange.length === 2 ? (blockRange[1][0] as number) : first - txHashes = blockRange[0][1] as string[] - - if ((blockRange[0][1] as string[]).length > 0 && blockRange.length === 2) { - throw new Error('wrong input') - } - } catch (e: any) { - client.config.logger.error( - 'Wrong input format for block execution, allowed format types: 5, 5-10, 5[0xba4b5fd92a26badad3cad22eb6f7c7e745053739b5f5d1e8a3afb00f8fb2a280,[TX_HASH_2],...], 5[*] (all txs in verbose mode)' - ) - process.exit() - } - await client.executeBlocks(first, last, txHashes) + // Special block execution debug mode (does not change any state) + await executeBlocks(client) } else { // Regular client start await client.start() @@ -318,9 +310,10 @@ async function runNode(config: Config) { } /** - * Returns enabled RPCServers + * Starts and returns enabled RPCServers */ -function runRpcServers(client: EthereumClient, config: Config, args: any) { +function startRPCServers(client: EthereumClient) { + const config = client.config const onRequest = (request: any) => { let msg = '' if (args.rpcDebug) { @@ -409,82 +402,171 @@ function runRpcServers(client: EthereumClient, config: Config, args: any) { } /** - * Main entry point to start a client + * Returns a configured common for devnet with a prefunded address */ -async function run() { - if (args.helprpc) { - // Display RPC help and exit - console.log('-'.repeat(27)) - console.log('JSON-RPC: Supported Methods') - console.log('-'.repeat(27)) - console.log() - for (const modName of modules.list) { - console.log(`${modName}:`) - const methods = RPCManager.getMethodNames((modules as any)[modName]) - for (const methodName of methods) { - console.log(`-> ${modName.toLowerCase()}_${methodName}`) - } - console.log() - } - console.log() - process.exit() +async function setupDevnet(prefundAddress: Address) { + const addr = prefundAddress.toString().slice(2) + const consensusConfig = + args.dev === 'pow' + ? { ethash: true } + : { + clique: { + period: 10, + epoch: 30000, + }, + } + const defaultChainData = { + config: { + chainId: 123456, + homesteadBlock: 0, + eip150Block: 0, + eip150Hash: '0x0000000000000000000000000000000000000000000000000000000000000000', + eip155Block: 0, + eip158Block: 0, + byzantiumBlock: 0, + constantinopleBlock: 0, + petersburgBlock: 0, + istanbulBlock: 0, + berlinBlock: 0, + londonBlock: 0, + ...consensusConfig, + }, + nonce: '0x0', + timestamp: '0x614b3731', + gasLimit: '0x47b760', + difficulty: '0x1', + mixHash: '0x0000000000000000000000000000000000000000000000000000000000000000', + coinbase: '0x0000000000000000000000000000000000000000', + number: '0x0', + gasUsed: '0x0', + parentHash: '0x0000000000000000000000000000000000000000000000000000000000000000', + baseFeePerGas: 7, + } + const extraData = '0x' + '0'.repeat(64) + addr + '0'.repeat(130) + const chainData = { + ...defaultChainData, + extraData, + alloc: { [addr]: { balance: '0x10000000000000000000' } }, } + const chainParams = await parseCustomParams(chainData, 'devnet') + const genesisState = await parseGenesisState(chainData) + const customChainParams: [IChain, GenesisState][] = [[chainParams, genesisState]] + return new Common({ + chain: 'devnet', + customChains: customChainParams, + hardfork: Hardfork.London, + }) +} - // give network id precedence over network name - const chain = args.networkId ?? args.network ?? Chain.Mainnet +/** + * Accept account input from command line + */ +async function inputAccounts() { + const accounts: Account[] = [] + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }) + + // Hide key input + ;(rl as any).input.on('keypress', function () { + // get the number of characters entered so far: + const len = (rl as any).line.length + // move cursor back to the beginning of the input: + readline.moveCursor((rl as any).output, -len, 0) + // clear everything to the right of the cursor: + readline.clearLine((rl as any).output, 1) + // replace the original input with asterisks: + for (let i = 0; i < len; i++) { + // eslint-disable-next-line no-extra-semi + ;(rl as any).output.write('*') + } + }) - // configure accounts for mining and prefunding in a local devnet - const accounts: [address: Address, privateKey: Buffer][] = [] - if (args.unlock) { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, + const question = (text: string) => { + return new Promise((resolve) => { + rl.question(text, resolve) }) + } - // Hide key input - ;(rl as any).input.on('keypress', function () { - // get the number of characters entered so far: - const len = (rl as any).line.length - // move cursor back to the beginning of the input: - readline.moveCursor((rl as any).output, -len, 0) - // clear everything to the right of the cursor: - readline.clearLine((rl as any).output, 1) - // replace the original input with asterisks: - for (let i = 0; i < len; i++) { - // eslint-disable-next-line no-extra-semi - ;(rl as any).output.write('*') + try { + for (const addressString of args.unlock) { + const address = Address.fromString(addressString) + const inputKey = await question( + `Please enter the 0x-prefixed private key to unlock ${address}:\n` + ) + ;(rl as any).history = (rl as any).history.slice(1) + const privKey = toBuffer(inputKey) + const derivedAddress = Address.fromPrivateKey(privKey) + if (address.equals(derivedAddress)) { + accounts.push([address, privKey]) + } else { + console.error( + `Private key does not match for ${address} (address derived: ${derivedAddress})` + ) + process.exit() } - }) - - const question = (text: string) => { - return new Promise((resolve) => { - rl.question(text, resolve) - }) } + } catch (e: any) { + console.error(`Encountered error unlocking account:\n${e.message}`) + process.exit() + } + rl.close() + return accounts +} - try { - for (const addressString of args.unlock) { - const address = Address.fromString(addressString) - const inputKey = await question( - `Please enter the 0x-prefixed private key to unlock ${address}:\n` - ) - ;(rl as any).history = (rl as any).history.slice(1) - const privKey = toBuffer(inputKey) - const derivedAddress = Address.fromPrivateKey(privKey) - if (address.equals(derivedAddress)) { - accounts.push([address, privKey]) - } else { - console.error( - `Private key does not match for ${address} (address derived: ${derivedAddress})` - ) - process.exit() - } - } - } catch (e: any) { - console.error(`Encountered error unlocking account:\n${e.message}`) - process.exit() +/** + * Output RPC help and exit + */ +function helprpc() { + console.log('-'.repeat(27)) + console.log('JSON-RPC: Supported Methods') + console.log('-'.repeat(27)) + console.log() + for (const modName of modules.list) { + console.log(`${modName}:`) + const methods = RPCManager.getMethodNames((modules as any)[modName]) + for (const methodName of methods) { + console.log(`-> ${modName.toLowerCase()}_${methodName}`) } - rl.close() + console.log() + } + console.log() + process.exit() +} + +/** + * Returns a randomly generated account + */ +function generateAccount(): Account { + const privKey = randomBytes(32) + const address = Address.fromPrivateKey(privKey) + console.log('='.repeat(50)) + console.log('Account generated for mining blocks:') + console.log(`Address: ${address}`) + console.log(`Private key: 0x${privKey.toString('hex')}`) + console.log('WARNING: Do not use this account for mainnet funds') + console.log('='.repeat(50)) + return [address, privKey] +} + +/** + * Main entry point to start a client + */ +async function run() { + if (args.helprpc) { + // Output RPC help and exit + return helprpc() + } + + // Give network id precedence over network name + const chain = args.networkId ?? args.network ?? Chain.Mainnet + + // Configure accounts for mining and prefunding in a local devnet + const accounts: Account[] = [] + if (args.unlock) { + accounts.push(...(await inputAccounts())) } let common = new Common({ chain, hardfork: Hardfork.Chainstart }) @@ -495,71 +577,13 @@ async function run() { // If generating new keys delete old chain data to prevent genesis block mismatch removeSync(`${args.datadir}/devnet`) // Create new account - const privKey = randomBytes(32) - const address = Address.fromPrivateKey(privKey) - accounts.push([address, privKey]) - console.log('='.repeat(50)) - console.log('Account generated for mining blocks:') - console.log(`Address: ${address}`) - console.log(`Private key: 0x${privKey.toString('hex')}`) - console.log('WARNING: Do not use this account for mainnet funds') - console.log('='.repeat(50)) - } - - const prefundAddress = accounts[0][0].toString().slice(2) - const consensusConfig = - args.dev === 'pow' - ? { ethash: true } - : { - clique: { - period: 10, - epoch: 30000, - }, - } - const defaultChainData = { - config: { - chainId: 123456, - homesteadBlock: 0, - eip150Block: 0, - eip150Hash: '0x0000000000000000000000000000000000000000000000000000000000000000', - eip155Block: 0, - eip158Block: 0, - byzantiumBlock: 0, - constantinopleBlock: 0, - petersburgBlock: 0, - istanbulBlock: 0, - berlinBlock: 0, - londonBlock: 0, - ...consensusConfig, - }, - nonce: '0x0', - timestamp: '0x614b3731', - gasLimit: '0x47b760', - difficulty: '0x1', - mixHash: '0x0000000000000000000000000000000000000000000000000000000000000000', - coinbase: '0x0000000000000000000000000000000000000000', - number: '0x0', - gasUsed: '0x0', - parentHash: '0x0000000000000000000000000000000000000000000000000000000000000000', - baseFeePerGas: 7, - } - const extraData = '0x' + '0'.repeat(64) + prefundAddress + '0'.repeat(130) - const chainData = { - ...defaultChainData, - extraData, - alloc: { [prefundAddress]: { balance: '0x10000000000000000000' } }, + accounts.push(generateAccount()) } - const chainParams = await parseCustomParams(chainData, 'devnet') - const genesisState = await parseGenesisState(chainData) - const customChainParams: [IChain, GenesisState][] = [[chainParams, genesisState]] - common = new Common({ - chain: 'devnet', - customChains: customChainParams, - hardfork: Hardfork.London, - }) + const prefundAddress = accounts[0][0] + common = await setupDevnet(prefundAddress) } - // configure common based on args given + // Configure common based on args given if ( (args.customChainParams || args.customGenesisState || args.gethGenesis) && (!(args.network === 'mainnet') || args.networkId) @@ -608,36 +632,38 @@ async function run() { ensureDirSync(configDirectory) const key = await Config.getClientKey(datadir, common) logger = getLogger(args) + const bootnodes = args.bootnodes ? parseMultiaddrs(args.bootnodes) : undefined + const multiaddrs = args.multiaddrs ? parseMultiaddrs(args.multiaddrs) : undefined const config = new Config({ + accounts, + bootnodes, common, - syncmode: args.syncmode, - lightserv: args.lightserv, datadir, - key, - transports: args.transports, - bootnodes: args.bootnodes ? parseMultiaddrs(args.bootnodes) : undefined, - port: args.port, - extIP: args.extIP, - multiaddrs: args.multiaddrs ? parseMultiaddrs(args.multiaddrs) : undefined, - logger, - saveReceipts: args.saveReceipts, - txLookupLimit: args.txLookupLimit, - maxPerRequest: args.maxPerRequest, - minPeers: args.minPeers, - maxPeers: args.maxPeers, - dnsAddr: args.dnsAddr, - dnsNetworks: args.dnsNetworks, debugCode: args.debugCode, discDns: args.discDns, discV4: args.discV4, + dnsAddr: args.dnsAddr, + dnsNetworks: args.dnsNetworks, + extIP: args.extIP, + key, + lightserv: args.lightserv, + logger, + maxPeers: args.maxPeers, + maxPerRequest: args.maxPerRequest, mine: args.mine || args.dev, - accounts, minerCoinbase: args.minerCoinbase, + minPeers: args.minPeers, + multiaddrs, + port: args.port, + saveReceipts: args.saveReceipts, + syncmode: args.syncmode, + transports: args.transports, + txLookupLimit: args.txLookupLimit, }) config.events.setMaxListeners(50) - const client = await runNode(config) - const servers = args.rpc || args.rpcEngine ? runRpcServers(client, config, args) : [] + const client = await startClient(config) + const servers = args.rpc || args.rpcEngine ? startRPCServers(client) : [] process.on('SIGINT', async () => { config.logger.info('Caught interrupt signal. Shutting down...') diff --git a/packages/client/browser/index.ts b/packages/client/browser/index.ts index a170b7ac5c..9cc41dc377 100644 --- a/packages/client/browser/index.ts +++ b/packages/client/browser/index.ts @@ -1,4 +1,5 @@ import Common, { Chain } from '@ethereumjs/common' +const level = require('level') // Blockchain export * from '../lib/blockchain/chain' @@ -20,7 +21,6 @@ export * from '../lib/net/protocol/flowcontrol' // Server export * from '../lib/net/server/server' export * from '../lib/net/server/libp2pserver' -import { Libp2pServer } from '../lib/net/server/libp2pserver' // EthereumClient export * from '../lib/client' @@ -38,42 +38,40 @@ export * from '../lib/sync/lightsync' // Utilities export * from '../lib/util' +import { parseMultiaddrs } from '../lib/util' import { Config } from '../lib/config' -import { Event } from '../lib/types' // Logging export * from './logging' import { getLogger } from './logging' export async function createClient(args: any) { - const logger = getLogger({ loglevel: args.loglevel ?? 'info' }) + const logger = getLogger({ loglevel: args.loglevel }) const datadir = args.datadir ?? Config.DATADIR_DEFAULT const common = new Common({ chain: args.network ?? Chain.Mainnet }) const key = await Config.getClientKey(datadir, common) + const bootnodes = args.bootnodes ? parseMultiaddrs(args.bootnodes) : undefined const config = new Config({ common, key, - servers: [new Libp2pServer({ multiaddrs: [], config: new Config({ key, logger }), ...args })], - syncmode: args.syncmode ?? 'full', + transports: ['libp2p'], + syncmode: args.syncmode, + bootnodes, + multiaddrs: [], logger, + maxPerRequest: args.maxPerRequest, + minPeers: args.minPeers, + maxPeers: args.maxPeers, + discDns: false, }) - return new EthereumClient({ config }) + config.events.setMaxListeners(50) + const chainDB = level(`${datadir}/${common.chainName()}`) + return new EthereumClient({ config, chainDB }) } export async function run(args: any) { const client = await createClient(args) - const { logger, chainCommon: common } = client.config - logger.info('Initializing Ethereumjs client...') - logger.info(`Connecting to network: ${common.chainName()}`) - client.config.events.on(Event.SERVER_ERROR, (err) => logger.error(err)) - client.config.events.on(Event.SERVER_LISTENING, (details) => { - logger.info(`Listener up transport=${details.transport} url=${details.url}`) - }) - client.config.events.on(Event.SYNC_SYNCHRONIZED, (height) => { - logger.info(`Synchronized blockchain at height ${height.toNumber}`) - }) await client.open() - logger.info('Synchronizing blockchain...') await client.start() return client } diff --git a/packages/client/browser/libp2pnode.ts b/packages/client/browser/libp2pnode.ts index b17f69239a..91763da633 100644 --- a/packages/client/browser/libp2pnode.ts +++ b/packages/client/browser/libp2pnode.ts @@ -3,16 +3,14 @@ * @memberof module:net/peer */ -import LibP2p from 'libp2p' import { NOISE } from '@chainsafe/libp2p-noise' +import LibP2p from 'libp2p' import PeerId from 'peer-id' import { Multiaddr } from 'multiaddr' -// types currently unavailable for below libp2p deps, -// tracking issue: https://github.com/libp2p/js-libp2p/issues/659 -const LibP2pWebsockets = require('libp2p-websockets') +import Bootstrap from 'libp2p-bootstrap' +const Websockets = require('libp2p-websockets') const filters = require('libp2p-websockets/src/filters') -const LibP2pBootstrap = require('libp2p-bootstrap') -const mplex = require('libp2p-mplex') +const MPLEX = require('libp2p-mplex') export interface Libp2pNodeOptions { /* Peer id */ @@ -31,16 +29,16 @@ export interface Libp2pNodeOptions { export class Libp2pNode extends LibP2p { constructor(options: Libp2pNodeOptions) { - const wsTransportKey = LibP2pWebsockets.prototype[Symbol.toStringTag] + const wsTransportKey = Websockets.prototype[Symbol.toStringTag] options.bootnodes = options.bootnodes ?? [] super({ peerId: options.peerId, addresses: options.addresses, modules: { - transport: [LibP2pWebsockets], - streamMuxer: [mplex], + transport: [Websockets], + streamMuxer: [MPLEX], connEncryption: [NOISE], - ['peerDiscovery']: [LibP2pBootstrap], + ['peerDiscovery']: [Bootstrap], }, config: { transport: { @@ -50,7 +48,7 @@ export class Libp2pNode extends LibP2p { }, peerDiscovery: { autoDial: false, - [LibP2pBootstrap.tag]: { + [Bootstrap.tag]: { interval: 2000, enabled: options.bootnodes.length > 0, list: options.bootnodes, diff --git a/packages/client/browser_sync.png b/packages/client/browser_sync.png deleted file mode 100644 index e782229656..0000000000 Binary files a/packages/client/browser_sync.png and /dev/null differ diff --git a/packages/client/examples/browser_sync.png b/packages/client/examples/browser_sync.png new file mode 100644 index 0000000000..5e72b64848 Binary files /dev/null and b/packages/client/examples/browser_sync.png differ diff --git a/packages/client/examples/light-browser-sync.md b/packages/client/examples/light-browser-sync.md index 0dc7aa2b3a..c1f29f8f7c 100644 --- a/packages/client/examples/light-browser-sync.md +++ b/packages/client/examples/light-browser-sync.md @@ -1,6 +1,6 @@ # Example: Light Browser Sync -## Light sync +## Light sync In this example, we will run two ethereumjs clients. The first will be a full sync client that connects to the rinkeby network and starts downloading the blockchain. The second will be a @@ -63,10 +63,9 @@ ethereumjs.run({ network: 'rinkeby', syncmode: 'light', bootnodes: '/ip4/127.0.0.1/tcp/50505/ws/p2p/QmYAuYxw6QX1x5aafs6g3bUrPbMDifP5pDun3N9zbVLpEa', - discDns: false }) ``` That's it! Now, you should start seeing headers being downloaded to the local storage of your browser. Since IndexDB is being used, even if you close and re-open the browser window, the headers you've downloaded will be saved. -![EthereumJS Client Libp2p Browser Syncing](./browser_sync.png?raw=true) \ No newline at end of file +![EthereumJS Client Libp2p Browser Syncing](./browser_sync.png?raw=true) diff --git a/packages/client/lib/client.ts b/packages/client/lib/client.ts index d51be11d38..83ee173392 100644 --- a/packages/client/lib/client.ts +++ b/packages/client/lib/client.ts @@ -1,12 +1,12 @@ -import events from 'events' +import { bufferToHex } from 'ethereumjs-util' +import { version as packageVersion } from '../package.json' import { MultiaddrLike } from './types' import { Config } from './config' import { EthereumService, FullEthereumService, LightEthereumService } from './service' import { Event } from './types' +import { FullSynchronizer } from './sync' // eslint-disable-next-line implicit-dependencies/no-implicit import type { LevelUp } from 'levelup' -import { FullSynchronizer } from './sync' -import { bufferToHex } from 'ethereumjs-util' export interface EthereumClientOptions { /* Client configuration */ @@ -52,7 +52,7 @@ export interface EthereumClientOptions { * lifecycle of included services. * @memberof module:node */ -export default class EthereumClient extends events.EventEmitter { +export default class EthereumClient { public config: Config public services: (FullEthereumService | LightEthereumService)[] @@ -64,8 +64,6 @@ export default class EthereumClient extends events.EventEmitter { * Create new node */ constructor(options: EthereumClientOptions) { - super() - this.config = options.config this.services = [ @@ -92,15 +90,24 @@ export default class EthereumClient extends events.EventEmitter { if (this.opened) { return false } + this.config.logger.info( + `Initializing Ethereumjs client version=v${packageVersion} network=${this.config.chainCommon.chainName()}` + ) this.config.events.on(Event.SERVER_ERROR, (error) => { this.config.logger.warn(`Server error: ${error.name} - ${error.message}`) }) - this.config.events.on(Event.SERVER_LISTENING, (details) => { - this.config.logger.info(`Server listening: ${details.transport} - ${details.url}`) + this.config.logger.info( + `Server listener up transport=${details.transport} url=${details.url}` + ) + }) + this.config.events.on(Event.SYNC_SYNCHRONIZED, (height) => { + this.config.logger.info(`Synchronized blockchain at height ${height}`) }) + await Promise.all(this.services.map((s) => s.open())) + this.opened = true } diff --git a/packages/client/lib/index_old.ts b/packages/client/lib/index_old.ts deleted file mode 100644 index e22e4b017a..0000000000 --- a/packages/client/lib/index_old.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Define a library component for lazy loading. Borrowed from - * https://github.com/bcoin-org/bcoin/blob/master/lib/bcoin.js - * @param {string} name - * @param {string} path - */ -exports.define = function define(name: string, path: string) { - let cache: any = null - Object.defineProperty(exports, name, { - enumerable: true, - get() { - if (!cache) { - cache = require(path) - } - return cache - }, - }) -} - -// Blockchain -exports.define('blockchain', './blockchain') -exports.define('Chain', './blockchain/chain') - -// Handler -exports.define('handler', './handler') -exports.define('Handler', './handler/handler') -exports.define('EthHandler', './handler/ethhandler') -exports.define('LesHandler', './handler/leshandler') - -// Peer -exports.define('peer', './net/peer') -exports.define('Peer', './net/peer/peer') -exports.define('RlpxPeer', './net/peer/rlpxpeer') -exports.define('Libp2pPeer', './net/peer/libp2ppeer') - -// Peer Pool -exports.define('PeerPool', './net/peerpool') - -// Protocol -exports.define('protocol', './net/protocol') -exports.define('Protocol', './net/protocol/protocol') -exports.define('EthProtocol', './net/protocol/ethprotocol') -exports.define('LesProtocol', './net/protocol/lesprotocol') -exports.define('FlowControl', './net/protocol/flowcontrol') - -// Server -exports.define('server', './net/server') -exports.define('Server', './net/server/server') -exports.define('RlpxServer', './net/server/rlpxserver') -exports.define('Libp2pServer', './net/server/libp2pserver') - -// EthereumClient -exports.define('EthereumClient', './client') - -// RPC Manager -exports.define('RPCManager', './rpc') - -// Config -exports.define('Config', 'config') - -// Service -exports.define('service', './service') -exports.define('Service', './service/service') -exports.define('EthereumService', './service/ethereumservice') - -// Synchronizer -exports.define('sync', './sync') -exports.define('Synchronizer', './sync/sync') -exports.define('FullSynchronizer', './sync/fullsync') -exports.define('LightSynchronizer', './sync/lightsync') - -// Utilities -exports.define('util', './util') - -// Logging -exports.define('logging', './logging') - -export = exports diff --git a/packages/client/lib/net/peer/libp2pnode.ts b/packages/client/lib/net/peer/libp2pnode.ts index aac7c041a8..3f0c54766e 100644 --- a/packages/client/lib/net/peer/libp2pnode.ts +++ b/packages/client/lib/net/peer/libp2pnode.ts @@ -3,18 +3,16 @@ * @memberof module:net/peer */ -import { Multiaddr } from 'multiaddr' -import LibP2p from 'libp2p' import { NOISE } from '@chainsafe/libp2p-noise' +import LibP2p from 'libp2p' +import { Multiaddr } from 'multiaddr' import PeerId from 'peer-id' -// types currently unavailable for below libp2p deps, -// tracking issue: https://github.com/libp2p/js-libp2p/issues/659 -const LibP2pTcp = require('libp2p-tcp') -const LibP2pWebsockets = require('libp2p-websockets') +import Bootstrap from 'libp2p-bootstrap' +const TCP = require('libp2p-tcp') +const Websockets = require('libp2p-websockets') const filters = require('libp2p-websockets/src/filters') -const LibP2pBootstrap = require('libp2p-bootstrap') -const LibP2pKadDht = require('libp2p-kad-dht') -const mplex = require('libp2p-mplex') +const KadDht = require('libp2p-kad-dht') +const MPLEX = require('libp2p-mplex') export interface Libp2pNodeOptions { /* Peer id */ @@ -33,17 +31,17 @@ export interface Libp2pNodeOptions { export class Libp2pNode extends LibP2p { constructor(options: Libp2pNodeOptions) { - const wsTransportKey = LibP2pWebsockets.prototype[Symbol.toStringTag] + const wsTransportKey = Websockets.prototype[Symbol.toStringTag] options.bootnodes = options.bootnodes ?? [] super({ peerId: options.peerId, addresses: options.addresses, modules: { - transport: [LibP2pTcp, LibP2pWebsockets], - streamMuxer: [mplex], + transport: [TCP, Websockets], + streamMuxer: [MPLEX], connEncryption: [NOISE], - ['peerDiscovery']: [LibP2pBootstrap], - ['dht']: LibP2pKadDht, + ['peerDiscovery']: [Bootstrap], + ['dht']: KadDht, }, config: { transport: { @@ -53,7 +51,7 @@ export class Libp2pNode extends LibP2p { }, peerDiscovery: { autoDial: false, - [LibP2pBootstrap.tag]: { + [Bootstrap.tag]: { interval: 2000, enabled: options.bootnodes.length > 0, list: options.bootnodes, diff --git a/packages/client/lib/net/server/libp2pserver.ts b/packages/client/lib/net/server/libp2pserver.ts index 6898a8db8d..e172fb56cf 100644 --- a/packages/client/lib/net/server/libp2pserver.ts +++ b/packages/client/lib/net/server/libp2pserver.ts @@ -1,6 +1,6 @@ import PeerId from 'peer-id' // eslint-disable-next-line implicit-dependencies/no-implicit -import crypto from 'libp2p-crypto' +import { keys } from 'libp2p-crypto' import { Multiaddr, multiaddr } from 'multiaddr' import { Event, Libp2pConnection as Connection } from '../../types' import { Libp2pNode } from '../peer/libp2pnode' @@ -81,11 +81,12 @@ export class Libp2pServer extends Server { this.config.logger.debug(`Peer discovered: ${peer}`) this.config.events.emit(Event.PEER_CONNECTED, peer) }) - this.node.connectionManager.on('peer:connect', (connection: Connection) => { - const [peerId, multiaddr] = this.getPeerInfo(connection) - const peer = this.createPeer(peerId, [multiaddr]) + this.node.connectionManager.on('peer:connect', async (connection: Connection) => { + const [peerId, multiaddr, inbound] = this.getPeerInfo(connection) + const peer = this.createPeer(peerId, [multiaddr], inbound) this.config.logger.debug(`Peer connected: ${peer}`) - this.config.events.emit(Event.PEER_CONNECTED, peer) + // note: do not call Event.PEER_CONNECTED here, it will + // be called after bindProtocols on peer:discovery }) this.node.connectionManager.on('peer:disconnect', (_connection: Connection) => { // TODO: do anything here on disconnect? @@ -151,21 +152,22 @@ export class Libp2pServer extends Server { } async getPeerId() { - const privKey = await crypto.keys.generateKeyPairFromSeed('ed25519', this.key, 512) - const protoBuf = crypto.keys.marshalPrivateKey(privKey) + const privKey = await keys.generateKeyPairFromSeed('ed25519', this.key, 512) + const protoBuf = keys.marshalPrivateKey(privKey) return PeerId.createFromPrivKey(protoBuf) } - getPeerInfo(connection: Connection): [PeerId, Multiaddr] { - return [connection.remotePeer, connection.remoteAddr] + getPeerInfo(connection: Connection): [peerId: PeerId, multiaddr: Multiaddr, inbound: boolean] { + return [connection.remotePeer, connection.remoteAddr, connection._stat.direction === 'inbound'] } - createPeer(peerId: PeerId, multiaddrs?: Multiaddr[]) { + createPeer(peerId: PeerId, multiaddrs?: Multiaddr[], inbound = false) { const peer = new Libp2pPeer({ config: this.config, id: peerId.toB58String(), multiaddrs, protocols: Array.from(this.protocols), + inbound, }) this.peers.set(peer.id, peer) return peer diff --git a/packages/client/lib/sync/fetcher/blockfetcher.ts b/packages/client/lib/sync/fetcher/blockfetcher.ts index 4e6f918602..325c74c5b2 100644 --- a/packages/client/lib/sync/fetcher/blockfetcher.ts +++ b/packages/client/lib/sync/fetcher/blockfetcher.ts @@ -109,10 +109,10 @@ export class BlockFetcher extends BlockFetcherBase { async store(blocks: Block[]) { try { const num = await this.chain.putBlocks(blocks) - this.debug(`Fetcher results stored in blockchain (blocks num=${blocks.length}).`) + this.debug(`Fetcher results stored in blockchain (blocks num=${blocks.length})`) this.config.events.emit(Event.SYNC_FETCHER_FETCHED, blocks.slice(0, num)) } catch (e: any) { - this.debug(`Error on storing fetcher results in blockchain (blocks num=${blocks.length}).`) + this.debug(`Error storing fetcher results in blockchain (blocks num=${blocks.length}): ${e}`) throw e } } diff --git a/packages/client/lib/sync/fetcher/fetcher.ts b/packages/client/lib/sync/fetcher/fetcher.ts index 875d3cc3c9..2bd2d9898d 100644 --- a/packages/client/lib/sync/fetcher/fetcher.ts +++ b/packages/client/lib/sync/fetcher/fetcher.ts @@ -268,7 +268,7 @@ export abstract class Fetcher extends Readable return false } if (this._readableState!.length > this.maxQueue) { - this.debug(`Readable state length extends max queue size, skip next job execution.`) + this.debug(`Readable state length exceeds max queue size, skip next job execution.`) return false } if (job.index > this.processed + this.maxQueue) { @@ -329,7 +329,7 @@ export abstract class Fetcher extends Readable this.finished++ cb() } catch (error: any) { - this.config.logger.warn(`Error along storing received block or header result: ${error}`) + this.config.logger.warn(`Error storing received block or header result: ${error}`) cb(error) } } diff --git a/packages/client/lib/sync/fetcher/headerfetcher.ts b/packages/client/lib/sync/fetcher/headerfetcher.ts index 7174aba59e..99d5b89e7d 100644 --- a/packages/client/lib/sync/fetcher/headerfetcher.ts +++ b/packages/client/lib/sync/fetcher/headerfetcher.ts @@ -82,8 +82,16 @@ export class HeaderFetcher extends BlockFetcherBase { headers = headers as BlockHeader[] - const first = new BN(headers[0].number) + if (headers.length === 0) { + this.config.logger.warn('No headers fetched are applicable for import') + return + } + const first = headers[0].number const hash = short(headers[0].hash()) const baseFeeAdd = this.config.chainCommon.gteHardfork(Hardfork.London) ? `baseFee=${headers[0].baseFeePerGas} ` diff --git a/packages/client/package.json b/packages/client/package.json index 1214fd1ac7..3fccf09489 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -101,6 +101,7 @@ "os-browserify": "^0.3.0", "pino": "^5.8.0", "pino-pretty": "^2.2.2", + "process": "^0.11.10", "pull-pair": "^1.1.0", "stream-browserify": "^3.0.0", "supertest": "^6.1.3", diff --git a/packages/client/test/net/server/libp2pserver.spec.ts b/packages/client/test/net/server/libp2pserver.spec.ts index f68be688b3..8d010729bb 100644 --- a/packages/client/test/net/server/libp2pserver.spec.ts +++ b/packages/client/test/net/server/libp2pserver.spec.ts @@ -59,6 +59,11 @@ tape('[Libp2pServer]', async (t) => { ) t.equals(server.key!.toString(), 'abcd', 'key is correct') t.equals(server.name, 'libp2p', 'get name') + t.equals( + (await server.getPeerId()).toB58String(), + '12D3KooWHnPxZvSVGxToTNaK1xd9z3J1TkQM2S2hLeX4bhraGE64', + 'computes correct peerId' + ) t.end() }) @@ -87,7 +92,7 @@ tape('[Libp2pServer]', async (t) => { }) t.test('should start/stop server and test banning', async (t) => { - t.plan(11) + t.plan(12) const config = new Config({ transports: [], logger: getLogger({ loglevel: 'off' }) }) const multiaddrs = [multiaddr('/ip4/6.6.6.6')] const server = new Libp2pServer({ config, multiaddrs, key: Buffer.from('4') }) @@ -148,7 +153,8 @@ tape('[Libp2pServer]', async (t) => { node.emit('peer:discovery', peerId2) td.when(server.getPeerInfo('conn3' as any)).thenReturn([peerId3, 'ma1' as any]) node.connectionManager.emit('peer:connect', 'conn3') - td.verify(server.createPeer(peerId3, ['ma1'] as any)) + td.verify(server.createPeer(peerId3, ['ma1'] as any, td.matchers.anything())) + t.ok((await server.start()) === false, 'server already started') await server.stop() t.notOk(server.running, 'stopped') }) diff --git a/packages/client/test/sync/fetcher/blockfetcher.spec.ts b/packages/client/test/sync/fetcher/blockfetcher.spec.ts index 1f7fe92191..b0bab5ae6f 100644 --- a/packages/client/test/sync/fetcher/blockfetcher.spec.ts +++ b/packages/client/test/sync/fetcher/blockfetcher.spec.ts @@ -4,6 +4,7 @@ import { BN } from 'ethereumjs-util' import { Config } from '../../../lib/config' import { Chain } from '../../../lib/blockchain/chain' import { wait } from '../../integration/util' +import { Event } from '../../../lib/types' tape('[BlockFetcher]', async (t) => { class PeerPool { @@ -101,6 +102,36 @@ tape('[BlockFetcher]', async (t) => { t.end() }) + t.test('store()', async (st) => { + td.reset() + st.plan(2) + + const config = new Config({ maxPerRequest: 5, transports: [] }) + const pool = new PeerPool() as any + const chain = new Chain({ config }) + chain.putBlocks = td.func() + const fetcher = new BlockFetcher({ + config, + pool, + chain, + first: new BN(1), + count: new BN(10), + timeout: 5, + }) + td.when(chain.putBlocks(td.matchers.anything())).thenReject(new Error('err0')) + try { + await fetcher.store([]) + } catch (err: any) { + st.ok(err.message === 'err0', 'store() threw on invalid block') + } + td.reset() + chain.putBlocks = td.func() + td.when(chain.putBlocks(td.matchers.anything())).thenResolve(1) + config.events.on(Event.SYNC_FETCHER_FETCHED, () => + st.pass('store() emitted SYNC_FETCHER_FETCHED event on putting blocks') + ) + await fetcher.store([]) + }) t.test('should reset td', (t) => { td.reset() t.end() diff --git a/packages/client/test/sync/fetcher/fetcher.spec.ts b/packages/client/test/sync/fetcher/fetcher.spec.ts index 9ba42034bb..5d4ec7bda2 100644 --- a/packages/client/test/sync/fetcher/fetcher.spec.ts +++ b/packages/client/test/sync/fetcher/fetcher.spec.ts @@ -70,8 +70,8 @@ tape('[Fetcher]', (t) => { }, 20) }) - t.test('should handle clearing queue', (t) => { - t.plan(2) + t.test('should handle queue management', (t) => { + t.plan(3) const config = new Config({ transports: [] }) const fetcher = new FetcherTest({ config, @@ -86,6 +86,16 @@ tape('[Fetcher]', (t) => { t.equals((fetcher as any).in.size(), 3, 'queue filled') fetcher.clear() t.equals((fetcher as any).in.size(), 0, 'queue cleared') + const job4 = { index: 3 } + const job5 = { index: 4 } + + ;(fetcher as any).in.insert(job1) + ;(fetcher as any).in.insert(job2) + ;(fetcher as any).in.insert(job3) + ;(fetcher as any).in.insert(job4) + ;(fetcher as any).in.insert(job5) + + t.ok(fetcher.next() === false, 'next() fails when heap length exceeds maxQueue') }) t.test('should reset td', (t) => { diff --git a/packages/client/test/sync/fetcher/headerfetcher.spec.ts b/packages/client/test/sync/fetcher/headerfetcher.spec.ts index 65e9dfa1d3..9b782aa489 100644 --- a/packages/client/test/sync/fetcher/headerfetcher.spec.ts +++ b/packages/client/test/sync/fetcher/headerfetcher.spec.ts @@ -2,6 +2,8 @@ import tape from 'tape' import td from 'testdouble' import { Config } from '../../../lib/config' import { BN } from 'ethereumjs-util' +import { Chain } from '../../../lib/blockchain' +import { Event } from '../../../lib/types' tape('[HeaderFetcher]', async (t) => { class PeerPool { @@ -42,6 +44,34 @@ tape('[HeaderFetcher]', async (t) => { t.end() }) + t.test('store()', async (st) => { + st.plan(2) + + const config = new Config({ maxPerRequest: 5, transports: [] }) + const pool = new PeerPool() as any + const chain = new Chain({ config }) + chain.putHeaders = td.func() + const fetcher = new HeaderFetcher({ + config, + pool, + chain, + first: new BN(1), + count: new BN(10), + timeout: 5, + }) + td.when(chain.putHeaders([0 as any])).thenReject(new Error('err0')) + try { + await fetcher.store([0 as any]) + } catch (err: any) { + st.ok(err.message === 'err0', 'store() threw on invalid header') + } + td.when(chain.putHeaders([1 as any])).thenResolve(1) + config.events.on(Event.SYNC_FETCHER_FETCHED, () => + st.pass('store() emitted SYNC_FETCHER_FETCHED event on putting headers') + ) + await fetcher.store([1 as any]) + }) + t.test('should reset td', (t) => { td.reset() t.end() diff --git a/packages/client/test/sync/lightsync.spec.ts b/packages/client/test/sync/lightsync.spec.ts index 1645149f7d..8275b3680b 100644 --- a/packages/client/test/sync/lightsync.spec.ts +++ b/packages/client/test/sync/lightsync.spec.ts @@ -4,6 +4,7 @@ import { BN } from 'ethereumjs-util' import { Config } from '../../lib/config' import { Chain } from '../../lib/blockchain' import { Event } from '../../lib/types' +import { BlockHeader } from '@ethereumjs/block' tape('[LightSynchronizer]', async (t) => { class PeerPool { @@ -93,8 +94,71 @@ tape('[LightSynchronizer]', async (t) => { } }) - t.test('should reset td', (t) => { + t.test('sync errors', async (st) => { td.reset() - t.end() + st.plan(1) + const config = new Config({ transports: [] }) + const pool = new PeerPool() as any + const chain = new Chain({ config }) + const sync = new LightSynchronizer({ + config, + interval: 1, + pool, + chain, + }) + sync.best = td.func() + sync.latest = td.func() + td.when(sync.best()).thenReturn({ les: { status: { headNum: new BN(2) } } } as any) + td.when(sync.latest(td.matchers.anything())).thenResolve({ + number: new BN(2), + hash: () => Buffer.from([]), + }) + td.when(HeaderFetcher.prototype.fetch()).thenResolve(undefined) + td.when(HeaderFetcher.prototype.fetch()).thenDo(() => + config.events.emit(Event.SYNC_FETCHER_FETCHED, [] as BlockHeader[]) + ) + config.logger.on('data', async (data) => { + if (data.message.includes('No headers fetched are applicable for import')) { + st.pass('generated correct warning message when no headers received') + config.logger.removeAllListeners() + await sync.stop() + await sync.close() + } + }) + await sync.sync() + }) + + t.test('import headers', async (st) => { + td.reset() + st.plan(1) + const config = new Config({ transports: [] }) + const pool = new PeerPool() as any + const chain = new Chain({ config }) + const sync = new LightSynchronizer({ + config, + interval: 1, + pool, + chain, + }) + sync.best = td.func() + sync.latest = td.func() + td.when(sync.best()).thenReturn({ les: { status: { headNum: new BN(2) } } } as any) + td.when(sync.latest(td.matchers.anything())).thenResolve({ + number: new BN(2), + hash: () => Buffer.from([]), + }) + td.when(HeaderFetcher.prototype.fetch()).thenResolve(undefined) + td.when(HeaderFetcher.prototype.fetch()).thenDo(() => + config.events.emit(Event.SYNC_FETCHER_FETCHED, [BlockHeader.fromHeaderData({})]) + ) + config.logger.on('data', async (data) => { + if (data.message.includes('Imported headers count=1')) { + st.pass('successfully imported new header') + config.logger.removeAllListeners() + await sync.stop() + await sync.close() + } + }) + await sync.sync() }) }) diff --git a/packages/client/webpack.config.js b/packages/client/webpack.config.js index ea62fe099c..959e494186 100644 --- a/packages/client/webpack.config.js +++ b/packages/client/webpack.config.js @@ -1,8 +1,17 @@ const { resolve } = require('path') +const { ProvidePlugin } = require('webpack') module.exports = { mode: 'production', entry: './dist.browser/browser/index.js', + plugins: [ + new ProvidePlugin({ + Buffer: ['buffer', 'Buffer'], + }), + new ProvidePlugin({ + process: 'process/browser', + }), + ], module: { rules: [ { @@ -36,6 +45,7 @@ module.exports = { }, resolve: { fallback: { + buffer: require.resolve('buffer'), crypto: require.resolve('crypto-browserify'), // used by: rlpxpeer, bin/cli.ts dgram: false, // used by: rlpxpeer via @ethereumjs/devp2p fs: false, // used by: FullSynchronizer via @ethereumjs/vm