From 2b1f2b66129300529f81572229f8453469caf65a Mon Sep 17 00:00:00 2001 From: yoozo Date: Fri, 19 Jul 2024 09:58:01 +0800 Subject: [PATCH] Strict node (#325) * some chagne * some change * some change * some chagne * some change * some change * Update deps, fix build issues * some changes * Update the dependency version of @subql/types-core. * Revert "Update the dependency version of @subql/types-core." This reverts commit 1a9acb19a6a114f52847eea26754769b03d03593. * enable ts strict * change log * some change --------- Co-authored-by: Scott Twiname --- .eslintrc.js | 8 +++ packages/common-ethereum/CHANGELOG.md | 3 + .../src/codegen/codegen-controller.spec.ts | 2 +- .../common-ethereum/src/project/models.ts | 4 +- packages/common-ethereum/src/project/utils.ts | 6 +- packages/node/CHANGELOG.md | 3 + packages/node/package.json | 2 +- .../node/src/configure/SubqueryProject.ts | 8 +-- .../node/src/ethereum/api.ethereum.test.ts | 20 +++---- packages/node/src/ethereum/api.ethereum.ts | 41 ++++++++----- .../node/src/ethereum/api.service.ethereum.ts | 24 ++++++-- packages/node/src/ethereum/block.ethereum.ts | 2 +- .../ethers/celo/celo-ws-provider.spec.ts | 4 +- .../ethers/json-rpc-batch-provider.spec.ts | 4 +- .../ethers/json-rpc-batch-provider.ts | 8 +-- .../src/ethereum/ethers/json-rpc-provider.ts | 6 +- .../ethereum/ethers/op/op-provider.spec.ts | 2 - .../node/src/ethereum/ethers/web/geturl.ts | 38 ++++++------ .../node/src/ethereum/ethers/web/index.ts | 59 +++++++++++-------- .../node/src/ethereum/ethers/web/types.ts | 2 +- packages/node/src/ethereum/utils.ethereum.ts | 7 +-- .../worker-block-dispatcher.service.ts | 2 +- .../dictionary/v1/ethDictionaryV1.spec.ts | 2 +- .../indexer/dictionary/v1/ethDictionaryV1.ts | 16 +++-- .../dictionary/v2/ethDictionaryV2.spec.ts | 7 ++- .../indexer/dictionary/v2/ethDictionaryV2.ts | 32 ++++++---- .../node/src/indexer/dictionary/v2/types.ts | 2 +- .../node/src/indexer/dictionary/v2/utils.ts | 6 +- .../node/src/indexer/dynamic-ds.service.ts | 2 +- packages/node/src/indexer/fetch.service.ts | 4 +- packages/node/src/indexer/indexer.manager.ts | 2 +- .../node/src/indexer/project.service.test.ts | 20 +++---- .../src/indexer/unfinalizedBlocks.service.ts | 5 +- packages/node/src/indexer/worker/worker.ts | 4 +- packages/node/src/init.ts | 25 +++----- packages/node/src/meta/meta.service.ts | 6 +- packages/node/src/utils/string.ts | 2 +- packages/node/tsconfig.json | 3 +- packages/types/CHANGELOG.md | 3 + packages/types/src/ethereum/interfaces.ts | 4 +- packages/types/src/project.ts | 38 ++++++++++-- yarn.lock | 22 +------ 42 files changed, 262 insertions(+), 198 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index e9c493abe5..d45144742e 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -112,4 +112,12 @@ module.exports = { '@typescript-eslint/parser': ['.ts', '.tsx'], }, }, + overrides: [ + { + files: ['*.test.ts', '*.spec.ts'], + rules: { + '@typescript-eslint/no-non-null-assertion': 'off', // @typescript-eslint/no-non-null-assertion + }, + }, + ], }; diff --git a/packages/common-ethereum/CHANGELOG.md b/packages/common-ethereum/CHANGELOG.md index fa03649314..1778cb7c79 100644 --- a/packages/common-ethereum/CHANGELOG.md +++ b/packages/common-ethereum/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Enable ts strict + ## [4.3.0] - 2024-07-11 ### Changed - Bump with `@subql/common` (#2487) diff --git a/packages/common-ethereum/src/codegen/codegen-controller.spec.ts b/packages/common-ethereum/src/codegen/codegen-controller.spec.ts index 6940e26f68..0eae5e984a 100644 --- a/packages/common-ethereum/src/codegen/codegen-controller.spec.ts +++ b/packages/common-ethereum/src/codegen/codegen-controller.spec.ts @@ -416,7 +416,7 @@ describe('Codegen spec', () => { }, ], }, - }; + } as any; await expect( generateAbis([ds], PROJECT_PATH, undefined as any, undefined as any, undefined as any) diff --git a/packages/common-ethereum/src/project/models.ts b/packages/common-ethereum/src/project/models.ts index 4794e11d03..9f4cfd183c 100644 --- a/packages/common-ethereum/src/project/models.ts +++ b/packages/common-ethereum/src/project/models.ts @@ -13,7 +13,6 @@ import { SubqlRuntimeHandler, SubqlRuntimeDatasource, SubqlCustomDatasource, - CustomDataSourceAsset, EthereumBlockFilter, SubqlBlockHandler, SubqlEventHandler, @@ -29,7 +28,6 @@ import { IsString, IsObject, ValidateNested, - IsEthereumAddress, registerDecorator, ValidationOptions, ValidationArguments, @@ -212,7 +210,7 @@ export class CustomDataSourceBase FileReferenceImpl) @ValidateNested({each: true}) - assets!: Map; + assets!: Map; @Type(() => FileReferenceImpl) @IsObject() processor!: FileReference; diff --git a/packages/common-ethereum/src/project/utils.ts b/packages/common-ethereum/src/project/utils.ts index ea15045807..5b1a6ba80b 100644 --- a/packages/common-ethereum/src/project/utils.ts +++ b/packages/common-ethereum/src/project/utils.ts @@ -11,6 +11,8 @@ import { EthereumHandlerKind, SubqlRuntimeDatasource, SecondLayerHandlerProcessorArray, + SubqlCustomHandler, + SubqlMapping, } from '@subql/types-ethereum'; import {fromBech32Address} from '@zilliqa-js/crypto'; import {buildMessage, isEthereumAddress, ValidateBy, ValidationOptions} from 'class-validator'; @@ -39,7 +41,9 @@ export function isCallHandlerProcessor( return hp.baseHandlerKind === EthereumHandlerKind.Call; } -export function isCustomDs(ds: SubqlDatasource): ds is SubqlCustomDatasource { +export function isCustomDs>( + ds: SubqlDatasource +): ds is SubqlCustomDatasource { return ds.kind !== EthereumDatasourceKind.Runtime && !!(ds as SubqlCustomDatasource).processor; } diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index faec7ed91c..ab25e63d68 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Enable ts strict mode + ## [4.7.3] - 2024-07-16 ### Fixed - Fix dockerfile missing set timezone to UTC (#329) diff --git a/packages/node/package.json b/packages/node/package.json index a65a541a6f..4b5e0ce840 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -26,7 +26,7 @@ "@nestjs/schedule": "^3.0.1", "@subql/common-ethereum": "workspace:*", "@subql/node-core": "^11.0.0", - "@subql/testing": "^2.1.1", + "@subql/testing": "^2.2.1", "@subql/types-ethereum": "workspace:*", "cacheable-lookup": "6", "ethers": "^5.7.0", diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index 1538803b90..a692b552db 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -39,10 +39,6 @@ export type EthereumProjectDsTemplate = export type SubqlProjectBlockFilter = EthereumBlockFilter & CronFilter; -const NOT_SUPPORT = (name: string) => { - throw new Error(`Manifest specVersion ${name} is not supported`); -}; - // This is the runtime type after we have mapped genesisHash to chainId and endpoint/dict have been provided when dealing with deployments type NetworkConfig = EthereumNetworkConfig & { chainId: string }; @@ -105,7 +101,9 @@ export class SubqueryProject implements ISubqueryProject { networkOverrides, ); } else { - NOT_SUPPORT(manifest.specVersion); + throw new Error( + `Manifest specVersion ${manifest.specVersion} is not supported`, + ); } } } diff --git a/packages/node/src/ethereum/api.ethereum.test.ts b/packages/node/src/ethereum/api.ethereum.test.ts index 372d69080e..403fe44594 100644 --- a/packages/node/src/ethereum/api.ethereum.test.ts +++ b/packages/node/src/ethereum/api.ethereum.test.ts @@ -74,11 +74,11 @@ describe('Api.ethereum', () => { expect(typeof blockData.transactions[0].receipt).toEqual('function'); expect(typeof blockData.logs[0].transaction.receipt).toEqual('function'); expect(typeof blockData.logs[0].transaction.from).toEqual('string'); - expect(typeof blockData.transactions[81].logs[0].transaction.from).toEqual( + expect(typeof blockData.transactions[81].logs![0].transaction.from).toEqual( 'string', ); expect( - typeof blockData.transactions[81].logs[0].transaction.receipt, + typeof blockData.transactions[81].logs![0].transaction.receipt, ).toEqual('function'); }); @@ -89,8 +89,8 @@ describe('Api.ethereum', () => { e.hash === '0x8e419d0e36d7f9c099a001fded516bd168edd9d27b4aec2bcd56ba3b3b955ccc', ); - const parsedTx = await ethApi.parseTransaction(tx, ds); - expect(parsedTx.logs[0].args).toBeTruthy(); + const parsedTx = await ethApi.parseTransaction(tx!, ds); + expect(parsedTx.logs![0].args).toBeTruthy(); }); it('Should decode transaction data and not clone object', async () => { @@ -99,7 +99,7 @@ describe('Api.ethereum', () => { e.hash === '0x8e419d0e36d7f9c099a001fded516bd168edd9d27b4aec2bcd56ba3b3b955ccc', ); - const parsedTx = await ethApi.parseTransaction(tx, ds); + const parsedTx = await ethApi.parseTransaction(tx!, ds); expect(parsedTx).toBe(tx); }); @@ -111,7 +111,7 @@ describe('Api.ethereum', () => { e.hash === '0xed62f7a7720fe6ae05dec45ad9dd4f53034a0aae2c140d229b1151504ee9a6c9', ); - const parsedLog = await ethApi.parseLog(tx.logs[0], ds); + const parsedLog = await ethApi.parseLog(tx!.logs![0], ds); expect(parsedLog).not.toHaveProperty('args'); expect(parsedLog).toBeTruthy(); }); @@ -122,7 +122,7 @@ describe('Api.ethereum', () => { (e) => e.hash === '0x8e419d0e36d7f9c099a001fded516bd168edd9d27b4aec2bcd56ba3b3b955ccc', - ).logs[1]; + )!.logs![1]; const parsedLog = await ethApi.parseLog(log, ds); expect(parsedLog).toBe(log); @@ -174,12 +174,12 @@ describe('Api.ethereum', () => { (tx) => tx.hash === '0xeb2e443f2d4e784193fa13bbbae2b85e6ee459e7b7b53f8ca098ffae9b25b059', - ); + )!; - const erc20Transfers = transaction.logs.filter((log) => + const erc20Transfers = transaction.logs!.filter((log) => filterLogsProcessor(log, filter_2), ); - const erc721Transfers = transaction.logs.filter((log) => + const erc721Transfers = transaction.logs!.filter((log) => filterLogsProcessor(log, filter_1), ); diff --git a/packages/node/src/ethereum/api.ethereum.ts b/packages/node/src/ethereum/api.ethereum.ts index 339de6cabe..1569a57e58 100644 --- a/packages/node/src/ethereum/api.ethereum.ts +++ b/packages/node/src/ethereum/api.ethereum.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import fs from 'fs'; import http from 'http'; import https from 'https'; @@ -96,10 +97,10 @@ export class EthereumApi implements ApiWrapper { // This is used within the sandbox when HTTP is used private nonBatchClient?: JsonRpcProvider; - private genesisBlock: Record; + private _genesisBlock?: Record; private contractInterfaces: Record = {}; - private chainId: number; - private name: string; + private chainId?: number; + private name?: string; // Ethereum POS private _supportsFinalization = true; @@ -147,6 +148,13 @@ export class EthereumApi implements ApiWrapper { } } + private get genesisBlock(): Record { + if (!this._genesisBlock) { + throw new Error('Genesis block is not available'); + } + return this._genesisBlock; + } + async init(): Promise { this.injectClient(); @@ -170,12 +178,15 @@ export class EthereumApi implements ApiWrapper { this.getSupportsTag('safe'), ]); - this.genesisBlock = genesisBlock; + this._genesisBlock = genesisBlock; this._supportsFinalization = supportsFinalization && supportsSafe; this.chainId = network.chainId; this.name = network.name; } catch (e) { if ((e as Error).message.startsWith('Invalid response')) { + if (!this.nonBatchClient) { + throw new Error('No suitable client found'); + } this.client = this.nonBatchClient; logger.warn( @@ -206,9 +217,9 @@ export class EthereumApi implements ApiWrapper { private injectClient(): void { const orig = this.client.send.bind(this.client); Object.defineProperty(this.client, 'send', { - value: (...args) => { + value: (method: string, args: any[]) => { this.eventEmitter.emit('rpcCall'); - return orig(...args); + return orig(method, args); }, }); } @@ -261,10 +272,12 @@ export class EthereumApi implements ApiWrapper { } getRuntimeChain(): string { + assert(this.name, 'Api has not been initialised'); return this.name; } getChainId(): number { + assert(this.chainId, 'Api has not been initialised'); return this.chainId; } @@ -337,15 +350,13 @@ export class EthereumApi implements ApiWrapper { block.transactions = block.transactions.map((tx) => ({ ...formatTransaction(tx, block), receipt: () => - this.getTransactionReceipt(tx.hash).then((r) => - formatReceipt(r, block), - ), + this.getTransactionReceipt(tx.hash).then((r) => formatReceipt(r)), logs: block.logs.filter((l) => l.transactionHash === tx.hash), })); this.eventEmitter.emit('fetchBlock'); return formatBlockUtil(block); - } catch (e) { + } catch (e: any) { throw this.handleError(e); } } @@ -386,13 +397,15 @@ export class EthereumApi implements ApiWrapper { ? this.client : this.nonBatchClient; + assert(client, 'Unable to find client to make SafeApi'); + return new SafeEthProvider(client, blockHeight); } private buildInterface( abiName: string, assets: Record, - ): Interface | undefined { + ): Interface { if (!assets[abiName]) { throw new Error(`ABI named "${abiName}" not referenced in assets`); } @@ -412,7 +425,7 @@ export class EthereumApi implements ApiWrapper { } this.contractInterfaces[abiName] = new Interface(abiObj); - } catch (e) { + } catch (e: any) { logger.error(`Unable to parse ABI: ${e.message}`); throw new Error('ABI is invalid'); } @@ -437,7 +450,7 @@ export class EthereumApi implements ApiWrapper { log.args = iface?.parseLog(log).args as T; return log; - } catch (e) { + } catch (e: any) { logger.warn(`Failed to parse log data: ${e.message}`); return log; } @@ -467,7 +480,7 @@ export class EthereumApi implements ApiWrapper { transaction.args = args; return transaction; - } catch (e) { + } catch (e: any) { logger.warn(`Failed to parse transaction data: ${e.message}`); return transaction; } diff --git a/packages/node/src/ethereum/api.service.ethereum.ts b/packages/node/src/ethereum/api.service.ethereum.ts index 9805283ab9..b5d66d8942 100644 --- a/packages/node/src/ethereum/api.service.ethereum.ts +++ b/packages/node/src/ethereum/api.service.ethereum.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { @@ -36,8 +37,11 @@ export class EthereumApiService extends ApiService< SafeEthProvider, IBlock[] | IBlock[] > { - private fetchBlocksFunction: FetchFunc; - private fetchBlocksBatches: GetFetchFunc = () => this.fetchBlocksFunction; + private fetchBlocksFunction?: FetchFunc; + private fetchBlocksBatches: GetFetchFunc = () => { + assert(this.fetchBlocksFunction, 'Fetch blocks function is not defined'); + return this.fetchBlocksFunction; + }; private nodeConfig: EthereumNodeConfig; constructor( @@ -114,14 +118,19 @@ export class EthereumApiService extends ApiService< get: (target, prop, receiver) => { const originalMethod = target[prop as keyof SafeEthProvider]; if (typeof originalMethod === 'function') { - return async (...args: any[]) => { + return async ( + ...args: Parameters + ): Promise> => { let retries = 0; let currentApi = target; - let throwingError: Error; + let throwingError: Error | undefined; while (retries < maxRetries) { try { - return await originalMethod.apply(currentApi, args); + return await (originalMethod as Function).apply( + currentApi, + args, + ); } catch (error: any) { // other than retryErrorCodes, other errors does not have anything to do with network request, retrying would not change its outcome if (!retryErrorCodes.includes(error?.code)) { @@ -140,6 +149,9 @@ export class EthereumApiService extends ApiService< logger.error( `Maximum retries (${maxRetries}) exceeded for api at height ${height}`, ); + if (!throwingError) { + throw new Error('Failed to make request, maximum retries failed'); + } throw throwingError; }; } @@ -198,7 +210,7 @@ export class EthereumApiService extends ApiService< fetchFunc, 'SubstrateUtil', 'fetchBlocksBatches', - ); + ) as FetchFunc; } else { this.fetchBlocksFunction = fetchFunc; } diff --git a/packages/node/src/ethereum/block.ethereum.ts b/packages/node/src/ethereum/block.ethereum.ts index fd9ac2c20c..233442a7f2 100644 --- a/packages/node/src/ethereum/block.ethereum.ts +++ b/packages/node/src/ethereum/block.ethereum.ts @@ -44,7 +44,7 @@ export function filterBlocksProcessor( export function filterTransactionsProcessor( transaction: EthereumTransaction, - filter: EthereumTransactionFilter, + filter?: EthereumTransactionFilter, address?: string, ): boolean { if (!filter) return true; diff --git a/packages/node/src/ethereum/ethers/celo/celo-ws-provider.spec.ts b/packages/node/src/ethereum/ethers/celo/celo-ws-provider.spec.ts index 78c169a525..5120b2cf25 100644 --- a/packages/node/src/ethereum/ethers/celo/celo-ws-provider.spec.ts +++ b/packages/node/src/ethereum/ethers/celo/celo-ws-provider.spec.ts @@ -21,9 +21,7 @@ describe('CeloRPCProviders', () => { new CeloJsonRpcBatchProvider(HTTP_ENDPOINT), ]; afterAll(async () => { - await Promise.all( - providers.map((p) => (p as WebSocketProvider)?.destroy?.()), - ); + await Promise.all(providers.map((p) => (p as any)?.destroy?.())); }); // This returns a value now, needs further investigation diff --git a/packages/node/src/ethereum/ethers/json-rpc-batch-provider.spec.ts b/packages/node/src/ethereum/ethers/json-rpc-batch-provider.spec.ts index dc9fc81d57..55ff53b2e6 100644 --- a/packages/node/src/ethereum/ethers/json-rpc-batch-provider.spec.ts +++ b/packages/node/src/ethereum/ethers/json-rpc-batch-provider.spec.ts @@ -34,7 +34,7 @@ describe('JsonRpcBatchProvider', () => { // Execute the send method multiple times to simulate successful requests const requestCount = 20; - const promises = []; + const promises: Promise[] = []; for (let i = 0; i < requestCount; i++) { const promise = batchProvider.send('eth_call', []); promises.push(promise); @@ -57,7 +57,7 @@ describe('JsonRpcBatchProvider', () => { ); // Execute the send method multiple times to simulate failed requests - const failedPromises = []; + const failedPromises: Promise[] = []; for (let i = 0; i < requestCount + 10; i++) { const failedPromise = batchProvider.send('eth_call', []); failedPromises.push(failedPromise); diff --git a/packages/node/src/ethereum/ethers/json-rpc-batch-provider.ts b/packages/node/src/ethereum/ethers/json-rpc-batch-provider.ts index 5431840c84..707085fb14 100644 --- a/packages/node/src/ethereum/ethers/json-rpc-batch-provider.ts +++ b/packages/node/src/ethereum/ethers/json-rpc-batch-provider.ts @@ -27,12 +27,12 @@ export class JsonRpcBatchProvider extends JsonRpcProvider { private failedBatchCount = 0; private batchSizeAdjustmentInterval = 10; // Adjust batch size after every 10 batches - _pendingBatchAggregator: NodeJS.Timer; + _pendingBatchAggregator: NodeJS.Timer | null = null; _pendingBatch: Array<{ request: { method: string; params: Array; id: number; jsonrpc: '2.0' }; resolve: (result: any) => void; reject: (error: Error) => void; - }>; + }> | null = null; _chainIdCache: string | null = null; @@ -93,7 +93,7 @@ export class JsonRpcBatchProvider extends JsonRpcProvider { private runRequests() { // Get teh current batch and clear it, so new requests // go into the next batch - const batch = this._pendingBatch; + const batch = this._pendingBatch ?? []; this._pendingBatch = null; this._pendingBatchAggregator = null; @@ -189,7 +189,7 @@ export class JsonRpcBatchProvider extends JsonRpcProvider { inflightRequest.reject(error); } else { if (inflightRequest.request.method === 'eth_chainId') { - this._chainIdCache = payload.result; + this._chainIdCache = payload.result ?? null; } inflightRequest.resolve(payload.result); } diff --git a/packages/node/src/ethereum/ethers/json-rpc-provider.ts b/packages/node/src/ethereum/ethers/json-rpc-provider.ts index c787b07fde..42be19d4c0 100644 --- a/packages/node/src/ethereum/ethers/json-rpc-provider.ts +++ b/packages/node/src/ethereum/ethers/json-rpc-provider.ts @@ -41,8 +41,8 @@ export class JsonRpcProvider extends BaseJsonRpcProvider { // We can expand this in the future to any call, but for now these // are the biggest wins and do not require any serializing parameters. - const cache = ['eth_chainId', 'eth_blockNumber'].indexOf(method) >= 0; - if (cache && this._cache[method]) { + const cache = ['eth_chainId', 'eth_blockNumber'].includes(method); + if (cache && !!this._cache[method]) { return this._cache[method]; } @@ -77,7 +77,7 @@ export class JsonRpcProvider extends BaseJsonRpcProvider { if (cache) { this._cache[method] = result; setTimeout(() => { - this._cache[method] = null; + delete this._cache[method]; }, 0); } diff --git a/packages/node/src/ethereum/ethers/op/op-provider.spec.ts b/packages/node/src/ethereum/ethers/op/op-provider.spec.ts index d1fff81be7..930cf11ed3 100644 --- a/packages/node/src/ethereum/ethers/op/op-provider.spec.ts +++ b/packages/node/src/ethereum/ethers/op/op-provider.spec.ts @@ -30,7 +30,6 @@ describe('OPRPCProviders', () => { await provider.getTransactionReceipt( '0x5496af6ad1d619279d82b8f4c94cf3f8da8c02f22481c66a840ae9dd3f5e1a23', ), - null, ); expect(receipt.l1Fee).toEqual(BigNumber.from('0x1375ad1b756e')); @@ -46,7 +45,6 @@ describe('OPRPCProviders', () => { await provider.getTransactionReceipt( '0x7c20ced906264f81929802ee6b642d003a236c542c5de6298ede5b2a4f7f9bb9', ), - null, ); expect(receipt.l1Fee).toBeUndefined(); diff --git a/packages/node/src/ethereum/ethers/web/geturl.ts b/packages/node/src/ethereum/ethers/web/geturl.ts index fa3e07046f..271c213caf 100644 --- a/packages/node/src/ethereum/ethers/web/geturl.ts +++ b/packages/node/src/ethereum/ethers/web/geturl.ts @@ -1,7 +1,7 @@ /* eslint-disable */ 'use strict'; -import http from 'http'; +import http, { RequestOptions } from 'http'; import https from 'https'; import { gunzipSync } from 'zlib'; import { parse } from 'url'; @@ -20,15 +20,19 @@ export { GetUrlResponse, Options }; function getResponse(request: http.ClientRequest): Promise { return new Promise((resolve, reject) => { request.once('response', (resp: http.IncomingMessage) => { + if (!resp.statusCode) { + return reject(new Error('response statusCode is undefined')); + } + const response: GetUrlResponse = { statusCode: resp.statusCode, - statusMessage: resp.statusMessage, + statusMessage: resp.statusMessage ?? '', headers: Object.keys(resp.headers).reduce((accum, name) => { let value = resp.headers[name]; if (Array.isArray(value)) { value = value.join(', '); } - accum[name] = value; + if (value !== undefined) accum[name] = value; return accum; }, <{ [name: string]: string }>{}), body: null, @@ -36,16 +40,15 @@ function getResponse(request: http.ClientRequest): Promise { //resp.setEncoding("utf8"); resp.on('data', (chunk: Uint8Array) => { - if (response.body == null) { - response.body = new Uint8Array(0); - } - response.body = concat([response.body, chunk]); + response.body = concat([response.body ?? new Uint8Array(0), chunk]); }); resp.on('end', () => { if (response.headers['content-encoding'] === 'gzip') { //const size = response.body.length; - response.body = arrayify(gunzipSync(response.body)); + response.body = response.body + ? arrayify(gunzipSync(response.body)) + : null; //console.log("Delta:", response.body.length - size, Buffer.from(response.body).toString()); } resolve(response); @@ -65,7 +68,7 @@ function getResponse(request: http.ClientRequest): Promise { } // The URL.parse uses null instead of the empty string -function nonnull(value: string): string { +function nonnull(value: string | null): string { if (value == null) { return ''; } @@ -85,22 +88,21 @@ export async function getUrl( // to this request object const url = parse(href); - const request = { + const headers = options.allowGzip + ? { ...options.headers, 'accept-encoding': 'gzip' } + : { ...options.headers }; + const request: RequestOptions = { protocol: nonnull(url.protocol), hostname: nonnull(url.hostname), port: nonnull(url.port), path: nonnull(url.pathname) + nonnull(url.search), method: options.method || 'GET', - headers: shallowCopy(options.headers || {}), - agent: null, + headers: shallowCopy(headers), + agent: undefined, }; - if (options.allowGzip) { - request.headers['accept-encoding'] = 'gzip'; - } - - let req: http.ClientRequest = null; + let req: http.ClientRequest | undefined; switch (nonnull(url.protocol)) { case 'http:': if (options?.agents?.http) { @@ -116,7 +118,7 @@ export async function getUrl( break; default: /* istanbul ignore next */ - logger.throwError( + return logger.throwError( `unsupported protocol ${url.protocol}`, Logger.errors.UNSUPPORTED_OPERATION, { diff --git a/packages/node/src/ethereum/ethers/web/index.ts b/packages/node/src/ethereum/ethers/web/index.ts index f23a47b4a7..d1c2e1dfdb 100644 --- a/packages/node/src/ethereum/ethers/web/index.ts +++ b/packages/node/src/ethereum/ethers/web/index.ts @@ -23,7 +23,7 @@ function staller(duration: number): Promise { }); } -function bodyify(value: any, type: string): string { +function bodyify(value: any, type: string | null): string | null { if (value == null) { return null; } @@ -117,7 +117,7 @@ function unpercent(value: string): Uint8Array { export function _fetchData( connection: string | ConnectionInfo, body?: Uint8Array, - processFunc?: (value: Uint8Array, response: FetchJsonResponse) => T, + processFunc?: (value: Uint8Array | null, response: FetchJsonResponse) => T, ): Promise { // How many times to retry in the event of a throttle const attemptLimit = @@ -150,7 +150,7 @@ export function _fetchData( const headers: { [key: string]: Header } = {}; - let url: string = null; + let url: string; // @TODO: Allow ConnectionInfo to override some of these values const options: Options = { @@ -225,6 +225,8 @@ export function _fetchData( if (connection.agents != null) { options.agents = connection.agents; } + } else { + throw new Error('invalid connection'); } const reData = new RegExp('^data:([^;:]*)?(;base64)?,(.*)$', 'i'); @@ -285,7 +287,7 @@ export function _fetchData( options.headers = flatHeaders; const runningTimeout = (function () { - let timer: NodeJS.Timer = null; + let timer: NodeJS.Timer | null = null; const promise: Promise = new Promise(function (resolve, reject) { if (timeout) { timer = setTimeout(() => { @@ -319,7 +321,7 @@ export function _fetchData( const runningFetch = (async function () { for (let attempt = 0; attempt < attemptLimit; attempt++) { - let response: GetUrlResponse = null; + let response: GetUrlResponse | null = null; try { response = await getUrl(url, options); @@ -367,18 +369,20 @@ export function _fetchData( } } catch (error) { response = (error).response; - if (response == null) { - runningTimeout.cancel(); - logger.throwError('missing response', Logger.errors.SERVER_ERROR, { + if (response == null) runningTimeout.cancel(); + throw logger.throwError( + 'missing response', + Logger.errors.SERVER_ERROR, + { requestBody: bodyify(options.body, flatHeaders['content-type']), requestMethod: options.method, serverError: error, url: url, - }); - } + }, + ); } - let body = response.body; + let body: Uint8Array | null = response.body; if (allow304 && response.statusCode === 304) { body = null; @@ -387,7 +391,7 @@ export function _fetchData( (response.statusCode < 200 || response.statusCode >= 300) ) { runningTimeout.cancel(); - logger.throwError('bad response', Logger.errors.SERVER_ERROR, { + throw logger.throwError('bad response', Logger.errors.SERVER_ERROR, { status: response.statusCode, headers: response.headers, body: bodyify( @@ -405,7 +409,7 @@ export function _fetchData( const result = await processFunc(body, response); runningTimeout.cancel(); return result; - } catch (error) { + } catch (error: any) { // Allow the processFunc to trigger a throttle if (error.throttleRetry && attempt < attemptLimit) { let tryAgain = true; @@ -424,7 +428,7 @@ export function _fetchData( } runningTimeout.cancel(); - logger.throwError( + throw logger.throwError( 'processing response error', Logger.errors.SERVER_ERROR, { @@ -463,7 +467,10 @@ export function fetchJson( json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any, ): Promise { - let processJsonFunc = (value: Uint8Array, response: FetchJsonResponse) => { + let processJsonFunc = ( + value: Uint8Array | null, + response: FetchJsonResponse, + ) => { let result: any = null; if (value != null) { try { @@ -486,7 +493,7 @@ export function fetchJson( // If we have json to send, we must // - add content-type of application/json (unless already overridden) // - convert the json to bytes - let body: Uint8Array = null; + let body: Uint8Array | undefined; if (json != null) { body = toUtf8Bytes(json); @@ -532,7 +539,7 @@ export function poll( } return new Promise(function (resolve, reject) { - let timer: NodeJS.Timer = null; + let timer: NodeJS.Timer | null = null; let done: boolean = false; // Returns true if cancel was successful. Unsuccessful cancel means we're already done. @@ -547,7 +554,7 @@ export function poll( return true; }; - if (options.timeout) { + if (options?.timeout) { timer = setTimeout(() => { if (cancel()) { reject(new Error('timeout')); @@ -555,7 +562,7 @@ export function poll( }, options.timeout); } - const retryLimit = options.retryLimit; + const retryLimit = options?.retryLimit ?? 0; let attempt = 0; function check() { @@ -566,9 +573,9 @@ export function poll( if (cancel()) { resolve(result); } - } else if (options.oncePoll) { + } else if (options?.oncePoll) { options.oncePoll.once('poll', check); - } else if (options.onceBlock) { + } else if (options?.onceBlock) { options.onceBlock.once('block', check); // Otherwise, exponential back-off (up to 10s) our next request @@ -582,13 +589,13 @@ export function poll( } let timeout = - options.interval * + options!.interval! * parseInt(String(Math.random() * Math.pow(2, attempt))); - if (timeout < options.floor) { - timeout = options.floor; + if (timeout < options!.floor!) { + timeout = options!.floor!; } - if (timeout > options.ceiling) { - timeout = options.ceiling; + if (timeout > options!.ceiling!) { + timeout = options!.ceiling!; } setTimeout(check, timeout); diff --git a/packages/node/src/ethereum/ethers/web/types.ts b/packages/node/src/ethereum/ethers/web/types.ts index 027a33df1f..5d5feb5b00 100644 --- a/packages/node/src/ethereum/ethers/web/types.ts +++ b/packages/node/src/ethereum/ethers/web/types.ts @@ -7,7 +7,7 @@ export type GetUrlResponse = { statusCode: number; statusMessage: string; headers: { [key: string]: string }; - body: Uint8Array; + body: Uint8Array | null; }; export type Options = { diff --git a/packages/node/src/ethereum/utils.ethereum.ts b/packages/node/src/ethereum/utils.ethereum.ts index b48d108181..9677842bb5 100644 --- a/packages/node/src/ethereum/utils.ethereum.ts +++ b/packages/node/src/ethereum/utils.ethereum.ts @@ -43,7 +43,7 @@ export function handleNumber(value: string | number): BigNumber { return BigNumber.from(value); } -export function formatBlock(block: Record): EthereumBlock { +export function formatBlock(block: any): EthereumBlock { return { ...block, difficulty: handleNumber(block.difficulty).toBigInt(), @@ -107,7 +107,7 @@ export function formatLog( export function formatTransaction( tx: Record, block: EthereumBlock, -): EthereumTransaction { +): Omit { return { ...(tx as Partial), from: handleAddress(tx.from), @@ -130,12 +130,11 @@ export function formatTransaction( toJSON(): string { return JSON.stringify(omit(this, ['block', 'receipt', 'toJSON'])); }, - } as EthereumTransaction; + } as Omit; } export function formatReceipt( receipt: Record, - block: EthereumBlock, ): R { return { ...receipt, diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 1171902967..ed7ff6fd6e 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -89,6 +89,6 @@ export class WorkerBlockDispatcherService worker: IndexerWorker, height: number, ): Promise { - await worker.fetchBlock(height, null); + await worker.fetchBlock(height, 0 /* Unused with ethereum*/); } } diff --git a/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.spec.ts b/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.spec.ts index 8304c04d21..bc9bce24da 100644 --- a/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.spec.ts +++ b/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.spec.ts @@ -81,7 +81,7 @@ describe('Eth Dictionary V1', () => { network: { chainId: '336', dictionary: 'https://foo.bar' } as any, } as any, new NodeConfig({} as any, true), - null, + '', ); expect((dictionary as any).chainId).toBe( diff --git a/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.ts index 13ad43ea72..f2703b98fb 100644 --- a/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/ethDictionaryV1.ts @@ -7,7 +7,6 @@ import { DictionaryQueryEntry as DictionaryV1QueryEntry, } from '@subql/types-core'; import { - EthereumBlockFilter, EthereumHandlerKind, EthereumLogFilter, EthereumTransactionFilter, @@ -32,12 +31,17 @@ const CHAIN_ALIASES_URL = const logger = getLogger('dictionary-v1'); export function appendDsOptions( - dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], + dsOptions: + | SubqlEthereumProcessorOptions + | SubqlEthereumProcessorOptions[] + | undefined, conditions: DictionaryQueryCondition[], ): void { const queryAddressLimit = yargsOptions.argv['query-address-limit']; if (Array.isArray(dsOptions)) { - const addresses = dsOptions.map((option) => option.address).filter(Boolean); + const addresses = dsOptions + .map((option) => option.address) + .filter((v): v is string => Boolean(v)); if (addresses.length > queryAddressLimit) { logger.debug( @@ -65,7 +69,7 @@ export function appendDsOptions( function eventFilterToQueryEntry( filter: EthereumLogFilter, - dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], + dsOptions?: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], ): DictionaryV1QueryEntry { const conditions: DictionaryQueryCondition[] = []; appendDsOptions(dsOptions, conditions); @@ -100,7 +104,7 @@ function eventFilterToQueryEntry( function callFilterToQueryEntry( filter: EthereumTransactionFilter, - dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], + dsOptions?: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], ): DictionaryV1QueryEntry { const conditions: DictionaryQueryCondition[] = []; appendDsOptions(dsOptions, conditions); @@ -230,7 +234,7 @@ export class EthDictionaryV1 extends DictionaryV1 { static async create( project: SubqueryProject, nodeConfig: NodeConfig, - dictionaryUrl?: string, + dictionaryUrl: string, ): Promise { /*Some dictionarys for EVM are built with other SDKs as they are chains with an EVM runtime * we maintain a list of aliases so we can map the evmChainId to the genesis hash of the other SDKs diff --git a/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.spec.ts b/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.spec.ts index 81d5349d83..87c7b2d7bb 100644 --- a/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.spec.ts +++ b/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.spec.ts @@ -99,6 +99,7 @@ const mockDs2: EthereumProjectDs[] = [ }, ], }, + processor: { file: '' }, }, ]; @@ -113,7 +114,7 @@ const nodeConfig = new NodeConfig({ function makeBlockHeightMap(mockDs: SubqlDatasource[]): BlockHeightMap { const m = new Map(); mockDs.forEach((ds, index, dataSources) => { - m.set(ds.startBlock, dataSources.slice(0, index + 1)); + m.set(ds.startBlock || 1, dataSources.slice(0, index + 1)); }); return new BlockHeightMap(m); } @@ -276,7 +277,7 @@ describe('eth dictionary v2', () => { const log = logs.find((l) => l.logIndex === 184); expect(log).toBeDefined(); - expect(log.transactionHash).toEqual( + expect(log!.transactionHash).toEqual( '0x5491f3f4b7ca6cc81f992a17e19bc9bafff408518c643c5a254de44b5a7b6d72', ); @@ -445,7 +446,7 @@ describe('buildDictionaryV2QueryEntry', () => { const queryEntry = buildDictionaryV2QueryEntry(ds); // Total 7 handlers were given, 1 is duplicate - expect(queryEntry.logs.length).toBe(6); + expect(queryEntry.logs!.length).toBe(6); }); it('should unique QueryEntry for duplicate dataSources', () => { diff --git a/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts b/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts index 2fe8c02368..f29d77cd56 100644 --- a/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts +++ b/packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts @@ -42,12 +42,14 @@ const MIN_FETCH_LIMIT = 200; const logger = getLogger('dictionary-v2'); function extractOptionAddresses( - dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], + dsOptions?: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], ): string[] { const queryAddressLimit = yargsOptions.argv['query-address-limit']; const addressArray: string[] = []; if (Array.isArray(dsOptions)) { - const addresses = dsOptions.map((option) => option.address).filter(Boolean); + const addresses = dsOptions + .map((option) => option.address) + .filter((address): address is string => Boolean(address)); if (addresses.length > queryAddressLimit) { logger.debug( @@ -67,12 +69,12 @@ function extractOptionAddresses( function callFilterToDictionaryCondition( filter: EthereumTransactionFilter, - dsOptions: SubqlEthereumProcessorOptions, + dsOptions?: SubqlEthereumProcessorOptions, ): EthDictionaryTxConditions { const txConditions: EthDictionaryTxConditions = {}; - const toArray = []; - const fromArray = []; - const funcArray = []; + const toArray: (string | null)[] = []; + const fromArray: string[] = []; + const funcArray: string[] = []; if (filter.from) { fromArray.push(filter.from.toLowerCase()); @@ -118,7 +120,7 @@ function callFilterToDictionaryCondition( function eventFilterToDictionaryCondition( filter: EthereumLogFilter, - dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], + dsOptions?: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[], ): EthDictionaryLogConditions { const logConditions: EthDictionaryLogConditions = {}; logConditions.address = extractOptionAddresses(dsOptions); @@ -146,7 +148,7 @@ function eventFilterToDictionaryCondition( function sanitiseDictionaryConditions( dictionaryConditions: EthDictionaryV2QueryEntry, ): EthDictionaryV2QueryEntry { - if (!dictionaryConditions.logs.length) { + if (!dictionaryConditions.logs?.length) { delete dictionaryConditions.logs; } else { dictionaryConditions.logs = uniqBy(dictionaryConditions.logs, (log) => @@ -154,7 +156,7 @@ function sanitiseDictionaryConditions( ); } - if (!dictionaryConditions.transactions.length) { + if (!dictionaryConditions.transactions?.length) { delete dictionaryConditions.transactions; } else { dictionaryConditions.transactions = uniqBy( @@ -192,6 +194,7 @@ export function buildDictionaryV2QueryEntry( filter.to !== undefined || filter.function !== undefined ) { + dictionaryConditions.transactions ??= []; dictionaryConditions.transactions.push( callFilterToDictionaryCondition(filter, ds.options), ); @@ -202,6 +205,7 @@ export function buildDictionaryV2QueryEntry( } case EthereumHandlerKind.Event: { const filter = handler.filter as EthereumLogFilter; + dictionaryConditions.logs ??= []; if (ds.groupedOptions) { dictionaryConditions.logs.push( eventFilterToDictionaryCondition(filter, ds.groupedOptions), @@ -279,13 +283,15 @@ export class EthDictionaryV2 extends DictionaryV2< const blocks: IBlock[] = ( (data.blocks as RawEthBlock[]) || [] ).map((b) => rawBlockToEthBlock(b, this.api)); + + if (!blocks.length) { + return undefined; + } return { batchBlocks: blocks, - lastBufferedHeight: blocks.length - ? blocks[blocks.length - 1].block.number - : undefined, + lastBufferedHeight: blocks[blocks.length - 1].block.number, }; - } catch (e) { + } catch (e: any) { logger.error(e, `Failed to handle block response}`); throw e; } diff --git a/packages/node/src/indexer/dictionary/v2/types.ts b/packages/node/src/indexer/dictionary/v2/types.ts index ea59dc5ea3..b40b75a158 100644 --- a/packages/node/src/indexer/dictionary/v2/types.ts +++ b/packages/node/src/indexer/dictionary/v2/types.ts @@ -78,7 +78,7 @@ export interface EthDictionaryLogConditions { } export interface EthDictionaryTxConditions { - to?: string[]; + to?: (string | null)[]; from?: string[]; data?: string[]; } diff --git a/packages/node/src/indexer/dictionary/v2/utils.ts b/packages/node/src/indexer/dictionary/v2/utils.ts index 805db01979..edba810926 100644 --- a/packages/node/src/indexer/dictionary/v2/utils.ts +++ b/packages/node/src/indexer/dictionary/v2/utils.ts @@ -35,13 +35,11 @@ export function rawBlockToEthBlock( ...formatTransaction(tx, ethBlock), logs: ethBlock.logs.filter((l) => l.transactionHash === tx.hash), receipt: (): Promise => - api - .getTransactionReceipt(tx.hash) - .then((r) => formatReceipt(r, ethBlock)), + api.getTransactionReceipt(tx.hash).then((r) => formatReceipt(r)), })); return formatBlockUtil(ethBlock); - } catch (e) { + } catch (e: any) { throw new Error( `Convert raw block to Eth block failed at ${block.header.number},${e.message}`, ); diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index af1bc46cab..44dcef30be 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -67,7 +67,7 @@ export class DynamicDsService extends BaseDynamicDsService< } } return dsObj; - } catch (e) { + } catch (e: any) { throw new Error(`Unable to create dynamic datasource.\n ${e.message}`); } } diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 8ef049374b..a347fc3c4a 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -85,7 +85,9 @@ export class FetchService extends BaseFetchService< } protected async initBlockDispatcher(): Promise { - await this.blockDispatcher.init(this.resetForNewDs.bind(this)); + await this.blockDispatcher.init((height) => + Promise.resolve(this.resetForNewDs(height)), + ); } protected async preLoopHook(): Promise { diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index f34d6f31ca..c00c2641ac 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -34,7 +34,7 @@ import { LightEthereumLog, } from '@subql/types-ethereum'; import { EthereumProjectDs } from '../configure/SubqueryProject'; -import { EthereumApi, EthereumApiService } from '../ethereum'; +import { EthereumApi } from '../ethereum'; import { filterBlocksProcessor, filterLogsProcessor, diff --git a/packages/node/src/indexer/project.service.test.ts b/packages/node/src/indexer/project.service.test.ts index 266f2c20dc..1ec28f1e0d 100644 --- a/packages/node/src/indexer/project.service.test.ts +++ b/packages/node/src/indexer/project.service.test.ts @@ -24,18 +24,18 @@ describe('ProjectService', () => { const apiService = mockApiService(); projectService = new ProjectService( - null, + null as any, apiService, - null, - null, - null, - null, - null, - null, + null as any, + null as any, + null as any, + null as any, + null as any, + null as any, {} as any, - null, - null, - null, + null as any, + null as any, + null as any, ); }); diff --git a/packages/node/src/indexer/unfinalizedBlocks.service.ts b/packages/node/src/indexer/unfinalizedBlocks.service.ts index b98df33fab..8954ed4b5b 100644 --- a/packages/node/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/unfinalizedBlocks.service.ts @@ -116,13 +116,16 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService { +async function initWorker(startHeight?: number): Promise { try { const app = await NestFactory.create(WorkerModule, { logger: new NestLogger(!!argv.debug), // TIP: If the worker is crashing comment out this line for better logging @@ -46,7 +46,7 @@ async function initWorker(startHeight: number): Promise { const workerService = app.get(WorkerService); initWorkerServices(app, workerService); - } catch (e) { + } catch (e: any) { console.log('Failed to start worker', e); logger.error(e, 'Failed to start worker'); throw e; diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts index a19d3ee4dd..e6c1539733 100644 --- a/packages/node/src/init.ts +++ b/packages/node/src/init.ts @@ -2,8 +2,13 @@ // SPDX-License-Identifier: GPL-3.0 import { NestFactory } from '@nestjs/core'; -import { findAvailablePort, notifyUpdates } from '@subql/common'; -import { exitWithError, getLogger, NestLogger } from '@subql/node-core'; +import { notifyUpdates } from '@subql/common'; +import { + exitWithError, + getLogger, + getValidPort, + NestLogger, +} from '@subql/node-core'; import { AppModule } from './app.module'; import { FetchService } from './indexer/fetch.service'; import { ProjectService } from './indexer/project.service'; @@ -12,7 +17,6 @@ const pjson = require('../package.json'); const { argv } = yargsOptions; -const DEFAULT_PORT = 3000; const logger = getLogger('subql-node'); notifyUpdates(pjson, logger); @@ -21,20 +25,7 @@ export async function bootstrap(): Promise { logger.info(`Current ${pjson.name} version is ${pjson.version}`); const debug = argv.debug; - const validate = (x: any) => { - const p = parseInt(x); - return isNaN(p) ? null : p; - }; - - const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT)); - if (!port) { - exitWithError( - `Unable to find available port (tried ports in range (${port}..${ - port + 10 - })). Try setting a free port manually by setting the --port flag`, - logger, - ); - } + const port = await getValidPort(argv.port); if (argv.unsafe) { logger.warn( diff --git a/packages/node/src/meta/meta.service.ts b/packages/node/src/meta/meta.service.ts index d5ea8da3b9..7eccfcd052 100644 --- a/packages/node/src/meta/meta.service.ts +++ b/packages/node/src/meta/meta.service.ts @@ -27,7 +27,8 @@ export class MetaService extends BaseMetaService { private lastReportedEnqueueBlocks = 0; private lastReportedFetchBlocks = 0; private lastReportedRpcCalls = 0; - private lastStatsReportedTs: Date; + private lastStatsReportedTs?: Date; + protected packageVersion = packageVersion; constructor( private nodeConfig: NodeConfig, @@ -36,7 +37,6 @@ export class MetaService extends BaseMetaService { super(storeCacheService, nodeConfig); } - protected packageVersion = packageVersion; protected sdkVersion(): { name: string; version: string } { return { name: 'ethersSdkVersion', version: ethersSdkVersion }; } @@ -75,7 +75,7 @@ export class MetaService extends BaseMetaService { @Interval(10000) blockFilteringSpeed(): void { - if (!this.nodeConfig.profiler) { + if (!this.nodeConfig.profiler || !this.lastStatsReportedTs) { return; } const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks; diff --git a/packages/node/src/utils/string.ts b/packages/node/src/utils/string.ts index 3fe7329a93..6e9a1db66e 100644 --- a/packages/node/src/utils/string.ts +++ b/packages/node/src/utils/string.ts @@ -5,7 +5,7 @@ import { EventFragment, FunctionFragment } from '@ethersproject/abi'; import { isHexString, hexStripZeros, hexDataSlice } from '@ethersproject/bytes'; import { id } from '@ethersproject/hash'; -export function stringNormalizedEq(a: string, b: string): boolean { +export function stringNormalizedEq(a: string, b?: string): boolean { return a.toLowerCase() === b?.toLowerCase(); } diff --git a/packages/node/tsconfig.json b/packages/node/tsconfig.json index 0ad2870bfe..de259635c6 100644 --- a/packages/node/tsconfig.json +++ b/packages/node/tsconfig.json @@ -4,7 +4,8 @@ "sourceMap": true, "tsBuildInfoFile": "dist/.tsbuildinfo", "rootDir": "src", - "outDir": "./dist" + "outDir": "./dist", + "strict": true }, "references": [{ "path": "../common-ethereum" }, { "path": "../types" }], "include": ["src/**/*"] diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index 80e025002c..9296e0daf2 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Enable ts strict + ## [3.11.0] - 2024-07-11 ### Changed - Bump with `@subql/types-core` (#2487) diff --git a/packages/types/src/ethereum/interfaces.ts b/packages/types/src/ethereum/interfaces.ts index b6d9aeb13b..b3c45ea04c 100644 --- a/packages/types/src/ethereum/interfaces.ts +++ b/packages/types/src/ethereum/interfaces.ts @@ -22,7 +22,7 @@ export interface EthereumTransactionFilter { * @example * to: '0x220866B1A2219f40e72f5c628B65D54268cA3A9D', **/ - to?: string; + to?: string | null; /** * The function sighash or function signature of the call. This is the first 32bytes of the data field * @example @@ -94,7 +94,7 @@ export type EthereumTransaction = { hash: string; input: string; nonce: bigint; - to: string; + to?: string; transactionIndex: bigint; value: bigint; type: string; diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 8b366d5072..d34a582d18 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -12,6 +12,7 @@ import { SecondLayerHandlerProcessor_0_0_0, SecondLayerHandlerProcessor_1_0_0, DsProcessor, + BaseCustomDataSource, } from '@subql/types-core'; import { EthereumBlock, @@ -236,16 +237,45 @@ export interface SubqlRuntimeDatasource, O = any -> extends ISubqlDatasource { +> extends BaseCustomDataSource /*ISubqlDatasource*/ { + /** + * The kind of the datasource, which is `ethereum/Runtime`. + * @type {K} + */ kind: K; - assets: Map; + /** + * Options to specify details about the contract and its interface + * @example + * options: { + * abi: 'erc20', + * address: '0x220866B1A2219f40e72f5c628B65D54268cA3A9D', + * } + * */ options?: SubqlEthereumProcessorOptions; + /** + * ABI or contract artifact files that are used for decoding. + * These are used for codegen to generate handler inputs and contract interfaces + * @example + * assets: new Map([ + * ['erc721', { file: "./abis/erc721.json" }], + * ['erc1155', { file: "./abis/erc1155.json" }], + * ]) + * */ + assets?: Map; + /** + * @example + * processor: { + * file: './node_modules/@subql/frontier-evm-processor/dist/bundle.js', + * options: { + * abi: 'erc20', + * address: '0x322E86852e492a7Ee17f28a78c663da38FB33bfb', + * } + * } + */ processor: Processor; } diff --git a/yarn.lock b/yarn.lock index dfc1e4e745..46995f6b09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3026,7 +3026,7 @@ __metadata: "@nestjs/testing": ^9.4.0 "@subql/common-ethereum": "workspace:*" "@subql/node-core": ^11.0.0 - "@subql/testing": ^2.1.1 + "@subql/testing": ^2.2.1 "@subql/types-ethereum": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -3051,7 +3051,7 @@ __metadata: languageName: unknown linkType: soft -"@subql/testing@npm:2.2.1": +"@subql/testing@npm:2.2.1, @subql/testing@npm:^2.2.1": version: 2.2.1 resolution: "@subql/testing@npm:2.2.1" dependencies: @@ -3060,15 +3060,6 @@ __metadata: languageName: node linkType: hard -"@subql/testing@npm:^2.1.1": - version: 2.1.1 - resolution: "@subql/testing@npm:2.1.1" - dependencies: - "@subql/types-core": ^0.7.0 - checksum: 6b81d0130d364984e1a565228523b414c2fd66d128b2795161872657546ea83a7e23b42d59f968241f18c114afd7c511886418db8e89679f8a35861e8c24e5a9 - languageName: node - linkType: hard - "@subql/types-core@npm:0.10.0, @subql/types-core@npm:^0.10.0": version: 0.10.0 resolution: "@subql/types-core@npm:0.10.0" @@ -3087,15 +3078,6 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:^0.7.0": - version: 0.7.0 - resolution: "@subql/types-core@npm:0.7.0" - dependencies: - package-json-type: ^1.0.3 - checksum: ba8d140333d315e80c28b44f29f56511302d0179001048e55ef2bbdded51deb7ddd37d77eee3d6fff1c18f2b944f2cd9f4e39da39e9f4bd9216d4e1b4bbf6378 - languageName: node - linkType: hard - "@subql/types-ethereum@workspace:*, @subql/types-ethereum@workspace:packages/types": version: 0.0.0-use.local resolution: "@subql/types-ethereum@workspace:packages/types"