From fd5476db590a68fa0558579c1ad9973d925a9b27 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 19 Mar 2024 12:26:27 +0200 Subject: [PATCH 01/23] update reduce chunk size logic --- src/components/Indexer/crawlerThread.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 8c5ffc8d0..e89617fb5 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -61,7 +61,7 @@ export async function proccesNetworkData(): Promise { // we can override the default value of 30 secs, by setting process.env.INDEXER_INTERVAL const interval = getCrawlingInterval() - + let { chunkSize } = rpcDetails while (true) { const networkHeight = await getNetworkHeight(provider) @@ -76,7 +76,6 @@ export async function proccesNetworkData(): Promise { ) if (networkHeight > startBlock) { - let { chunkSize } = rpcDetails const remainingBlocks = networkHeight - startBlock const blocksToProcess = Math.min(chunkSize, remainingBlocks) INDEXER_LOGGER.logMessage( @@ -94,13 +93,14 @@ export async function proccesNetworkData(): Promise { updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) lastIndexedBlock = processedBlocks.lastBlock + chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize } catch (error) { INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_ERROR, `network: ${rpcDetails.network} Error: ${error.message} `, true ) - chunkSize = Math.floor(chunkSize / 2) + chunkSize = Math.floor(chunkSize / 2) < 1 ? 1 : Math.floor(chunkSize / 2) INDEXER_LOGGER.logMessage( `network: ${rpcDetails.network} Reducing chunk size ${chunkSize} `, true From 134b43adb1e2422fff055d85f3af5c06a232ac4d Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 25 Mar 2024 09:32:04 +0200 Subject: [PATCH 02/23] split indexing logic --- src/components/Indexer/crawlerThread.ts | 36 +++++++++++++++++++------ src/components/Indexer/utils.ts | 19 +++++++++++-- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index e89617fb5..48c923cb8 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -4,7 +4,8 @@ import { getDeployedContractBlock, getNetworkHeight, processBlocks, - processChunkLogs + processChunkLogs, + retrieveChunkEvents } from './utils.js' import { Blockchain } from '../../utils/blockchain.js' import { BlocksEvents, SupportedNetwork } from '../../@types/blockchain.js' @@ -13,6 +14,7 @@ import { sleep } from '../../utils/util.js' import { EVENTS } from '../../utils/index.js' import { INDEXER_LOGGER } from '../../utils/logging/common.js' import { getDatabase } from '../../utils/database.js' +import { Log } from 'ethers' export interface ReindexTask { txId: string @@ -81,23 +83,19 @@ export async function proccesNetworkData(): Promise { INDEXER_LOGGER.logMessage( `network: ${rpcDetails.network} processing ${blocksToProcess} blocks ...` ) - + let chunkEvents: Log[] = [] try { - const processedBlocks = await processBlocks( + chunkEvents = await retrieveChunkEvents( signer, provider, rpcDetails.chainId, startBlock, blocksToProcess ) - updateLastIndexedBlockNumber(processedBlocks.lastBlock) - checkNewlyIndexedAssets(processedBlocks.foundEvents) - lastIndexedBlock = processedBlocks.lastBlock - chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize } catch (error) { INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_ERROR, - `network: ${rpcDetails.network} Error: ${error.message} `, + `Get events for network: ${rpcDetails.network} failure: ${error.message} `, true ) chunkSize = Math.floor(chunkSize / 2) < 1 ? 1 : Math.floor(chunkSize / 2) @@ -106,6 +104,28 @@ export async function proccesNetworkData(): Promise { true ) } + if (chunkEvents && chunkEvents.length > 0) { + try { + const processedBlocks = await processBlocks( + chunkEvents, + signer, + provider, + rpcDetails.chainId, + startBlock, + blocksToProcess + ) + updateLastIndexedBlockNumber(processedBlocks.lastBlock) + checkNewlyIndexedAssets(processedBlocks.foundEvents) + lastIndexedBlock = processedBlocks.lastBlock + chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize + } catch (error) { + INDEXER_LOGGER.log( + LOG_LEVELS_STR.LEVEL_ERROR, + `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, + true + ) + } + } } processReindex() await sleep(interval) diff --git a/src/components/Indexer/utils.ts b/src/components/Indexer/utils.ts index 55deaae49..51ba782c7 100644 --- a/src/components/Indexer/utils.ts +++ b/src/components/Indexer/utils.ts @@ -78,13 +78,13 @@ export const getNetworkHeight = async (provider: JsonRpcApiProvider) => { return networkHeight } -export const processBlocks = async ( +export const retrieveChunkEvents = async ( signer: Signer, provider: JsonRpcApiProvider, network: number, lastIndexedBlock: number, count: number -): Promise => { +): Promise => { try { const eventHashes = Object.keys(EVENT_HASHES) const startIndex = lastIndexedBlock + 1 @@ -93,6 +93,21 @@ export const processBlocks = async ( toBlock: lastIndexedBlock + count, topics: [eventHashes] }) + return blockLogs + } catch (error) { + throw new Error(` Error processing chunk of blocks events ${error.message}`) + } +} + +export const processBlocks = async ( + blockLogs: ethers.Log[], + signer: Signer, + provider: JsonRpcApiProvider, + network: number, + lastIndexedBlock: number, + count: number +): Promise => { + try { const events = await processChunkLogs(blockLogs, signer, provider, network) return { From fd9dca9bba7fa715cb52f316d385ec69de2758f2 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 25 Mar 2024 09:36:16 +0200 Subject: [PATCH 03/23] move check from thread to util --- src/components/Indexer/crawlerThread.ts | 40 ++++++++++++------------- src/components/Indexer/utils.ts | 5 +++- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 48c923cb8..a83c171c5 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -104,27 +104,25 @@ export async function proccesNetworkData(): Promise { true ) } - if (chunkEvents && chunkEvents.length > 0) { - try { - const processedBlocks = await processBlocks( - chunkEvents, - signer, - provider, - rpcDetails.chainId, - startBlock, - blocksToProcess - ) - updateLastIndexedBlockNumber(processedBlocks.lastBlock) - checkNewlyIndexedAssets(processedBlocks.foundEvents) - lastIndexedBlock = processedBlocks.lastBlock - chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize - } catch (error) { - INDEXER_LOGGER.log( - LOG_LEVELS_STR.LEVEL_ERROR, - `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, - true - ) - } + try { + const processedBlocks = await processBlocks( + chunkEvents, + signer, + provider, + rpcDetails.chainId, + startBlock, + blocksToProcess + ) + updateLastIndexedBlockNumber(processedBlocks.lastBlock) + checkNewlyIndexedAssets(processedBlocks.foundEvents) + lastIndexedBlock = processedBlocks.lastBlock + chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize + } catch (error) { + INDEXER_LOGGER.log( + LOG_LEVELS_STR.LEVEL_ERROR, + `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, + true + ) } } processReindex() diff --git a/src/components/Indexer/utils.ts b/src/components/Indexer/utils.ts index 51ba782c7..4cdc73b90 100644 --- a/src/components/Indexer/utils.ts +++ b/src/components/Indexer/utils.ts @@ -108,7 +108,10 @@ export const processBlocks = async ( count: number ): Promise => { try { - const events = await processChunkLogs(blockLogs, signer, provider, network) + const events: any[] | BlocksEvents = + blockLogs && blockLogs.length > 0 + ? await processChunkLogs(blockLogs, signer, provider, network) + : [] return { lastBlock: lastIndexedBlock + count, From 16d21bf03687e1cdbf2240ff34710d07f9dddd6d Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 8 Apr 2024 10:08:50 +0300 Subject: [PATCH 04/23] temp start node at block 5390003 --- src/components/Indexer/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 3417fa0db..f8a5921cf 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -52,11 +52,11 @@ export class OceanIndexer { return network } - public async startThreads(): Promise { + public startThreads(): void { for (const network of this.supportedChains) { const chainId = parseInt(network) const rpcDetails: SupportedNetwork = this.getSupportedNetwork(chainId) - const lastIndexedBlock = await this.getLastIndexedBlock(chainId) + const lastIndexedBlock = 5390003 const workerData = { rpcDetails, lastIndexedBlock } INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_INFO, From 4974708906532e06deb7b9cf0ac5519350c7cfd3 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 8 Apr 2024 10:16:40 +0300 Subject: [PATCH 05/23] update db --- src/components/Indexer/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index f8a5921cf..6092c5f6e 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -52,11 +52,11 @@ export class OceanIndexer { return network } - public startThreads(): void { + public async startThreads(): Promise { for (const network of this.supportedChains) { const chainId = parseInt(network) const rpcDetails: SupportedNetwork = this.getSupportedNetwork(chainId) - const lastIndexedBlock = 5390003 + const lastIndexedBlock = await this.getLastIndexedBlock(chainId) const workerData = { rpcDetails, lastIndexedBlock } INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_INFO, @@ -131,6 +131,7 @@ export class OceanIndexer { public async getLastIndexedBlock(network: number): Promise { const dbconn = this.db.indexer try { + await dbconn.update(network, 5390003) const indexer = await dbconn.retrieve(network) return indexer?.lastIndexedBlock } catch (err) { From 50c70e370f9ba5e8d3b9f1b8757aa5d23882d538 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 8 Apr 2024 14:23:44 +0300 Subject: [PATCH 06/23] debug logs --- src/components/Indexer/crawlerThread.ts | 1 + src/components/Indexer/processor.ts | 3 ++- src/components/Indexer/utils.ts | 11 +++++++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index a83c171c5..fe470544b 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -113,6 +113,7 @@ export async function proccesNetworkData(): Promise { startBlock, blocksToProcess ) + console.log('processed block', processedBlocks.lastBlock) updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) lastIndexedBlock = processedBlocks.lastBlock diff --git a/src/components/Indexer/processor.ts b/src/components/Indexer/processor.ts index ee4c64ed4..a48a7cbc4 100644 --- a/src/components/Indexer/processor.ts +++ b/src/components/Indexer/processor.ts @@ -257,7 +257,8 @@ export class MetadataEventProcessor extends BaseEventProcessor { ): Promise { try { const nftFactoryAddress = getContractAddress(chainId, 'ERC721Factory') - const nftFactoryContract = await getNFTFactory(signer, nftFactoryAddress) + console.log('getNFTFactory signer ', await signer.getAddress()) + const nftFactoryContract = getNFTFactory(signer, nftFactoryAddress) if ( getAddress(await nftFactoryContract.erc721List(event.address)) !== diff --git a/src/components/Indexer/utils.ts b/src/components/Indexer/utils.ts index 4cdc73b90..4c3aedff7 100644 --- a/src/components/Indexer/utils.ts +++ b/src/components/Indexer/utils.ts @@ -118,6 +118,7 @@ export const processBlocks = async ( foundEvents: events } } catch (error) { + console.error('processBlocks err: ', error) throw new Error(` Error processing chunk of blocks events ${error.message}`) } } @@ -244,14 +245,20 @@ export const getNFTFactory = (signer: Signer, address: string): ethers.Contract address = getAddress(address) return getContract(signer, 'ERC721Factory', address) } - function getContract( signer: Signer, contractName: string, address: string ): ethers.Contract { + console.log('getContract ++ ', contractName, address) const abi = getContractDefinition(contractName) - return new ethers.Contract(getAddress(address), abi, signer) // was provider.getSigner() => thow no account + try { + const contract = new ethers.Contract(getAddress(address), abi, signer) // was provider.getSigner() => thow no account + return contract + } catch (err) { + console.error('getContract err: ', err) + throw err + } } function getContractDefinition(contractName: string): any { From cb33aa1e285d20eb5df6688fd8a9ba372208e955 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 8 Apr 2024 14:39:53 +0300 Subject: [PATCH 07/23] refactor get Dt Contract --- src/components/Indexer/processor.ts | 21 +++++++++------------ src/components/Indexer/utils.ts | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/components/Indexer/processor.ts b/src/components/Indexer/processor.ts index a48a7cbc4..e5ad6df64 100644 --- a/src/components/Indexer/processor.ts +++ b/src/components/Indexer/processor.ts @@ -17,7 +17,7 @@ import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templat import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20TemplateEnterprise.sol/ERC20TemplateEnterprise.json' assert { type: 'json' } import { getDatabase } from '../../utils/database.js' import { PROTOCOL_COMMANDS, EVENTS, MetadataStates } from '../../utils/constants.js' -import { getNFTFactory, getContractAddress } from './utils.js' +import { getNFTFactory, getContractAddress, getDtContract } from './utils.js' import { INDEXER_LOGGER } from '../../utils/logging/common.js' import { Purgatory } from './purgatory.js' import { getConfiguration } from '../../utils/index.js' @@ -257,7 +257,6 @@ export class MetadataEventProcessor extends BaseEventProcessor { ): Promise { try { const nftFactoryAddress = getContractAddress(chainId, 'ERC721Factory') - console.log('getNFTFactory signer ', await signer.getAddress()) const nftFactoryContract = getNFTFactory(signer, nftFactoryAddress) if ( @@ -468,6 +467,7 @@ export class OrderStartedEventProcessor extends BaseEventProcessor { async processEvent( event: ethers.Log, chainId: number, + signer: Signer, provider: JsonRpcApiProvider ): Promise { const decodedEventData = await this.getEventData( @@ -483,11 +483,9 @@ export class OrderStartedEventProcessor extends BaseEventProcessor { `Processed new order for service index ${serviceIndex} at ${timestamp}`, true ) - const datatokenContract = new Contract( - event.address, - ERC20Template.abi, - await provider.getSigner() - ) + console.log('getDtContract signer ', await signer.getAddress()) + const datatokenContract = getDtContract(signer, event.address) + const nftAddress = await datatokenContract.getERC721Address() const did = 'did:op:' + @@ -534,6 +532,7 @@ export class OrderReusedEventProcessor extends BaseEventProcessor { async processEvent( event: ethers.Log, chainId: number, + signer: Signer, provider: JsonRpcApiProvider ): Promise { const decodedEventData = await this.getEventData( @@ -546,11 +545,9 @@ export class OrderReusedEventProcessor extends BaseEventProcessor { const payer = decodedEventData.args[1].toString() INDEXER_LOGGER.logMessage(`Processed reused order at ${timestamp}`, true) - const datatokenContract = new Contract( - event.address, - ERC20Template.abi, - await provider.getSigner() - ) + console.log('getDtContract signer ', await signer.getAddress()) + const datatokenContract = getDtContract(signer, event.address) + const nftAddress = await datatokenContract.getERC721Address() const did = 'did:op:' + diff --git a/src/components/Indexer/utils.ts b/src/components/Indexer/utils.ts index 4c3aedff7..9f5021897 100644 --- a/src/components/Indexer/utils.ts +++ b/src/components/Indexer/utils.ts @@ -209,10 +209,20 @@ export const processChunkLogs = async ( storeEvents[event.type] = processExchangeRateChanged() } else if (event.type === EVENTS.ORDER_STARTED) { const processor = getOrderStartedEventProcessor(chainId) - storeEvents[event.type] = await processor.processEvent(log, chainId, provider) + storeEvents[event.type] = await processor.processEvent( + log, + chainId, + signer, + provider + ) } else if (event.type === EVENTS.ORDER_REUSED) { const processor = getOrderReusedEventProcessor(chainId) - storeEvents[event.type] = await processor.processEvent(log, chainId, provider) + storeEvents[event.type] = await processor.processEvent( + log, + chainId, + signer, + provider + ) } else if (event.type === EVENTS.TOKEN_URI_UPDATE) { storeEvents[event.type] = processTokenUriUpadate() } @@ -241,6 +251,11 @@ export const getNFTContract = (signer: Signer, address: string): ethers.Contract return getContract(signer, 'ERC721Template', address) } +export const getDtContract = (signer: Signer, address: string): ethers.Contract => { + address = getAddress(address) + return getContract(signer, 'ERC20Template', address) +} + export const getNFTFactory = (signer: Signer, address: string): ethers.Contract => { address = getAddress(address) return getContract(signer, 'ERC721Factory', address) @@ -250,10 +265,9 @@ function getContract( contractName: string, address: string ): ethers.Contract { - console.log('getContract ++ ', contractName, address) const abi = getContractDefinition(contractName) try { - const contract = new ethers.Contract(getAddress(address), abi, signer) // was provider.getSigner() => thow no account + const contract = new ethers.Contract(getAddress(address), abi, signer) return contract } catch (err) { console.error('getContract err: ', err) @@ -267,6 +281,8 @@ function getContractDefinition(contractName: string): any { return ERC721Factory.abi case 'ERC721Template': return ERC721Template.abi + case 'ERC20Template': + return ERC20Template.abi default: return ERC721Factory.abi } From 26c0cef3c578e6946542900065b62a198dd1e312 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 8 Apr 2024 14:54:47 +0300 Subject: [PATCH 08/23] pre final clean up --- src/components/Indexer/crawlerThread.ts | 3 ++- src/components/Indexer/processor.ts | 2 -- src/components/Indexer/utils.ts | 9 +-------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index fe470544b..14116b42a 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -113,7 +113,6 @@ export async function proccesNetworkData(): Promise { startBlock, blocksToProcess ) - console.log('processed block', processedBlocks.lastBlock) updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) lastIndexedBlock = processedBlocks.lastBlock @@ -124,6 +123,8 @@ export async function proccesNetworkData(): Promise { `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, true ) + updateLastIndexedBlockNumber(startBlock + blocksToProcess) + lastIndexedBlock = startBlock + blocksToProcess } } processReindex() diff --git a/src/components/Indexer/processor.ts b/src/components/Indexer/processor.ts index e5ad6df64..408ec5496 100644 --- a/src/components/Indexer/processor.ts +++ b/src/components/Indexer/processor.ts @@ -483,7 +483,6 @@ export class OrderStartedEventProcessor extends BaseEventProcessor { `Processed new order for service index ${serviceIndex} at ${timestamp}`, true ) - console.log('getDtContract signer ', await signer.getAddress()) const datatokenContract = getDtContract(signer, event.address) const nftAddress = await datatokenContract.getERC721Address() @@ -545,7 +544,6 @@ export class OrderReusedEventProcessor extends BaseEventProcessor { const payer = decodedEventData.args[1].toString() INDEXER_LOGGER.logMessage(`Processed reused order at ${timestamp}`, true) - console.log('getDtContract signer ', await signer.getAddress()) const datatokenContract = getDtContract(signer, event.address) const nftAddress = await datatokenContract.getERC721Address() diff --git a/src/components/Indexer/utils.ts b/src/components/Indexer/utils.ts index 9f5021897..88d0cc003 100644 --- a/src/components/Indexer/utils.ts +++ b/src/components/Indexer/utils.ts @@ -118,7 +118,6 @@ export const processBlocks = async ( foundEvents: events } } catch (error) { - console.error('processBlocks err: ', error) throw new Error(` Error processing chunk of blocks events ${error.message}`) } } @@ -266,13 +265,7 @@ function getContract( address: string ): ethers.Contract { const abi = getContractDefinition(contractName) - try { - const contract = new ethers.Contract(getAddress(address), abi, signer) - return contract - } catch (err) { - console.error('getContract err: ', err) - throw err - } + return new ethers.Contract(getAddress(address), abi, signer) } function getContractDefinition(contractName: string): any { From df232d2007dc8e6f9dbeb89b0f6200fdc5d068e9 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Mon, 8 Apr 2024 14:58:20 +0300 Subject: [PATCH 09/23] remove fixed start block --- src/components/Indexer/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 6092c5f6e..3417fa0db 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -131,7 +131,6 @@ export class OceanIndexer { public async getLastIndexedBlock(network: number): Promise { const dbconn = this.db.indexer try { - await dbconn.update(network, 5390003) const indexer = await dbconn.retrieve(network) return indexer?.lastIndexedBlock } catch (err) { From 99d615dc18f89fddbe1bd4a21bdd3026b84224ff Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 16:12:22 +0300 Subject: [PATCH 10/23] added lock until finishing processing started blocks chunk --- src/components/Indexer/crawlerThread.ts | 122 +++++++++++++----------- 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 14116b42a..b1fe50a6f 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -64,70 +64,82 @@ export async function proccesNetworkData(): Promise { // we can override the default value of 30 secs, by setting process.env.INDEXER_INTERVAL const interval = getCrawlingInterval() let { chunkSize } = rpcDetails - while (true) { - const networkHeight = await getNetworkHeight(provider) + let lockProccessing = false - const startBlock = - lastIndexedBlock && lastIndexedBlock > deployedBlock - ? lastIndexedBlock - : deployedBlock + while (true) { + if (!lockProccessing) { + lockProccessing = false + const networkHeight = await getNetworkHeight(provider) - INDEXER_LOGGER.logMessage( - `network: ${rpcDetails.network} Start block ${startBlock} network height ${networkHeight}`, - true - ) + const startBlock = + lastIndexedBlock && lastIndexedBlock > deployedBlock + ? lastIndexedBlock + : deployedBlock - if (networkHeight > startBlock) { - const remainingBlocks = networkHeight - startBlock - const blocksToProcess = Math.min(chunkSize, remainingBlocks) INDEXER_LOGGER.logMessage( - `network: ${rpcDetails.network} processing ${blocksToProcess} blocks ...` + `network: ${rpcDetails.network} Start block ${startBlock} network height ${networkHeight}`, + true ) - let chunkEvents: Log[] = [] - try { - chunkEvents = await retrieveChunkEvents( - signer, - provider, - rpcDetails.chainId, - startBlock, - blocksToProcess - ) - } catch (error) { - INDEXER_LOGGER.log( - LOG_LEVELS_STR.LEVEL_ERROR, - `Get events for network: ${rpcDetails.network} failure: ${error.message} `, - true - ) - chunkSize = Math.floor(chunkSize / 2) < 1 ? 1 : Math.floor(chunkSize / 2) + + if (networkHeight > startBlock) { + const remainingBlocks = networkHeight - startBlock + const blocksToProcess = Math.min(chunkSize, remainingBlocks) INDEXER_LOGGER.logMessage( - `network: ${rpcDetails.network} Reducing chunk size ${chunkSize} `, - true - ) - } - try { - const processedBlocks = await processBlocks( - chunkEvents, - signer, - provider, - rpcDetails.chainId, - startBlock, - blocksToProcess - ) - updateLastIndexedBlockNumber(processedBlocks.lastBlock) - checkNewlyIndexedAssets(processedBlocks.foundEvents) - lastIndexedBlock = processedBlocks.lastBlock - chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize - } catch (error) { - INDEXER_LOGGER.log( - LOG_LEVELS_STR.LEVEL_ERROR, - `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, - true + `network: ${rpcDetails.network} processing ${blocksToProcess} blocks ...` ) - updateLastIndexedBlockNumber(startBlock + blocksToProcess) - lastIndexedBlock = startBlock + blocksToProcess + let chunkEvents: Log[] = [] + try { + chunkEvents = await retrieveChunkEvents( + signer, + provider, + rpcDetails.chainId, + startBlock, + blocksToProcess + ) + } catch (error) { + INDEXER_LOGGER.log( + LOG_LEVELS_STR.LEVEL_ERROR, + `Get events for network: ${rpcDetails.network} failure: ${error.message} `, + true + ) + chunkSize = Math.floor(chunkSize / 2) < 1 ? 1 : Math.floor(chunkSize / 2) + INDEXER_LOGGER.logMessage( + `network: ${rpcDetails.network} Reducing chunk size ${chunkSize} `, + true + ) + } + try { + const processedBlocks = await processBlocks( + chunkEvents, + signer, + provider, + rpcDetails.chainId, + startBlock, + blocksToProcess + ) + updateLastIndexedBlockNumber(processedBlocks.lastBlock) + checkNewlyIndexedAssets(processedBlocks.foundEvents) + lastIndexedBlock = processedBlocks.lastBlock + lockProccessing = false + chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize + } catch (error) { + INDEXER_LOGGER.log( + LOG_LEVELS_STR.LEVEL_ERROR, + `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, + true + ) + updateLastIndexedBlockNumber(startBlock + blocksToProcess) + lastIndexedBlock = startBlock + blocksToProcess + lockProccessing = false + } } + processReindex() + } else { + INDEXER_LOGGER.logMessage( + `Processing already in progress for network ${rpcDetails.network} waiting untill finishing the current processing ...`, + true + ) } - processReindex() await sleep(interval) } } From f735619ab435716973ea9a698c3c47e6667e11f2 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 16:31:41 +0300 Subject: [PATCH 11/23] update lock condition --- src/components/Indexer/crawlerThread.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index b1fe50a6f..828e69e31 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -68,7 +68,7 @@ export async function proccesNetworkData(): Promise { while (true) { if (!lockProccessing) { - lockProccessing = false + lockProccessing = true const networkHeight = await getNetworkHeight(provider) const startBlock = From 2a7a73d0f8db5bda98f85dc5dc627ed1ec9d0243 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 17:10:23 +0300 Subject: [PATCH 12/23] added some logs --- src/components/Indexer/crawlerThread.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 828e69e31..e66e2db3b 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -67,6 +67,9 @@ export async function proccesNetworkData(): Promise { let lockProccessing = false while (true) { + console.log('lockProcessing == ', lockProccessing) + console.log('lockProcessing == ', lastIndexedBlock) + console.log('chunkSize == ', chunkSize) if (!lockProccessing) { lockProccessing = true const networkHeight = await getNetworkHeight(provider) From 705048e565dbe4216e6e29d8c69dc78e8ec44082 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 17:25:05 +0300 Subject: [PATCH 13/23] remove polygon mock network for integration tests --- src/test/utils/utils.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/utils/utils.ts b/src/test/utils/utils.ts index bb2d50cd9..e45826687 100644 --- a/src/test/utils/utils.ts +++ b/src/test/utils/utils.ts @@ -138,12 +138,6 @@ export function getMockSupportedNetworks(): RPCS { network: 'development', rpc: 'http://127.0.0.1:8545', chunkSize: 100 - }, - '137': { - chainId: 137, - network: 'polygon', - rpc: 'https://polygon-rpc.com', - chunkSize: 1000 } } return mockSupportedNetworks From 3818adb78f2899218849e10a0fc4a58ab9bde428 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 17:46:14 +0300 Subject: [PATCH 14/23] add more unlock processing methods --- src/components/Indexer/crawlerThread.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index e66e2db3b..f4c4d388b 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -68,7 +68,7 @@ export async function proccesNetworkData(): Promise { while (true) { console.log('lockProcessing == ', lockProccessing) - console.log('lockProcessing == ', lastIndexedBlock) + console.log('lastIndexedBlock == ', lastIndexedBlock) console.log('chunkSize == ', chunkSize) if (!lockProccessing) { lockProccessing = true @@ -135,6 +135,8 @@ export async function proccesNetworkData(): Promise { lastIndexedBlock = startBlock + blocksToProcess lockProccessing = false } + } else { + lockProccessing = false } processReindex() } else { From 29c7e22fbd1d8715342327cf62fd717f352daa20 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 17:56:11 +0300 Subject: [PATCH 15/23] clean ups --- src/components/Indexer/crawlerThread.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index f4c4d388b..9e867aa1d 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -67,9 +67,6 @@ export async function proccesNetworkData(): Promise { let lockProccessing = false while (true) { - console.log('lockProcessing == ', lockProccessing) - console.log('lastIndexedBlock == ', lastIndexedBlock) - console.log('chunkSize == ', chunkSize) if (!lockProccessing) { lockProccessing = true const networkHeight = await getNetworkHeight(provider) From 0033449c92a26a1f67dc3cbbf5d56d6a11aa66f6 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 18:13:52 +0300 Subject: [PATCH 16/23] added more debug log --- src/components/Indexer/crawlerThread.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 9e867aa1d..9909a7a8d 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -67,10 +67,12 @@ export async function proccesNetworkData(): Promise { let lockProccessing = false while (true) { + console.log('lockProcessing == ', lockProccessing) + console.log('lastIndexedBlock == ', lastIndexedBlock) if (!lockProccessing) { lockProccessing = true const networkHeight = await getNetworkHeight(provider) - + console.log('networkHeight == ', networkHeight) const startBlock = lastIndexedBlock && lastIndexedBlock > deployedBlock ? lastIndexedBlock @@ -83,7 +85,9 @@ export async function proccesNetworkData(): Promise { if (networkHeight > startBlock) { const remainingBlocks = networkHeight - startBlock + console.log('remainingBlocks == ', remainingBlocks) const blocksToProcess = Math.min(chunkSize, remainingBlocks) + console.log('blocksToProcess == ', blocksToProcess) INDEXER_LOGGER.logMessage( `network: ${rpcDetails.network} processing ${blocksToProcess} blocks ...` ) From 6daef1189854c44f9d061a76f5ca45dcba92ad48 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 18:28:33 +0300 Subject: [PATCH 17/23] small tweak --- src/components/Indexer/crawlerThread.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 9909a7a8d..a58e60ea5 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -121,7 +121,7 @@ export async function proccesNetworkData(): Promise { startBlock, blocksToProcess ) - updateLastIndexedBlockNumber(processedBlocks.lastBlock) + await updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) lastIndexedBlock = processedBlocks.lastBlock lockProccessing = false @@ -132,14 +132,12 @@ export async function proccesNetworkData(): Promise { `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, true ) - updateLastIndexedBlockNumber(startBlock + blocksToProcess) + await updateLastIndexedBlockNumber(startBlock + blocksToProcess) lastIndexedBlock = startBlock + blocksToProcess lockProccessing = false } - } else { - lockProccessing = false } - processReindex() + await processReindex() } else { INDEXER_LOGGER.logMessage( `Processing already in progress for network ${rpcDetails.network} waiting untill finishing the current processing ...`, From f75881d09e1cb50abf7e4a8af18ff6805b4ca248 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 23:16:55 +0300 Subject: [PATCH 18/23] update loc --- src/components/Indexer/crawlerThread.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index a58e60ea5..86f2706b3 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -124,7 +124,6 @@ export async function proccesNetworkData(): Promise { await updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) lastIndexedBlock = processedBlocks.lastBlock - lockProccessing = false chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize } catch (error) { INDEXER_LOGGER.log( @@ -134,10 +133,14 @@ export async function proccesNetworkData(): Promise { ) await updateLastIndexedBlockNumber(startBlock + blocksToProcess) lastIndexedBlock = startBlock + blocksToProcess - lockProccessing = false } } await processReindex() + lockProccessing = false + INDEXER_LOGGER.logMessage( + `Finished processing blocks ${startBlock} to ${lastIndexedBlock} for network ${rpcDetails.network}`, + true + ) } else { INDEXER_LOGGER.logMessage( `Processing already in progress for network ${rpcDetails.network} waiting untill finishing the current processing ...`, From ccef8ff0ba998b3d6be4496fa789605c999f1943 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Tue, 9 Apr 2024 23:30:43 +0300 Subject: [PATCH 19/23] one more update --- src/components/Indexer/crawlerThread.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 86f2706b3..c3040f330 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -35,7 +35,7 @@ const blockchain = new Blockchain(rpcDetails.rpc, rpcDetails.chainId) const provider = blockchain.getProvider() const signer = blockchain.getSigner() -async function updateLastIndexedBlockNumber(block: number): Promise { +async function updateLastIndexedBlockNumber(block: number): Promise { try { const { indexer } = await getDatabase() const updatedIndex = await indexer.update(rpcDetails.chainId, block) @@ -43,6 +43,7 @@ async function updateLastIndexedBlockNumber(block: number): Promise { `New last indexed block : ${updatedIndex.lastIndexedBlock}`, true ) + return updatedIndex.lastIndexedBlock } catch (err) { INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_ERROR, @@ -121,9 +122,8 @@ export async function proccesNetworkData(): Promise { startBlock, blocksToProcess ) - await updateLastIndexedBlockNumber(processedBlocks.lastBlock) + lastIndexedBlock = await updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) - lastIndexedBlock = processedBlocks.lastBlock chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize } catch (error) { INDEXER_LOGGER.log( @@ -131,8 +131,9 @@ export async function proccesNetworkData(): Promise { `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, true ) - await updateLastIndexedBlockNumber(startBlock + blocksToProcess) - lastIndexedBlock = startBlock + blocksToProcess + lastIndexedBlock = await updateLastIndexedBlockNumber( + startBlock + blocksToProcess + ) } } await processReindex() From 5d0d7bc1ec3c62c33abe373096de8926fedfdf6b Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Wed, 10 Apr 2024 08:36:59 +0300 Subject: [PATCH 20/23] try using db indexed block --- src/components/Indexer/crawlerThread.ts | 28 +++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index c3040f330..2188e158a 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -52,6 +52,22 @@ async function updateLastIndexedBlockNumber(block: number): Promise { ) } } + +async function getLastIndexedBlock(): Promise { + const { indexer } = await getDatabase() + try { + const networkDetails = await indexer.retrieve(rpcDetails.chainId) + return networkDetails?.lastIndexedBlock + } catch (err) { + INDEXER_LOGGER.log( + LOG_LEVELS_STR.LEVEL_ERROR, + 'Error retrieving last indexed block', + true + ) + return null + } +} + export async function proccesNetworkData(): Promise { const deployedBlock = getDeployedContractBlock(rpcDetails.chainId) if (deployedBlock == null && lastIndexedBlock == null) { @@ -66,18 +82,18 @@ export async function proccesNetworkData(): Promise { const interval = getCrawlingInterval() let { chunkSize } = rpcDetails let lockProccessing = false - while (true) { console.log('lockProcessing == ', lockProccessing) - console.log('lastIndexedBlock == ', lastIndexedBlock) + console.log('indexedBlock == ', await getLastIndexedBlock()) + console.log(' lastIndexedBlock == ', lastIndexedBlock) + if (!lockProccessing) { lockProccessing = true + const indexedBlock = await getLastIndexedBlock() const networkHeight = await getNetworkHeight(provider) - console.log('networkHeight == ', networkHeight) + const startBlock = - lastIndexedBlock && lastIndexedBlock > deployedBlock - ? lastIndexedBlock - : deployedBlock + indexedBlock && indexedBlock > deployedBlock ? indexedBlock : deployedBlock INDEXER_LOGGER.logMessage( `network: ${rpcDetails.network} Start block ${startBlock} network height ${networkHeight}`, From 66c030289dfa1d4d264ac934578f65f0b67196b8 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Wed, 10 Apr 2024 09:02:12 +0300 Subject: [PATCH 21/23] cleanups --- src/components/Indexer/crawlerThread.ts | 17 +++++------------ src/components/Indexer/index.ts | 19 ++----------------- src/test/unit/indexer/indexer.test.ts | 1 - 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index 2188e158a..f83976e4f 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -26,10 +26,9 @@ const REINDEX_QUEUE: ReindexTask[] = [] interface ThreadData { rpcDetails: SupportedNetwork - lastIndexedBlock: number } -let { rpcDetails, lastIndexedBlock } = workerData as ThreadData +const { rpcDetails } = workerData as ThreadData const blockchain = new Blockchain(rpcDetails.rpc, rpcDetails.chainId) const provider = blockchain.getProvider() @@ -70,7 +69,7 @@ async function getLastIndexedBlock(): Promise { export async function proccesNetworkData(): Promise { const deployedBlock = getDeployedContractBlock(rpcDetails.chainId) - if (deployedBlock == null && lastIndexedBlock == null) { + if (deployedBlock == null && (await getLastIndexedBlock()) == null) { INDEXER_LOGGER.logMessage( `chain: ${rpcDetails.chainId} Both deployed block and last indexed block are null. Cannot proceed further on this chain`, true @@ -83,10 +82,6 @@ export async function proccesNetworkData(): Promise { let { chunkSize } = rpcDetails let lockProccessing = false while (true) { - console.log('lockProcessing == ', lockProccessing) - console.log('indexedBlock == ', await getLastIndexedBlock()) - console.log(' lastIndexedBlock == ', lastIndexedBlock) - if (!lockProccessing) { lockProccessing = true const indexedBlock = await getLastIndexedBlock() @@ -138,7 +133,7 @@ export async function proccesNetworkData(): Promise { startBlock, blocksToProcess ) - lastIndexedBlock = await updateLastIndexedBlockNumber(processedBlocks.lastBlock) + await updateLastIndexedBlockNumber(processedBlocks.lastBlock) checkNewlyIndexedAssets(processedBlocks.foundEvents) chunkSize = chunkSize !== 1 ? chunkSize : rpcDetails.chunkSize } catch (error) { @@ -147,15 +142,13 @@ export async function proccesNetworkData(): Promise { `Processing event from network failed network: ${rpcDetails.network} Error: ${error.message} `, true ) - lastIndexedBlock = await updateLastIndexedBlockNumber( - startBlock + blocksToProcess - ) + await updateLastIndexedBlockNumber(startBlock + blocksToProcess) } } await processReindex() lockProccessing = false INDEXER_LOGGER.logMessage( - `Finished processing blocks ${startBlock} to ${lastIndexedBlock} for network ${rpcDetails.network}`, + `Finished processing blocks ${startBlock} to ${indexedBlock} for network ${rpcDetails.network}`, true ) } else { diff --git a/src/components/Indexer/index.ts b/src/components/Indexer/index.ts index 3417fa0db..3f505031f 100644 --- a/src/components/Indexer/index.ts +++ b/src/components/Indexer/index.ts @@ -52,12 +52,12 @@ export class OceanIndexer { return network } + // eslint-disable-next-line require-await public async startThreads(): Promise { for (const network of this.supportedChains) { const chainId = parseInt(network) const rpcDetails: SupportedNetwork = this.getSupportedNetwork(chainId) - const lastIndexedBlock = await this.getLastIndexedBlock(chainId) - const workerData = { rpcDetails, lastIndexedBlock } + const workerData = { rpcDetails } INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_INFO, `Starting worker for network ${network} with ${JSON.stringify(workerData)}`, @@ -128,21 +128,6 @@ export class OceanIndexer { } } - public async getLastIndexedBlock(network: number): Promise { - const dbconn = this.db.indexer - try { - const indexer = await dbconn.retrieve(network) - return indexer?.lastIndexedBlock - } catch (err) { - INDEXER_LOGGER.log( - LOG_LEVELS_STR.LEVEL_ERROR, - 'Error retrieving last indexed block', - true - ) - return null - } - } - public getIndexingQueue(): ReindexTask[] { return INDEXING_QUEUE.slice() } diff --git a/src/test/unit/indexer/indexer.test.ts b/src/test/unit/indexer/indexer.test.ts index 36f8af16c..b7c7d5fa4 100644 --- a/src/test/unit/indexer/indexer.test.ts +++ b/src/test/unit/indexer/indexer.test.ts @@ -27,7 +27,6 @@ describe('OceanIndexer', () => { } stub(oceanIndexer as any, 'startThreads').callsFake(() => { - oceanIndexer.getLastIndexedBlock = stub().resolves(0) oceanIndexer.startThreads = async () => { try { const network = '1' From a5ed8025612539bcc17d3eca0fce3e459ad123ae Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Wed, 10 Apr 2024 09:03:16 +0300 Subject: [PATCH 22/23] more cleanups --- src/components/Indexer/crawlerThread.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index f83976e4f..a7add75b2 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -97,9 +97,7 @@ export async function proccesNetworkData(): Promise { if (networkHeight > startBlock) { const remainingBlocks = networkHeight - startBlock - console.log('remainingBlocks == ', remainingBlocks) const blocksToProcess = Math.min(chunkSize, remainingBlocks) - console.log('blocksToProcess == ', blocksToProcess) INDEXER_LOGGER.logMessage( `network: ${rpcDetails.network} processing ${blocksToProcess} blocks ...` ) From 5abd0530fca0da93ba5d845ad7044915e9c226f2 Mon Sep 17 00:00:00 2001 From: Bogdan Fazakas Date: Wed, 10 Apr 2024 18:18:43 +0300 Subject: [PATCH 23/23] remove log --- src/components/Indexer/crawlerThread.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/components/Indexer/crawlerThread.ts b/src/components/Indexer/crawlerThread.ts index a7add75b2..3495cb7a2 100644 --- a/src/components/Indexer/crawlerThread.ts +++ b/src/components/Indexer/crawlerThread.ts @@ -145,10 +145,6 @@ export async function proccesNetworkData(): Promise { } await processReindex() lockProccessing = false - INDEXER_LOGGER.logMessage( - `Finished processing blocks ${startBlock} to ${indexedBlock} for network ${rpcDetails.network}`, - true - ) } else { INDEXER_LOGGER.logMessage( `Processing already in progress for network ${rpcDetails.network} waiting untill finishing the current processing ...`,