Blockchain service

packages/node-core/CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Changed
 - Update copyright header to 2025
+- Implement new BlockchainService architecture for easier integration of other blockchains (#2517)
 
 ### Fixed
 - Various typos

packages/node-core/src/blockchain.service.ts

@@ -0,0 +1,76 @@
+// Copyright 2020-2025 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import {BaseCustomDataSource, BaseDataSource, IProjectNetworkConfig} from '@subql/types-core';
+import {DatasourceParams, Header, IBaseIndexerWorker, IBlock, ISubqueryProject} from './indexer';
+
+export interface ICoreBlockchainService<
+  DS extends BaseDataSource = BaseDataSource,
+  SubQueryProject extends ISubqueryProject<IProjectNetworkConfig, DS> = ISubqueryProject<IProjectNetworkConfig, DS>,
+> {
+  /* The semver of the node */
+  packageVersion: string;
+
+  // Project service
+  onProjectChange(project: SubQueryProject): Promise<void> | void;
+  /* Not all networks have a block timestamp, e.g. Shiden */
+  getBlockTimestamp(height: number): Promise<Date | undefined>;
+}
+
+export interface IBlockchainService<
+  DS extends BaseDataSource = BaseDataSource,
+  CDS extends DS & BaseCustomDataSource = BaseCustomDataSource & DS,
+  SubQueryProject extends ISubqueryProject<IProjectNetworkConfig, DS> = ISubqueryProject<IProjectNetworkConfig, DS>,
+  SafeAPI = any,
+  LightBlock = any,
+  FullBlock = any,
+  Worker extends IBaseIndexerWorker = IBaseIndexerWorker,
+> extends ICoreBlockchainService<DS, SubQueryProject> {
+  blockHandlerKind: string;
+
+  // Block dispatcher service
+  fetchBlocks(blockNums: number[]): Promise<IBlock<LightBlock>[] | IBlock<FullBlock>[]>; // TODO this probably needs to change to get light block type correct
+  /* This is the worker equivalent of fetchBlocks, it provides a context to allow syncing anything between workers */
+  fetchBlockWorker(worker: Worker, blockNum: number, context: {workers: Worker[]}): Promise<Header>;
+
+  // /* Not all networks have a block timestamp, e.g. Shiden */
+  // getBlockTimestamp(height: number): Promise<Date | undefined>;
+
+  // Block dispatcher
+  /* Gets the size of the block, used to calculate a median */
+  getBlockSize(block: IBlock): number;
+
+  // Fetch service
+  /**
+   * The finalized header. If the chain doesn't have concrete finalization this could be a probablilistic finalization
+   * */
+  getFinalizedHeader(): Promise<Header>;
+  /**
+   * Gets the latest height of the chain, this should be unfinalized.
+   * Or if the chain has instant finalization this would be the same as the finalized height.
+   * */
+  getBestHeight(): Promise<number>;
+  /**
+   *  The chain interval in milliseconds, if it is not consistent then provide a best estimate
+   * */
+  getChainInterval(): Promise<number>;
+
+  // Unfinalized blocks
+  getHeaderForHash(hash: string): Promise<Header>;
+  getHeaderForHeight(height: number): Promise<Header>;
+
+  // Dynamic Ds sevice
+  /**
+   * Applies and validates parameters to a template DS
+   * */
+  updateDynamicDs(params: DatasourceParams, template: DS | CDS): Promise<void>;
+
+  isCustomDs: (x: DS | CDS) => x is CDS;
+  isRuntimeDs: (x: DS | CDS) => x is DS;
+
+  // Indexer manager
+  /**
+   * Gets an API instance to a specific height so any state queries return data as represented at that height.
+   * */
+  getSafeApi(block: LightBlock | FullBlock): Promise<SafeAPI>;
+}

packages/node-core/src/index.ts

@@ -17,3 +17,4 @@ export * from './indexer';
 export * from './subcommands';
 export * from './yargs';
 export * from './admin';
+export * from './blockchain.service';

packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts

@@ -26,6 +26,7 @@ export type ProcessBlockResponse = {
 };
 
 export interface IBlockDispatcher<B> {
+  init(onDynamicDsCreated: (height: number) => void): Promise<void>;
   // now within enqueueBlock should handle getLatestBufferHeight
   enqueueBlocks(heights: (IBlock<B> | number)[], latestBufferHeight: number): void | Promise<void>;
   queueSize: number;

packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts

@@ -1,21 +1,22 @@
 // Copyright 2020-2025 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {OnApplicationShutdown} from '@nestjs/common';
-import {EventEmitter2} from '@nestjs/event-emitter';
-import {Interval} from '@nestjs/schedule';
-import {NodeConfig} from '../../configure';
-import {IProjectUpgradeService} from '../../configure/ProjectUpgrade.service';
-import {IndexerEvent} from '../../events';
-import {getBlockHeight, IBlock, PoiSyncService} from '../../indexer';
-import {getLogger} from '../../logger';
-import {exitWithError, monitorWrite} from '../../process';
-import {profilerWrap} from '../../profiler';
-import {Queue, AutoQueue, RampQueue, delay, isTaskFlushedError} from '../../utils';
-import {StoreService} from '../store.service';
-import {IStoreModelProvider} from '../storeModelProvider';
-import {IProjectService, ISubqueryProject} from '../types';
-import {BaseBlockDispatcher, ProcessBlockResponse} from './base-block-dispatcher';
+import { OnApplicationShutdown } from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { Interval } from '@nestjs/schedule';
+import { BaseDataSource } from '@subql/types-core';
+import { IBlockchainService } from '../../blockchain.service';
+import { NodeConfig } from '../../configure';
+import { IProjectUpgradeService } from '../../configure/ProjectUpgrade.service';
+import { IndexerEvent } from '../../events';
+import { getBlockHeight, IBlock, PoiSyncService, StoreService } from '../../indexer';
+import { getLogger } from '../../logger';
+import { exitWithError, monitorWrite } from '../../process';
+import { profilerWrap } from '../../profiler';
+import { Queue, AutoQueue, RampQueue, delay, isTaskFlushedError } from '../../utils';
+import { IStoreModelProvider } from '../storeModelProvider';
+import { IIndexerManager, IProjectService, ISubqueryProject } from '../types';
+import { BaseBlockDispatcher } from './base-block-dispatcher';
 
 const logger = getLogger('BlockDispatcherService');
 
@@ -24,10 +25,9 @@ type BatchBlockFetcher<B> = (heights: number[]) => Promise<IBlock<B>[]>;
 /**
  * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing
  */
-export abstract class BlockDispatcher<B, DS>
+export class BlockDispatcher<B, DS extends BaseDataSource>
   extends BaseBlockDispatcher<Queue<IBlock<B> | number>, DS, B>
-  implements OnApplicationShutdown
-{
+  implements OnApplicationShutdown {
   private fetchQueue: AutoQueue<IBlock<B>>;
   private processQueue: AutoQueue<void>;
 
@@ -36,9 +36,6 @@ export abstract class BlockDispatcher<B, DS>
   private fetching = false;
   private isShutdown = false;
 
-  protected abstract getBlockSize(block: IBlock<B>): number;
-  protected abstract indexBlock(block: IBlock<B>): Promise<ProcessBlockResponse>;
-
   constructor(
     nodeConfig: NodeConfig,
     eventEmitter: EventEmitter2,
@@ -48,7 +45,8 @@ export abstract class BlockDispatcher<B, DS>
     storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     project: ISubqueryProject,
-    fetchBlocksBatches: BatchBlockFetcher<B>
+    blockchainService: IBlockchainService<DS>,
+    private indexerManager: IIndexerManager<B, DS>
   ) {
     super(
       nodeConfig,
@@ -63,16 +61,20 @@ export abstract class BlockDispatcher<B, DS>
     );
     this.processQueue = new AutoQueue(nodeConfig.batchSize * 3, 1, nodeConfig.timeout, 'Process');
     this.fetchQueue = new RampQueue(
-      this.getBlockSize.bind(this),
+      blockchainService.getBlockSize.bind(this),
       nodeConfig.batchSize,
       nodeConfig.batchSize * 3,
       nodeConfig.timeout,
       'Fetch'
     );
     if (this.nodeConfig.profiler) {
-      this.fetchBlocksBatches = profilerWrap(fetchBlocksBatches, 'BlockDispatcher', 'fetchBlocksBatches');
+      this.fetchBlocksBatches = profilerWrap(
+        blockchainService.fetchBlocks.bind(blockchainService),
+        'BlockDispatcher',
+        'fetchBlocksBatches'
+      );
     } else {
-      this.fetchBlocksBatches = fetchBlocksBatches;
+      this.fetchBlocksBatches = blockchainService.fetchBlocks.bind(blockchainService);
     }
   }
 
@@ -161,7 +163,10 @@ export abstract class BlockDispatcher<B, DS>
                   await this.preProcessBlock(header);
                   monitorWrite(`Processing from main thread`);
                   // Inject runtimeVersion here to enhance api.at preparation
-                  const processBlockResponse = await this.indexBlock(block);
+                  const processBlockResponse = await this.indexerManager.indexBlock(
+                    block,
+                    await this.projectService.getDataSources(block.getHeader().blockHeight)
+                  );
                   await this.postProcessBlock(header, processBlockResponse);
                   //set block to null for garbage collection
                   (block as any) = null;
@@ -172,8 +177,7 @@ export abstract class BlockDispatcher<B, DS>
                   }
                   logger.error(
                     e,
-                    `Failed to index block at height ${header.blockHeight} ${
-                      e.handler ? `${e.handler}(${e.stack ?? ''})` : ''
+                    `Failed to index block at height ${header.blockHeight} ${e.handler ? `${e.handler}(${e.stack ?? ''})` : ''
                     }`
                   );
                   throw e;
@@ -194,7 +198,7 @@ export abstract class BlockDispatcher<B, DS>
               // Do nothing, fetching the block was flushed, this could be caused by forked blocks or dynamic datasources
               return;
             }
-            exitWithError(new Error(`Failed to enqueue fetched block to process`, {cause: e}), logger);
+            exitWithError(new Error(`Failed to enqueue fetched block to process`, { cause: e }), logger);
           });
 
         this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {
@@ -203,7 +207,7 @@ export abstract class BlockDispatcher<B, DS>
       }
     } catch (e: any) {
       if (!this.isShutdown) {
-        exitWithError(new Error(`Failed to process blocks from queue`, {cause: e}), logger);
+        exitWithError(new Error(`Failed to process blocks from queue`, { cause: e }), logger);
       }
     } finally {
       this.fetching = false;