subquery · jiqiang90 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023
diff --git a/packages/common-algorand/CHANGELOG.md b/packages/common-algorand/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Update `@subql/common` and relevant changes to support endBlock feature (#83)
 
 ## [3.0.2] - 2023-10-26
 ### Changed

diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/types-algorand": "workspace:*",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",

diff --git a/packages/common-algorand/src/project/models.ts b/packages/common-algorand/src/project/models.ts
@@ -1,7 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {ProcessorImpl} from '@subql/common';
+import {BaseDataSource, ProcessorImpl} from '@subql/common';
 import {
   CustomDataSourceAsset as AlgorandCustomDataSourceAsset,
   AlgorandBlockFilter,
@@ -11,7 +11,6 @@ import {
   AlgorandHandlerKind,
   AlgorandRuntimeDataSource,
   AlgorandRuntimeHandler,
-  AlgorandRuntimeHandlerFilter,
   AlgorandCustomDataSource,
   AlgorandTransactionHandler,
   AlgorandTransactionFilter,
@@ -164,17 +163,13 @@ export class CustomMapping implements BaseMapping<AlgorandCustomHandler> {
   file: string;
 }
 
-export class RuntimeDataSourceBase implements AlgorandRuntimeDataSource {
+export class RuntimeDataSourceBase extends BaseDataSource implements AlgorandRuntimeDataSource {
   @IsEnum(AlgorandDataSourceKind, {groups: [AlgorandDataSourceKind.Runtime]})
   kind: AlgorandDataSourceKind.Runtime;
 
   @Type(() => RuntimeMapping)
   @ValidateNested()
   mapping: RuntimeMapping;
-
-  @IsOptional()
-  @IsInt()
-  startBlock?: number;
 }
 
 export class FileReferenceImpl implements FileReference {
@@ -183,6 +178,7 @@ export class FileReferenceImpl implements FileReference {
 }
 
 export class CustomDataSourceBase<K extends string, M extends CustomMapping, O = any>
+  extends BaseDataSource
   implements AlgorandCustomDataSource<K, M>
 {
   @IsString()
@@ -192,10 +188,6 @@ export class CustomDataSourceBase<K extends string, M extends CustomMapping, O =
   @ValidateNested()
   mapping: M;
 
-  @IsOptional()
-  @IsInt()
-  startBlock?: number;
-
   @Type(() => FileReferenceImpl)
   @ValidateNested({each: true})
   assets: Map<string, AlgorandCustomDataSourceAsset>;

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Update `@subql/node-core` with fixes and support for endBlock feature (#83)
 
 ## [3.1.1] - 2023-10-27
 ### Fixed

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -25,9 +25,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/common-algorand": "workspace:*",
-    "@subql/node-core": "^6.1.1",
+    "@subql/node-core": "^6.2.0",
     "@subql/types-algorand": "workspace:*",
     "algosdk": "^2.2.0",
     "axios": "^1.3.4",

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -6,31 +6,18 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  Worker,
   SmartBatchService,
   StoreService,
   PoiService,
   PoiSyncService,
   StoreCacheService,
   IProjectService,
-  IDynamicDsService,
-  HostStore,
-  HostDynamicDS,
   WorkerBlockDispatcher,
-  IUnfinalizedBlocksService,
-  HostConnectionPoolState,
   ConnectionPoolStateManager,
-  connectionPoolStateHostFunctions,
-  baseWorkerFunctions,
-  storeHostFunctions,
-  dynamicDsHostFunctions,
   IProjectUpgradeService,
-  HostUnfinalizedBlocks,
-  cacheHostFunctions,
-  HostCache,
   InMemoryCacheService,
+  createIndexerWorker,
 } from '@subql/node-core';
-import { Store, Cache } from '@subql/types-core';
 import { AlgorandApiConnection } from '../../algorand';
 import {
   AlgorandProjectDs,
@@ -39,50 +26,12 @@ import {
 import { DynamicDsService } from '../dynamic-ds.service';
 import { BlockContent } from '../types';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';
+import { IIndexerWorker } from '../worker/worker';
 
 type IndexerWorker = IIndexerWorker & {
   terminate: () => Promise<number>;
 };
 
-async function createIndexerWorker(
-  store: Store,
-  cache: Cache,
-  dynamicDsService: IDynamicDsService<AlgorandProjectDs>,
-  unfinalizedBlocksService: IUnfinalizedBlocksService<BlockContent>,
-  connectionPoolState: ConnectionPoolStateManager<AlgorandApiConnection>,
-  root: string,
-  startHeight: number,
-): Promise<IndexerWorker> {
-  const indexerWorker = Worker.create<
-    IInitIndexerWorker,
-    // HostDynamicDS<AlgorandProjectDs> & HostStore & HostUnfinalizedBlocks
-    HostDynamicDS<AlgorandProjectDs> &
-      HostStore &
-      HostCache &
-      HostUnfinalizedBlocks &
-      HostConnectionPoolState<AlgorandApiConnection>
-  >(
-    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
-    [...baseWorkerFunctions, 'initWorker'],
-    {
-      ...storeHostFunctions(store),
-      ...cacheHostFunctions(cache),
-      ...dynamicDsHostFunctions(dynamicDsService),
-      unfinalizedBlocksProcess:
-        unfinalizedBlocksService.processUnfinalizedBlockHeader.bind(
-          unfinalizedBlocksService,
-        ),
-      ...connectionPoolStateHostFunctions(connectionPoolState),
-    },
-    root,
-  );
-
-  await indexerWorker.initWorker(startHeight);
-
-  return indexerWorker;
-}
-
 @Injectable()
 export class WorkerBlockDispatcherService
   extends WorkerBlockDispatcher<AlgorandProjectDs, IndexerWorker>
@@ -119,7 +68,14 @@ export class WorkerBlockDispatcherService
       project,
       dynamicDsService,
       () =>
-        createIndexerWorker(
+        createIndexerWorker<
+          IIndexerWorker,
+          AlgorandApiConnection,
+          BlockContent,
+          AlgorandProjectDs
+        >(
+          path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
+          [],
           storeService.getStore(),
           cacheService.getCache(),
           dynamicDsService,

diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts
@@ -9,6 +9,7 @@ import {
   ConnectionPoolStateManager,
   WorkerConnectionPoolStateManager,
   InMemoryCacheService,
+  WorkerInMemoryCacheService,
 } from '@subql/node-core';
 import { AlgorandApiService, AlgorandApiConnection } from '../../algorand';
 import { SubqueryProject } from '../../configure/SubqueryProject';
@@ -18,7 +19,6 @@ import { IndexerManager } from '../indexer.manager';
 import { ProjectService } from '../project.service';
 import { SandboxService } from '../sandbox.service';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { WorkerInMemoryCacheService } from './worker.cache.service';
 import { WorkerService } from './worker.service';
 import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service';
 

diff --git a/packages/node/src/indexer/worker/worker.cache.service.ts b/packages/node/src/indexer/worker/worker.cache.service.ts
diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts
@@ -55,10 +55,6 @@ async function initWorker(startHeight: number): Promise<void> {
 
 export type IIndexerWorker = IBaseIndexerWorker;
 
-export type IInitIndexerWorker = IIndexerWorker & {
-  initWorker: typeof initWorker;
-};
-
 (global as any).host = createWorkerHost([], {
   initWorker,
 });
diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts
@@ -8,10 +8,12 @@ import {
   ConnectionPoolService,
   ConnectionPoolStateManager,
   DbModule,
+  InMemoryCacheService,
   PoiService,
+  PoiSyncService,
+  StoreCacheService,
   StoreService,
   TestRunner,
-  StoreCacheService,
 } from '@subql/node-core';
 import { AlgorandApiService } from '../algorand';
 import { ConfigureModule } from '../configure/configure.module';
@@ -23,10 +25,12 @@ import { SandboxService } from '../indexer/sandbox.service';
 import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 @Module({
   providers: [
+    InMemoryCacheService,
     StoreService,
     StoreCacheService,
     EventEmitter2,
     PoiService,
+    PoiSyncService,
     SandboxService,
     DsProcessorService,
     DynamicDsService,

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -82,7 +82,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`,
       type: 'string',
     },
     'dictionary-resolver': {

diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Import `@subql/types-core` global into global so its no longer needed to update tsconfig in projects (#83)
 
 ## [3.0.1] - 2023-10-26
 ### Changed

diff --git a/packages/types/package.json b/packages/types/package.json
@@ -24,6 +24,6 @@
     "algosdk": "^2.2.0"
   },
   "dependencies": {
-    "@subql/types-core": "^0.2.0"
+    "@subql/types-core": "^0.3.0"
   }
 }
diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {SafeAPI} from './interfaces';
+import '@subql/types-core/dist/global';
 
 declare global {
   const api: SafeAPI;