Merge pull request #74 from subquery/sync-20231031

Sync @subql dependencies
subquery · Oct 31, 2023 · 05e2f97 · 05e2f97
2 parents f5cebbe + b941c7d
commit 05e2f97
Show file tree

Hide file tree

Showing 15 changed files with 61 additions and 122 deletions.
diff --git a/packages/common-near/CHANGELOG.md b/packages/common-near/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Update `@subql/common` and relevant changes to support endBlock feature (#74)
 
 ## [3.0.1] - 2023-10-26
 ### Changed

diff --git a/packages/common-near/package.json b/packages/common-near/package.json
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/types-near": "workspace:*",
     "js-yaml": "^4.1.0",
     "reflect-metadata": "^0.1.13"

diff --git a/packages/common-near/src/project/models.ts b/packages/common-near/src/project/models.ts
@@ -1,7 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {BlockFilterImpl, ProcessorImpl} from '@subql/common';
+import {BaseDataSource, BlockFilterImpl, ProcessorImpl} from '@subql/common';
 import {BaseMapping, FileReference, Processor} from '@subql/types-core';
 import {
   CustomDataSourceAsset as NearCustomDataSourceAsset,
@@ -201,17 +201,12 @@ export class CustomMapping implements BaseMapping<NearCustomHandler> {
   file: string;
 }
 
-export class RuntimeDataSourceBase implements NearRuntimeDatasource {
+export class RuntimeDataSourceBase extends BaseDataSource implements NearRuntimeDatasource {
   @IsEnum(NearDatasourceKind, {groups: [NearDatasourceKind.Runtime]})
   kind: NearDatasourceKind.Runtime;
   @Type(() => RuntimeMapping)
   @ValidateNested()
   mapping: RuntimeMapping;
-  //must be greater than genesis block
-  @Min(9820210)
-  @IsOptional()
-  @IsInt()
-  startBlock?: number;
 }
 
 export class FileReferenceImpl implements FileReference {
@@ -220,16 +215,14 @@ export class FileReferenceImpl implements FileReference {
 }
 
 export class CustomDataSourceBase<K extends string, M extends CustomMapping, O = any>
+  extends BaseDataSource
   implements NearCustomDatasource<K, M>
 {
   @IsString()
   kind: K;
   @Type(() => CustomMapping)
   @ValidateNested()
   mapping: M;
-  @IsOptional()
-  @IsInt()
-  startBlock?: number;
   @Type(() => FileReferenceImpl)
   @ValidateNested({each: true})
   assets: Map<string, NearCustomDataSourceAsset>;

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Update `@subql/node-core` with fixes and support for endBlock feature (#74)
 
 ## [3.1.1] - 2023-10-27
 ### Fixed

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -25,9 +25,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/common-near": "workspace:*",
-    "@subql/node-core": "^6.1.1",
+    "@subql/node-core": "^6.2.0",
     "@subql/types-near": "workspace:*",
     "cron-converter": "^1.0.2",
     "eventemitter2": "^6.4.5",

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -6,30 +6,18 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  Worker,
   SmartBatchService,
   StoreService,
   PoiService,
   StoreCacheService,
   IProjectService,
-  IDynamicDsService,
-  HostStore,
-  HostDynamicDS,
   WorkerBlockDispatcher,
-  IUnfinalizedBlocksService,
-  HostConnectionPoolState,
   ConnectionPoolStateManager,
-  connectionPoolStateHostFunctions,
   IProjectUpgradeService,
-  baseWorkerFunctions,
-  storeHostFunctions,
-  dynamicDsHostFunctions,
-  HostUnfinalizedBlocks,
   PoiSyncService,
-  cacheHostFunctions,
   InMemoryCacheService,
+  createIndexerWorker,
 } from '@subql/node-core';
-import { Store, Cache } from '@subql/types-core';
 import { NearDatasource } from '@subql/types-near';
 import {
   NearProjectDs,
@@ -39,48 +27,12 @@ import { DynamicDsService } from '../dynamic-ds.service';
 import { NearApiConnection } from '../nearApi.connection';
 import { BlockContent } from '../types';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';
+import { IIndexerWorker } from '../worker/worker';
 
 type IndexerWorker = IIndexerWorker & {
   terminate: () => Promise<number>;
 };
 
-async function createIndexerWorker(
-  store: Store,
-  cache: Cache,
-  dynamicDsService: IDynamicDsService<NearDatasource>,
-  unfinalizedBlocksService: IUnfinalizedBlocksService<BlockContent>,
-  connectionPoolState: ConnectionPoolStateManager<NearApiConnection>,
-  root: string,
-  startHeight: number,
-): Promise<IndexerWorker> {
-  const indexerWorker = Worker.create<
-    IInitIndexerWorker,
-    HostDynamicDS<NearDatasource> &
-      HostStore &
-      HostUnfinalizedBlocks &
-      HostConnectionPoolState<NearApiConnection>
-  >(
-    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
-    [...baseWorkerFunctions, 'initWorker'],
-    {
-      ...storeHostFunctions(store),
-      ...cacheHostFunctions(cache),
-      ...dynamicDsHostFunctions(dynamicDsService),
-      unfinalizedBlocksProcess:
-        unfinalizedBlocksService.processUnfinalizedBlockHeader.bind(
-          unfinalizedBlocksService,
-        ),
-      ...connectionPoolStateHostFunctions(connectionPoolState),
-    },
-    root,
-  );
-
-  await indexerWorker.initWorker(startHeight);
-
-  return indexerWorker;
-}
-
 @Injectable()
 export class WorkerBlockDispatcherService
   extends WorkerBlockDispatcher<NearDatasource, IndexerWorker>
@@ -116,7 +68,14 @@ export class WorkerBlockDispatcherService
       project,
       dynamicDsService,
       () =>
-        createIndexerWorker(
+        createIndexerWorker<
+          IIndexerWorker,
+          NearApiConnection,
+          BlockContent,
+          NearDatasource
+        >(
+          path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
+          [],
           storeService.getStore(),
           cacheService.getCache(),
           dynamicDsService,

diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts
@@ -10,6 +10,7 @@ import {
   ConnectionPoolStateManager,
   NodeConfig,
   InMemoryCacheService,
+  WorkerInMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../../configure/SubqueryProject';
 import { ApiService } from '../api.service';
@@ -22,7 +23,6 @@ import { SandboxService } from '../sandbox.service';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
 import { WorkerService } from '../worker/worker.service';
 import { WorkerUnfinalizedBlocksService } from '../worker/worker.unfinalizedBlocks.service';
-import { WorkerInMemoryCacheService } from './worker.cache.service';
 
 @Module({
   providers: [

diff --git a/packages/node/src/indexer/worker/worker.cache.service.ts b/packages/node/src/indexer/worker/worker.cache.service.ts
diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts
@@ -55,10 +55,6 @@ async function initWorker(startHeight: number): Promise<void> {
 
 export type IIndexerWorker = IBaseIndexerWorker;
 
-export type IInitIndexerWorker = IIndexerWorker & {
-  initWorker: typeof initWorker;
-};
-
 (global as any).host = createWorkerHost([], {
   initWorker,
 });
diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts
@@ -8,8 +8,11 @@ import {
   ConnectionPoolService,
   ConnectionPoolStateManager,
   DbModule,
+  InMemoryCacheService,
   NodeConfig,
   PoiService,
+  PoiSyncService,
+  StoreCacheService,
   StoreService,
   TestRunner,
 } from '@subql/node-core';
@@ -28,9 +31,12 @@ import { MetaModule } from '../meta/meta.module';
 
 @Module({
   providers: [
+    InMemoryCacheService,
     StoreService,
+    StoreCacheService,
     EventEmitter2,
     PoiService,
+    PoiSyncService,
     SandboxService,
     DsProcessorService,
     DynamicDsService,

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -236,7 +236,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`,
       type: 'string',
     },
     ipfs: {

diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Import `@subql/types-core` global into global so its no longer needed to update tsconfig in projects (#74)
 
 ## [3.0.1] - 2023-10-26
 ### Changed

diff --git a/packages/types/package.json b/packages/types/package.json
@@ -17,7 +17,7 @@
     "/dist"
   ],
   "dependencies": {
-    "@subql/types-core": "^0.2.0",
+    "@subql/types-core": "^0.3.0",
     "bn.js": "5.2.1",
     "near-api-js": "^1.1.0"
   },

diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {providers} from 'near-api-js';
+import '@subql/types-core/dist/global';
 
 declare global {
   const api: providers.JsonRpcProvider;