diff --git a/packages/common-algorand/CHANGELOG.md b/packages/common-algorand/CHANGELOG.md index b834e53e..528d6641 100644 --- a/packages/common-algorand/CHANGELOG.md +++ b/packages/common-algorand/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Update `@subql/common` and relevant changes to support endBlock feature (#83) ## [3.0.2] - 2023-10-26 ### Changed diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json index c317df94..c8effd8c 100644 --- a/packages/common-algorand/package.json +++ b/packages/common-algorand/package.json @@ -14,7 +14,7 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "^3.2.0", + "@subql/common": "^3.3.0", "@subql/types-algorand": "workspace:*", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", diff --git a/packages/common-algorand/src/project/models.ts b/packages/common-algorand/src/project/models.ts index acacf7af..ff73b008 100644 --- a/packages/common-algorand/src/project/models.ts +++ b/packages/common-algorand/src/project/models.ts @@ -1,7 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {ProcessorImpl} from '@subql/common'; +import {BaseDataSource, ProcessorImpl} from '@subql/common'; import { CustomDataSourceAsset as AlgorandCustomDataSourceAsset, AlgorandBlockFilter, @@ -11,7 +11,6 @@ import { AlgorandHandlerKind, AlgorandRuntimeDataSource, AlgorandRuntimeHandler, - AlgorandRuntimeHandlerFilter, AlgorandCustomDataSource, AlgorandTransactionHandler, AlgorandTransactionFilter, @@ -164,17 +163,13 @@ export class CustomMapping implements BaseMapping { file: string; } -export class RuntimeDataSourceBase implements AlgorandRuntimeDataSource { +export class RuntimeDataSourceBase extends BaseDataSource implements AlgorandRuntimeDataSource { @IsEnum(AlgorandDataSourceKind, {groups: [AlgorandDataSourceKind.Runtime]}) kind: AlgorandDataSourceKind.Runtime; @Type(() => RuntimeMapping) @ValidateNested() mapping: RuntimeMapping; - - @IsOptional() - @IsInt() - startBlock?: number; } export class FileReferenceImpl implements FileReference { @@ -183,6 +178,7 @@ export class FileReferenceImpl implements FileReference { } export class CustomDataSourceBase + extends BaseDataSource implements AlgorandCustomDataSource { @IsString() @@ -192,10 +188,6 @@ export class CustomDataSourceBase FileReferenceImpl) @ValidateNested({each: true}) assets: Map; diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 5f98752f..f9527b9b 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update `@subql/node-core` with fixes and support for endBlock feature (#83) ## [3.1.1] - 2023-10-27 ### Fixed diff --git a/packages/node/package.json b/packages/node/package.json index d67f271f..e6ffb88d 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -25,9 +25,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.2.0", + "@subql/common": "^3.3.0", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "^6.1.1", + "@subql/node-core": "^6.2.0", "@subql/types-algorand": "workspace:*", "algosdk": "^2.2.0", "axios": "^1.3.4", diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 76a64fdd..e79eddf9 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -6,31 +6,18 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - Worker, SmartBatchService, StoreService, PoiService, PoiSyncService, StoreCacheService, IProjectService, - IDynamicDsService, - HostStore, - HostDynamicDS, WorkerBlockDispatcher, - IUnfinalizedBlocksService, - HostConnectionPoolState, ConnectionPoolStateManager, - connectionPoolStateHostFunctions, - baseWorkerFunctions, - storeHostFunctions, - dynamicDsHostFunctions, IProjectUpgradeService, - HostUnfinalizedBlocks, - cacheHostFunctions, - HostCache, InMemoryCacheService, + createIndexerWorker, } from '@subql/node-core'; -import { Store, Cache } from '@subql/types-core'; import { AlgorandApiConnection } from '../../algorand'; import { AlgorandProjectDs, @@ -39,50 +26,12 @@ import { import { DynamicDsService } from '../dynamic-ds.service'; import { BlockContent } from '../types'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; -import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker'; +import { IIndexerWorker } from '../worker/worker'; type IndexerWorker = IIndexerWorker & { terminate: () => Promise; }; -async function createIndexerWorker( - store: Store, - cache: Cache, - dynamicDsService: IDynamicDsService, - unfinalizedBlocksService: IUnfinalizedBlocksService, - connectionPoolState: ConnectionPoolStateManager, - root: string, - startHeight: number, -): Promise { - const indexerWorker = Worker.create< - IInitIndexerWorker, - // HostDynamicDS & HostStore & HostUnfinalizedBlocks - HostDynamicDS & - HostStore & - HostCache & - HostUnfinalizedBlocks & - HostConnectionPoolState - >( - path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), - [...baseWorkerFunctions, 'initWorker'], - { - ...storeHostFunctions(store), - ...cacheHostFunctions(cache), - ...dynamicDsHostFunctions(dynamicDsService), - unfinalizedBlocksProcess: - unfinalizedBlocksService.processUnfinalizedBlockHeader.bind( - unfinalizedBlocksService, - ), - ...connectionPoolStateHostFunctions(connectionPoolState), - }, - root, - ); - - await indexerWorker.initWorker(startHeight); - - return indexerWorker; -} - @Injectable() export class WorkerBlockDispatcherService extends WorkerBlockDispatcher @@ -119,7 +68,14 @@ export class WorkerBlockDispatcherService project, dynamicDsService, () => - createIndexerWorker( + createIndexerWorker< + IIndexerWorker, + AlgorandApiConnection, + BlockContent, + AlgorandProjectDs + >( + path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), + [], storeService.getStore(), cacheService.getCache(), dynamicDsService, diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index 26dc848b..a11bbada 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -9,6 +9,7 @@ import { ConnectionPoolStateManager, WorkerConnectionPoolStateManager, InMemoryCacheService, + WorkerInMemoryCacheService, } from '@subql/node-core'; import { AlgorandApiService, AlgorandApiConnection } from '../../algorand'; import { SubqueryProject } from '../../configure/SubqueryProject'; @@ -18,7 +19,6 @@ import { IndexerManager } from '../indexer.manager'; import { ProjectService } from '../project.service'; import { SandboxService } from '../sandbox.service'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; -import { WorkerInMemoryCacheService } from './worker.cache.service'; import { WorkerService } from './worker.service'; import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service'; diff --git a/packages/node/src/indexer/worker/worker.cache.service.ts b/packages/node/src/indexer/worker/worker.cache.service.ts deleted file mode 100644 index b8bb16ec..00000000 --- a/packages/node/src/indexer/worker/worker.cache.service.ts +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -// TODO remove and use version in node-core - -import { isMainThread } from 'worker_threads'; -import { Injectable } from '@nestjs/common'; -import { HostCache, hostCacheToCache } from '@subql/node-core'; -import { Cache } from '@subql/types-core'; - -@Injectable() -export class WorkerInMemoryCacheService { - constructor(private host: HostCache) { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - } - - getCache(): Cache { - return hostCacheToCache(this.host); - } -} diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index b01e52a0..b72da24b 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -55,10 +55,6 @@ async function initWorker(startHeight: number): Promise { export type IIndexerWorker = IBaseIndexerWorker; -export type IInitIndexerWorker = IIndexerWorker & { - initWorker: typeof initWorker; -}; - (global as any).host = createWorkerHost([], { initWorker, }); diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index a04e7906..1395c517 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -8,10 +8,12 @@ import { ConnectionPoolService, ConnectionPoolStateManager, DbModule, + InMemoryCacheService, PoiService, + PoiSyncService, + StoreCacheService, StoreService, TestRunner, - StoreCacheService, } from '@subql/node-core'; import { AlgorandApiService } from '../algorand'; import { ConfigureModule } from '../configure/configure.module'; @@ -23,10 +25,12 @@ import { SandboxService } from '../indexer/sandbox.service'; import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; @Module({ providers: [ + InMemoryCacheService, StoreService, StoreCacheService, EventEmitter2, PoiService, + PoiSyncService, SandboxService, DsProcessorService, DynamicDsService, diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index f459ee5b..8a73210b 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -82,7 +82,7 @@ export const yargsOptions = yargs(hideBin(process.argv)) }, debug: { demandOption: false, - describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`, + describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`, type: 'string', }, 'dictionary-resolver': { diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index a045df62..ca423a7a 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Import `@subql/types-core` global into global so its no longer needed to update tsconfig in projects (#83) ## [3.0.1] - 2023-10-26 ### Changed diff --git a/packages/types/package.json b/packages/types/package.json index 9b66c48d..3255b621 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -24,6 +24,6 @@ "algosdk": "^2.2.0" }, "dependencies": { - "@subql/types-core": "^0.2.0" + "@subql/types-core": "^0.3.0" } } diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts index 59d600b1..9e8f4ff5 100644 --- a/packages/types/src/global.ts +++ b/packages/types/src/global.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import {SafeAPI} from './interfaces'; +import '@subql/types-core/dist/global'; declare global { const api: SafeAPI; diff --git a/yarn.lock b/yarn.lock index 6c5a121f..410ca563 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2916,7 +2916,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-algorand@workspace:packages/common-algorand" dependencies: - "@subql/common": ^3.2.0 + "@subql/common": ^3.3.0 "@subql/types-algorand": "workspace:*" "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 @@ -2935,11 +2935,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.2.0, @subql/common@npm:^3.2.0": - version: 3.2.0 - resolution: "@subql/common@npm:3.2.0" +"@subql/common@npm:3.3.0, @subql/common@npm:^3.3.0": + version: 3.3.0 + resolution: "@subql/common@npm:3.3.0" dependencies: - "@subql/types-core": 0.2.0 + "@subql/types-core": 0.3.0 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2949,7 +2949,7 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: bddf220f66e008a6b1ea40081143cc7ad71e144597a9a44cb59acb865923746006d13cfa376b49794fa81ca14200e905f5bea7292f6d70416353fdbe9f009dc1 + checksum: 50d904c0c7e0b8caeb1f53a968c9c5aa223fa8bc822ab164edbe7043a922e5c200d1ac67996358d9152d443e9288c3e5164c906181cc305a0471ffa6c8e2fdea languageName: node linkType: hard @@ -2965,9 +2965,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.2.0 + "@subql/common": ^3.3.0 "@subql/common-algorand": "workspace:*" - "@subql/node-core": ^6.1.1 + "@subql/node-core": ^6.2.0 "@subql/types-algorand": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -2996,19 +2996,19 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:^6.1.1": - version: 6.1.1 - resolution: "@subql/node-core@npm:6.1.1" +"@subql/node-core@npm:^6.2.0": + version: 6.2.0 + resolution: "@subql/node-core@npm:6.2.0" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 3.2.0 + "@subql/common": 3.3.0 "@subql/testing": 2.1.0 - "@subql/types": 3.2.0 - "@subql/utils": 2.4.5-0 + "@subql/types": 3.3.0 + "@subql/utils": 2.5.0 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 @@ -3023,7 +3023,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: 7709de62a75d2461c185d88600be657c43d9a8b0ab90355d5cece694b0077c57a329f4c5963f29213e0b0da117141c09c965561219cdc26614015441b99c9158 + checksum: ec4deffd88a94067a634349b8b809415a3d0dbc28a3c187df8482409fedef85ce343f497ff68e0c0b86ea0520f4864aee306986a62be0e4e20df14920ec66b40 languageName: node linkType: hard @@ -3040,7 +3040,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/types-algorand@workspace:packages/types" dependencies: - "@subql/types-core": ^0.2.0 + "@subql/types-core": ^0.3.0 "@types/app-module-path": ^2.2.0 algosdk: ^2.2.0 peerDependencies: @@ -3048,12 +3048,12 @@ __metadata: languageName: unknown linkType: soft -"@subql/types-core@npm:0.2.0, @subql/types-core@npm:^0.2.0": - version: 0.2.0 - resolution: "@subql/types-core@npm:0.2.0" +"@subql/types-core@npm:0.3.0, @subql/types-core@npm:^0.3.0": + version: 0.3.0 + resolution: "@subql/types-core@npm:0.3.0" dependencies: package-json-type: ^1.0.3 - checksum: ef538b84cb0c32f41269fdb6708e9e0657da9e7f0469ef24882e0b392a797eaf35364a7ef9368c0c89b799d6522a66cbf17781deae1edfdc8a17f9cd43aed95f + checksum: fa9ec3bb27d33b57a18425553f6f9239dca439f37f296ae49dd1da958a3d6f14d73aa0831c057828618798926049c0308f3cba68276c80bfbff2fbfe265d1bf3 languageName: node linkType: hard @@ -3066,20 +3066,20 @@ __metadata: languageName: node linkType: hard -"@subql/types@npm:3.2.0": - version: 3.2.0 - resolution: "@subql/types@npm:3.2.0" +"@subql/types@npm:3.3.0": + version: 3.3.0 + resolution: "@subql/types@npm:3.3.0" dependencies: - "@subql/types-core": 0.2.0 + "@subql/types-core": 0.3.0 peerDependencies: "@polkadot/api": ^10 - checksum: dbd592a2b7322bcea301bbcd320e1706f0af67330f8a0bf7893f5de5d7df58e139a635a69b4d0a1fc14b7b54972d898ebe54c8303ae3a139db892d673f74cc02 + checksum: 1a8227edae7f77f08de1f255660a8425cffa29ca178deb0a06f7713860af214bdf77092857001e9f12d3124554d092374d23607601266c40c339aea53cbc8547 languageName: node linkType: hard -"@subql/utils@npm:2.4.5-0": - version: 2.4.5-0 - resolution: "@subql/utils@npm:2.4.5-0" +"@subql/utils@npm:2.5.0": + version: 2.5.0 + resolution: "@subql/utils@npm:2.5.0" dependencies: "@polkadot/util": ^12.2.1 "@polkadot/util-crypto": ^12.2.1 @@ -3096,7 +3096,7 @@ __metadata: rotating-file-stream: ^3.0.2 semver: ^7.5.2 tar: ^6.1.11 - checksum: 7d76203fcb25251225ce7a2a56bf3de8e2a98caed4f1598ca50dea4bf96511bec71b7d4bdbec8648a7b76b654259a94dbdf2c66105961e8aebe780c318ad2ae0 + checksum: 27d958f916f519aba9257d5bb4d310f0bb08d628098643c417342fbf864e6138de28d7fd0c8a3b3177f93668544b0655a6b483c9b1fe5fabbd8b08babc2ea36a languageName: node linkType: hard