Rework store flush interval implementation (#2650)

* Rework store flush interval implementation

* Update changelog

* Fix build issues

* Fix test

* Update changelogs

packages/cli/CHANGELOG.md

@@ -5,9 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-
 ### Added
-- Add some constants for Starknet 
+- Add some constants for Starknet
+
+### Changed
+- Added internal note relating to some tests
 
 ## [5.4.0] - 2024-12-11
 ### Changed

packages/cli/src/controller/codegen-controller.test.ts

@@ -8,60 +8,54 @@ import {codegen} from './codegen-controller';
 
 jest.setTimeout(30000);
 
+const projectPath = path.join(__dirname, '../../test/schemaTest');
+
 describe('Codegen can generate schema', () => {
   afterEach(async () => {
     await rimraf(path.join(__dirname, '../../test/schemaTest/src'));
   });
 
   it('codegen with correct schema should pass', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await expect(codegen(projectPath)).resolves.not.toThrow();
   });
 
   it('codegen with incorrect schema field should fail', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await expect(codegen(projectPath, ['project-bad-schema.yaml'])).rejects.toThrow(/is not an valid type/);
   });
 
   it('codegen with entities that uses reserved names should throw', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await expect(codegen(projectPath, ['project-bad-entity.yaml'])).rejects.toThrow(
       'EntityName: exampleEntityFilter cannot end with reservedKey: filter'
     );
   });
 
   it('Codegen should be able to generate ABIs from template datasources', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath, ['project-templates-abi.yaml']);
     await expect(
       fs.promises.readFile(`${projectPath}/src/types/abi-interfaces/Erc721.ts`, 'utf8')
     ).resolves.toBeTruthy();
   });
 
   it('Should not fail, if ds does not have any assets', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await expect(codegen(projectPath, ['project-no-assets.yaml'])).resolves.not.toThrow();
   });
 
   it('Codegen should be able to generate ABIs from customName datasources', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath);
     await expect(
       fs.promises.readFile(`${projectPath}/src/types/abi-interfaces/Erc721.ts`, 'utf8')
     ).resolves.toBeTruthy();
   });
 
   it('Should clean out existing types directory', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath);
     await codegen(projectPath, ['project-no-abi.yaml']);
 
     // should not contain abi directory
-    await expect(fs.promises.readFile(`${projectPath}/src/types/abi-interfaces/Erc721.ts`, 'utf8')).rejects.toThrow();
+    await expect(fs.promises.readFile(`${projectPath}/src/types/abi-interfaces/erc721.ts`, 'utf8')).rejects.toThrow();
   });
 
   it('should generate contracts on different glob paths', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath, ['typechain-test.yaml']);
 
     await expect(
@@ -78,19 +72,16 @@ describe('Codegen can generate schema', () => {
   });
 
   it('Should not generate ABI for non evm ds', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath, ['non-evm-project.yaml']);
     expect(fs.existsSync(`${projectPath}/src/types/abi-interfaces/`)).toBeFalsy();
   });
 
   it('Should not generate proto-interfaces if no chaintypes are provided', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath, ['project-cosmos.yaml']);
     expect(fs.existsSync(`${projectPath}/src/types/proto-interfaces/`)).toBeFalsy();
   });
 
   it('Should dedupe enums', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath, ['project-duplicate-enum.yaml']);
 
     const fooFile = await fs.promises.readFile(`${projectPath}/src/types/models/Foo.ts`, 'utf8');
@@ -104,7 +95,6 @@ describe('Codegen can generate schema', () => {
 
   // github issue #2211
   it('codegen file should import model files with correct case-sensitive names', async () => {
-    const projectPath = path.join(__dirname, '../../test/schemaTest');
     await codegen(projectPath, ['project-case-sensitive-import-entity.yaml']);
 
     const codegenFile = await fs.promises.readFile(`${projectPath}/src/types/models/index.ts`, 'utf8');

packages/cli/src/controller/codegen-controller.ts

@@ -246,7 +246,9 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL
       datasources as CosmosRuntimeDatasource[]
     );
   }
+  // TODO what about custom datasource processors, e.g. FrontierEvmProcessor, EthermintProcessor
   const ethManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.ethereum);
+
   // Todo, starknet codegen not supported yet
   const starknetManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.starknet);
 

packages/cli/test/schemaTest/project.yaml

@@ -5,7 +5,7 @@ schema:
   file: './schema.graphql'
 runner:
   node:
-    name: '@subql/node'
+    name: '@subql/node-ethereum'
     version: '>=3.0.1'
   query:
     name: '@subql/query'

packages/node-core/CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed the inconsistency between the `monitor-file-size` flag and the expected behavior.(#2644)
 - Improved block range validation in POI endpoint with custom class-validator decorator
 - When setting a smaller batch size and the current processing height reaches latestFinalizedHeight, it causes the program to exit unexpectedly.
+- Store flush interval having a chance of trying to flush with invalid metadata (#2650)
 
 ## [16.1.0] - 2024-12-11
 ### Changed
@@ -153,7 +154,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [10.10.2] - 2024-07-10
 ### Fixed
-- Fix issue admin api can not get `dbSize` due to it not been set in _metadata table
+- Fix issue admin api can not get `dbSize` due to it not been set in \_metadata table
 
 ## [10.10.1] - 2024-07-09
 ### Added
@@ -618,7 +619,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Base58 encoding check for POI (#1788)
-- Fix project _startHeight been blocked by poiSync (#1792)
+- Fix project \_startHeight been blocked by poiSync (#1792)
 
 ## [2.4.4] - 2023-06-07
 ### Fixed

packages/node-core/src/configure/ProjectUpgrade.service.spec.ts

@@ -1,7 +1,6 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {SchedulerRegistry} from '@nestjs/schedule';
 import {Sequelize} from '@subql/x-sequelize';
 import {CacheMetadataModel, IStoreModelProvider, ISubqueryProject, StoreCacheService, StoreService} from '../indexer';
 import {NodeConfig} from './NodeConfig';
@@ -292,7 +291,7 @@ describe('Project Upgrades', () => {
     let storeCache: IStoreModelProvider;
 
     beforeEach(async () => {
-      storeCache = new StoreCacheService({} as any, {} as any, {} as any, new SchedulerRegistry());
+      storeCache = new StoreCacheService({} as any, {} as any, {} as any);
       // eslint-disable-next-line @typescript-eslint/dot-notation
       (storeCache as any).cachedModels['_metadata'] = mockMetadata();
 

packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts

@@ -4,7 +4,6 @@
 import {readFileSync} from 'fs';
 import * as path from 'path';
 import {EventEmitter2} from '@nestjs/event-emitter';
-import {SchedulerRegistry} from '@nestjs/schedule';
 import {buildSchemaFromString} from '@subql/utils';
 import {IndexesOptions, QueryTypes, Sequelize} from '@subql/x-sequelize';
 import {GraphQLSchema} from 'graphql';
@@ -37,7 +36,7 @@ async function setup(
     },
   } as unknown as ISubqueryProject;
 
-  const storeCache = new StoreCacheService(sequelize, config, new EventEmitter2(), new SchedulerRegistry());
+  const storeCache = new StoreCacheService(sequelize, config, new EventEmitter2());
 
   const storeService = new StoreService(sequelize, config, storeCache, project);
 

packages/node-core/src/indexer/core.module.ts

@@ -3,7 +3,6 @@
 
 import {Module} from '@nestjs/common';
 import {EventEmitter2} from '@nestjs/event-emitter';
-import {SchedulerRegistry} from '@nestjs/schedule';
 import {Sequelize} from '@subql/x-sequelize';
 import {AdminController, AdminListener} from '../admin/admin.controller';
 import {NodeConfig} from '../configure';
@@ -32,7 +31,7 @@ import {storeModelFactory} from './storeModelProvider';
     {
       provide: 'IStoreModelProvider',
       useFactory: storeModelFactory,
-      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+      inject: [NodeConfig, EventEmitter2, Sequelize],
     },
     AdminListener,
   ],

packages/node-core/src/indexer/poi/poi.service.spec.ts

@@ -2,14 +2,12 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {EventEmitter2} from '@nestjs/event-emitter';
-import {SchedulerRegistry} from '@nestjs/schedule';
 import {Test, TestingModule} from '@nestjs/testing';
 import {delay} from '@subql/common';
 import {Sequelize, Transaction} from '@subql/x-sequelize';
 import {NodeConfig} from '../../configure';
 import {ProofOfIndex} from '../entities/Poi.entity';
 import {StoreCacheService} from '../storeModelProvider';
-import {METADATA_ENTITY_NAME} from '../storeModelProvider/metadata/utils';
 import {PoiService} from './poi.service';
 
 jest.mock('@subql/x-sequelize', () => {
@@ -62,7 +60,7 @@ describe('PoiService', () => {
     } as unknown as NodeConfig;
 
     const sequelize = new Sequelize();
-    storeCache = new StoreCacheService(sequelize, nodeConfig, new EventEmitter2(), new SchedulerRegistry());
+    storeCache = new StoreCacheService(sequelize, nodeConfig, new EventEmitter2());
 
     storeCache.init('height', {} as any, {} as any);
     (storeCache as any).cachedModels._metadata = {