Merge branch 'develop' into feat/job-launcher-server/abuse

.github/workflows/ci-dependency-review.yaml

@@ -11,4 +11,4 @@ jobs:
       - name: "Checkout Repository"
         uses: actions/[email protected]
       - name: "Dependency Review"
-        uses: actions/dependency-review-action@v4.4.0
+        uses: actions/dependency-review-action@v4.5.0

packages/apps/dashboard/server/.env.example

@@ -11,6 +11,8 @@ HMT_PRICE_SOURCE_API_KEY=
 HMT_PRICE_FROM=
 HMT_PRICE_TO=
 HCAPTCHA_API_KEY=
+NETWORK_USAGE_FILTER_MONTHS=
+NETWORKS_AVAILABLE_CACHE_TTL=
 
 # Redis
 REDIS_HOST=localhost

packages/apps/dashboard/server/.gitignore

@@ -35,4 +35,7 @@ lerna-debug.log*
 !.vscode/extensions.json
 
 # Redis Data
-/redis_data
+/redis_data
+
+.env.development
+.env.production

packages/apps/dashboard/server/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.8'
+name: 'dashboard-server'
 
 services:
   redis:

packages/apps/dashboard/server/jest.config.ts

@@ -0,0 +1,15 @@
+module.exports = {
+  coverageDirectory: '../coverage',
+  collectCoverageFrom: ['**/*.(t|j)s'],
+  moduleFileExtensions: ['js', 'json', 'ts'],
+  rootDir: 'src',
+  testEnvironment: 'node',
+  testRegex: '.*\\.spec\\.ts$',
+  transform: {
+    '^.+\\.(t)s$': 'ts-jest',
+  },
+  moduleNameMapper: {
+    '^uuid$': require.resolve('uuid'),
+    '^typeorm$': require.resolve('typeorm'),
+  },
+};

packages/apps/dashboard/server/package.json

@@ -11,20 +11,26 @@
     "start": "nest start",
     "start:dev": "nest start --watch",
     "start:debug": "nest start --debug --watch",
-    "start:prod": "node dist/main",
-    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix"
+    "start:prod": "node dist/src/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand"
   },
   "dependencies": {
     "@human-protocol/sdk": "*",
-    "@nestjs/axios": "^3.0.2",
+    "@nestjs/axios": "^3.1.2",
     "@nestjs/cache-manager": "^2.2.2",
     "@nestjs/common": "^10.2.7",
     "@nestjs/config": "^3.2.3",
     "@nestjs/core": "^10.2.8",
     "@nestjs/mapped-types": "*",
     "@nestjs/platform-express": "^10.3.10",
-    "cache-manager-redis-store": "^3.0.1",
+    "cache-manager": "^5.4.0",
+    "cache-manager-redis-yet": "^5.1.5",
     "dayjs": "^1.11.12",
+    "lodash": "^4.17.21",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.2.0"
   },

packages/apps/dashboard/server/src/app.module.ts

@@ -21,8 +21,12 @@ import { StatsModule } from './modules/stats/stats.module';
       validationSchema: Joi.object({
         HOST: Joi.string().required(),
         PORT: Joi.number().port().default(3000),
+        REDIS_HOST: Joi.string(),
+        REDIS_PORT: Joi.number(),
         SUBGRAPH_API_KEY: Joi.string().required(),
         HCAPTCHA_API_KEY: Joi.string().required(),
+        CACHE_HMT_PRICE_TTL: Joi.number(),
+        CACHE_HMT_GENERAL_STATS_TTL: Joi.number(),
       }),
     }),
     CacheModule.registerAsync(CacheFactoryConfig),

packages/apps/dashboard/server/src/common/config/cache-factory.config.ts

@@ -1,7 +1,15 @@
 import { CacheModuleAsyncOptions } from '@nestjs/common/cache';
 import { ConfigModule } from '@nestjs/config';
+import * as _ from 'lodash';
 import { RedisConfigService } from './redis-config.service';
-import { redisStore } from 'cache-manager-redis-store';
+import { redisStore } from 'cache-manager-redis-yet';
+import { Logger } from '@nestjs/common';
+
+const logger = new Logger('CacheFactoryRedisStore');
+
+const throttledRedisErrorLog = _.throttle((error) => {
+  logger.error('Redis client network error', error);
+}, 1000 * 5);
 
 export const CacheFactoryConfig: CacheModuleAsyncOptions = {
   isGlobal: true,
@@ -12,7 +20,11 @@ export const CacheFactoryConfig: CacheModuleAsyncOptions = {
         host: configService.redisHost,
         port: configService.redisPort,
       },
+      disableOfflineQueue: true,
     });
+
+    store.client.on('error', throttledRedisErrorLog);
+
     return {
       store: () => store,
     };

packages/apps/dashboard/server/src/common/config/env-config.service.ts

@@ -14,6 +14,10 @@ const DEFAULT_HCAPTCHA_STATS_FILE = 'hcaptchaStats.json';
 export const HCAPTCHA_STATS_START_DATE = '2022-07-01';
 export const HCAPTCHA_STATS_API_START_DATE = '2024-09-14';
 export const HMT_STATS_START_DATE = '2021-04-06';
+export const MINIMUM_HMT_TRANSFERS = 5;
+export const DEFAULT_NETWORK_USAGE_FILTER_MONTHS = 1;
+export const DEFAULT_NETWORKS_AVAILABLE_CACHE_TTL = 2 * 60;
+export const MINIMUM_ESCROWS_COUNT = 1;
 
 @Injectable()
 export class EnvironmentConfigService {
@@ -81,4 +85,18 @@ export class EnvironmentConfigService {
   get reputationSource(): string {
     return this.configService.getOrThrow<string>('REPUTATION_SOURCE_URL');
   }
+
+  get networkUsageFilterMonths(): number {
+    return this.configService.get<number>(
+      'NETWORK_USAGE_FILTER_MONTHS',
+      DEFAULT_NETWORK_USAGE_FILTER_MONTHS,
+    );
+  }
+
+  get networkAvailableCacheTtl(): number {
+    return this.configService.get<number>(
+      'NETWORKS_AVAILABLE_CACHE_TTL',
+      DEFAULT_NETWORKS_AVAILABLE_CACHE_TTL,
+    );
+  }
 }

packages/apps/dashboard/server/src/common/config/network-config.service.spec.ts

@@ -0,0 +1,217 @@
+import { createMock } from '@golevelup/ts-jest';
+import { Test, TestingModule } from '@nestjs/testing';
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Cache } from 'cache-manager';
+import { Logger } from '@nestjs/common';
+import { StatisticsClient } from '@human-protocol/sdk';
+import { EnvironmentConfigService } from '../../common/config/env-config.service';
+import { ChainId, NETWORKS } from '@human-protocol/sdk';
+import { MainnetsId } from '../../common/utils/constants';
+import { HttpService } from '@nestjs/axios';
+import { NetworkConfigService } from './network-config.service';
+import { ConfigService } from '@nestjs/config';
+
+jest.mock('@human-protocol/sdk', () => ({
+  ...jest.requireActual('@human-protocol/sdk'),
+  StatisticsClient: jest.fn(),
+}));
+
+describe('NetworkConfigService', () => {
+  let networkConfigService: NetworkConfigService;
+  let cacheManager: Cache;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        NetworkConfigService,
+        { provide: HttpService, useValue: createMock<HttpService>() },
+        {
+          provide: CACHE_MANAGER,
+          useValue: {
+            get: jest.fn(),
+            set: jest.fn(),
+          },
+        },
+        {
+          provide: EnvironmentConfigService,
+          useValue: {
+            networkUsageFilterMonths: 3,
+            networkAvailableCacheTtl: 1000,
+          },
+        },
+        ConfigService,
+        Logger,
+      ],
+    }).compile();
+
+    networkConfigService =
+      module.get<NetworkConfigService>(NetworkConfigService);
+    cacheManager = module.get<Cache>(CACHE_MANAGER);
+  });
+
+  it('should regenerate network list when cache TTL expires', async () => {
+    const mockNetworkList = [
+      ChainId.MAINNET,
+      ChainId.BSC_MAINNET,
+      ChainId.POLYGON,
+      ChainId.XLAYER,
+      ChainId.MOONBEAM,
+      ChainId.CELO,
+      ChainId.AVALANCHE,
+    ];
+
+    // Step 1: Initial request - populate cache
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
+    jest.spyOn(cacheManager, 'set').mockResolvedValue(undefined);
+
+    const mockStatisticsClient = {
+      getHMTDailyData: jest
+        .fn()
+        .mockResolvedValue([{ totalTransactionCount: 7 }]),
+      getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
+    };
+    (StatisticsClient as jest.Mock).mockImplementation(
+      () => mockStatisticsClient,
+    );
+
+    // First call should populate cache
+    const firstCallResult = await networkConfigService.getAvailableNetworks();
+
+    expect(firstCallResult).toEqual(mockNetworkList);
+    expect(cacheManager.set).toHaveBeenCalledWith(
+      'available-networks',
+      mockNetworkList,
+      1000,
+    );
+
+    // Step 2: Simulate TTL expiration by returning null from cache
+    jest.spyOn(cacheManager, 'get').mockResolvedValueOnce(null);
+
+    // Second call after TTL should re-generate the network list
+    const secondCallResult = await networkConfigService.getAvailableNetworks();
+    expect(secondCallResult).toEqual(mockNetworkList);
+
+    // Ensure the cache is set again with the regenerated network list
+    expect(cacheManager.set).toHaveBeenCalledWith(
+      'available-networks',
+      mockNetworkList,
+      1000,
+    );
+  });
+
+  it('should return cached networks if available', async () => {
+    const cachedNetworks = [ChainId.MAINNET, ChainId.POLYGON];
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(cachedNetworks);
+
+    const result = await networkConfigService.getAvailableNetworks();
+    expect(result).toEqual(cachedNetworks);
+    expect(cacheManager.get).toHaveBeenCalledWith('available-networks');
+  });
+
+  it('should fetch and filter available networks correctly', async () => {
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
+    const mockStatisticsClient = {
+      getHMTDailyData: jest
+        .fn()
+        .mockResolvedValue([
+          { totalTransactionCount: 4 },
+          { totalTransactionCount: 3 },
+        ]),
+      getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
+    };
+    (StatisticsClient as jest.Mock).mockImplementation(
+      () => mockStatisticsClient,
+    );
+
+    const result = await networkConfigService.getAvailableNetworks();
+    expect(result).toEqual(
+      expect.arrayContaining([ChainId.MAINNET, ChainId.POLYGON]),
+    );
+
+    expect(cacheManager.set).toHaveBeenCalledWith(
+      'available-networks',
+      result,
+      1000,
+    );
+  });
+
+  it('should exclude networks without sufficient HMT transfers', async () => {
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
+    const mockStatisticsClient = {
+      getHMTDailyData: jest
+        .fn()
+        .mockResolvedValue([{ totalTransactionCount: 2 }]),
+      getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
+    };
+    (StatisticsClient as jest.Mock).mockImplementation(
+      () => mockStatisticsClient,
+    );
+
+    const result = await networkConfigService.getAvailableNetworks();
+    expect(result).toEqual([]);
+  });
+
+  it('should handle missing network configuration gracefully', async () => {
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
+
+    const originalNetworkConfig = NETWORKS[MainnetsId.MAINNET];
+    NETWORKS[MainnetsId.MAINNET] = undefined;
+
+    const mockStatisticsClient = {
+      getHMTDailyData: jest
+        .fn()
+        .mockResolvedValue([
+          { totalTransactionCount: 3 },
+          { totalTransactionCount: 3 },
+        ]),
+      getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
+    };
+    (StatisticsClient as jest.Mock).mockImplementation(
+      () => mockStatisticsClient,
+    );
+
+    const result = await networkConfigService.getAvailableNetworks();
+
+    expect(result).not.toContain(MainnetsId.MAINNET);
+    expect(result).toEqual(expect.arrayContaining([]));
+
+    NETWORKS[MainnetsId.MAINNET] = originalNetworkConfig;
+  });
+
+  it('should handle errors in getHMTDailyData gracefully', async () => {
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
+    const mockStatisticsClient = {
+      getHMTDailyData: jest
+        .fn()
+        .mockRejectedValue(new Error('Failed to fetch HMT data')),
+      getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
+    };
+    (StatisticsClient as jest.Mock).mockImplementation(
+      () => mockStatisticsClient,
+    );
+
+    const result = await networkConfigService.getAvailableNetworks();
+    expect(result).toEqual([]);
+  });
+
+  it('should handle errors in getEscrowStatistics gracefully', async () => {
+    jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
+    const mockStatisticsClient = {
+      getHMTDailyData: jest
+        .fn()
+        .mockResolvedValue([
+          { totalTransactionCount: 3 },
+          { totalTransactionCount: 2 },
+        ]),
+      getEscrowStatistics: jest
+        .fn()
+        .mockRejectedValue(new Error('Failed to fetch escrow stats')),
+    };
+    (StatisticsClient as jest.Mock).mockImplementation(
+      () => mockStatisticsClient,
+    );
+
+    const result = await networkConfigService.getAvailableNetworks();
+    expect(result).toEqual([]);
+  });
+});