Trying to restore balance waiting routine

zp-relayer/common/serviceUtils.ts

@@ -102,6 +102,7 @@ export function buildTxManager(
       gasPriceBumpFactor: tmConfig.TX_MIN_GAS_PRICE_BUMP_FACTOR,
       gasPriceSurplus: tmConfig.TX_GAS_PRICE_SURPLUS,
       gasPriceMaxFeeLimit: tmConfig.TX_MAX_FEE_PER_GAS_LIMIT,
+      waitingFundsTimeout: tmConfig.BALANCE_CHECK_TIMEOUT
     })
   } else {
     throw new Error('Unsupported network backend')

zp-relayer/configs/common/txManagerConfig.ts

@@ -7,10 +7,12 @@ import { zBN } from './utils'
 const zBaseConfig = z
   .object({
     TX_PRIVATE_KEY: z.string(),
+    RELAYER_INSUFFICIENT_BALANCE_CHECK_TIMEOUT: z.coerce.number().default(5000),
   })
   .transform(o => ({
     TX_ADDRESS: new Web3().eth.accounts.privateKeyToAccount(o.TX_PRIVATE_KEY).address,
     TX_PRIVATE_KEY: o.TX_PRIVATE_KEY,
+    BALANCE_CHECK_TIMEOUT: o.RELAYER_INSUFFICIENT_BALANCE_CHECK_TIMEOUT,
   }))
 
 const zTxGas = z

zp-relayer/lib/network/evm/EvmTxManager.ts

@@ -22,14 +22,17 @@ import { Mutex } from 'async-mutex'
 import BN from 'bn.js'
 import type { Redis } from 'ioredis'
 import Web3 from 'web3'
+import { toBN } from 'web3-utils'
 import type { TransactionConfig } from 'web3-core'
 import { Logger } from 'winston'
+import promiseRetry from 'promise-retry'
 
 export interface EvmTxManagerConfig {
   redis: Redis
   gasPriceBumpFactor: number
   gasPriceSurplus: number
   gasPriceMaxFeeLimit: BN | null
+  waitingFundsTimeout: number
 }
 
 type ExtraInfo = TransactionConfig
@@ -264,4 +267,27 @@ export class EvmTxManager implements TransactionManager<ExtraInfo> {
         })
     )
   }
+
+  waitingForFunds(minimumBalance: BN, cb: (balance: BN) => void): Promise<void> {
+    return promiseRetry(
+      async retry => {
+        logger.debug('Getting manager balance')
+        const newBalance = toBN(await this.web3.eth.getBalance(this.address))
+        const balanceLog = { balance: newBalance.toString(10), minimumBalance: minimumBalance.toString(10) }
+        if (newBalance.gte(minimumBalance)) {
+          logger.info('Relayer has minimum necessary balance', balanceLog)
+          cb(newBalance)
+        } else {
+          logger.warn('Relayer balance is still less than the minimum', balanceLog)
+          retry(new Error('Not enough balance'))
+        }
+      },
+      {
+        forever: true,
+        factor: 1,
+        maxTimeout: this.config.waitingFundsTimeout,
+        minTimeout: this.config.waitingFundsTimeout,
+      }
+    )
+  }
 }

zp-relayer/lib/network/tron/TronTxManager.ts

@@ -1,4 +1,5 @@
 import { PreparedTx, SendAttempt, SendError, SendTx, TransactionManager, TxInfo } from '../types'
+import BN from 'bn.js'
 
 interface ExtraInfo {}
 
@@ -69,4 +70,8 @@ export class TronTxManager implements TransactionManager<ExtraInfo> {
     const preparedTx = await this.prepareTx(sendTx)
     return this.sendPreparedTx(preparedTx)
   }
+
+  waitingForFunds(minimumBalance: BN, cb: (balance: BN) => void): Promise<void> {
+    throw new Error('Method not implemented');
+  }
 }

zp-relayer/lib/network/types.ts

@@ -1,6 +1,7 @@
 import type { TransactionConfig } from 'web3-core'
 import type { EthereumContract } from './evm/EvmContract'
 import type { TronContract } from './tron/TronContract'
+import BN from 'bn.js'
 
 export enum Network {
   Tron = 'tron',
@@ -75,6 +76,7 @@ export interface TransactionManager<E> {
     attempt?: SendAttempt<E>
     error?: SendError
   }>
+  waitingForFunds(minimumBalance: BN, cb: (balance: BN) => void): Promise<void>;
 }
 
 export interface INetworkContract {

zp-relayer/workers/poolTxWorker.ts

@@ -1,12 +1,15 @@
 import { logger } from '@/lib/appLogger'
 import { JobState, PoolTx, WorkerTxType } from '@/queue/poolTxQueue'
+import { poolTxQueue } from '@/queue/poolTxQueue'
 import { sentTxQueue } from '@/queue/sentTxQueue'
 import { TX_QUEUE_NAME } from '@/utils/constants'
 import { withErrorLog, withMutex } from '@/utils/helpers'
 import { TxValidationError } from '@/validation/tx/common'
 import { Job, Worker } from 'bullmq'
 import Redis from 'ioredis'
 import type { IPoolWorkerConfig } from './workerTypes'
+import { isInsufficientBalanceError } from '@/utils/web3Errors'
+import { toBN } from 'web3-utils'
 
 const REVERTED_SET = 'reverted'
 const RECHECK_ERROR = 'Waiting for next check'
@@ -54,6 +57,7 @@ export async function createPoolTxWorker({ redis, mutex, pool, txManager }: IPoo
     const processResult = await pool.buildTx(job.data)
     const { data, func } = processResult
 
+    const gas = 2000000;
     const preparedTx = await txManager.prepareTx({
       txDesc: {
         to: pool.network.pool.address(), // TODO: mpc
@@ -68,13 +72,26 @@ export async function createPoolTxWorker({ redis, mutex, pool, txManager }: IPoo
       },
       extraData: {
         // TODO: abstract gas for EVM
-        gas: 2000000,
+        gas,
       },
     })
     const sendAttempt = preparedTx[1]
     try {
       await txManager.sendPreparedTx(preparedTx)
     } catch (e) {
+      if (isInsufficientBalanceError(e as Error)) {
+        if (sendAttempt.extraData.gas && sendAttempt.extraData.gasPrice) {
+          const minimumBalance = toBN(sendAttempt.extraData.gas).mul(toBN(sendAttempt.extraData.gasPrice));
+          logger.error('Insufficient balance, waiting for funds', { minimumBalance: minimumBalance.toString(10) })
+
+          await Promise.all([poolTxQueue.pause(), sentTxQueue.pause()])
+          txManager.waitingForFunds(
+            minimumBalance,
+            () => Promise.all([poolTxQueue.resume(), sentTxQueue.resume()])
+          )
+        }
+      }
+
       logger.warn('Tx send failed; it will be re-sent later', {
         txHash: preparedTx[1].txHash,
         error: (e as Error).message,