From 34d348bcf7ee435ffaab5044160774a616b1e536 Mon Sep 17 00:00:00 2001 From: Henry Tsai <17891086+thehenrytsai@users.noreply.github.com> Date: Fri, 13 Nov 2020 09:46:42 -0800 Subject: [PATCH] feat(ref-imp): #919 - Added feature to disable observer in bitcoin service --- lib/bitcoin/BitcoinProcessor.ts | 87 +++++++++++++------------- lib/bitcoin/IBitcoinConfig.ts | 2 +- tests/bitcoin/BitcoinProcessor.spec.ts | 71 ++++++++++++++++----- 3 files changed, 101 insertions(+), 59 deletions(-) diff --git a/lib/bitcoin/BitcoinProcessor.ts b/lib/bitcoin/BitcoinProcessor.ts index ad3f55908..caffbad45 100644 --- a/lib/bitcoin/BitcoinProcessor.ts +++ b/lib/bitcoin/BitcoinProcessor.ts @@ -57,18 +57,12 @@ export interface IBlockInfo { */ export default class BitcoinProcessor { - /** Prefix used to identify Sidetree transactions in Bitcoin's blockchain. */ - public readonly sidetreePrefix: string; - /** The first Sidetree block in Bitcoin's blockchain. */ public readonly genesisBlockNumber: number; /** Store for the state of sidetree transactions. */ private readonly transactionStore: MongoDbTransactionStore; - /** Number of seconds between transaction queries */ - public pollPeriod: number; - /** Days of notice before the wallet is depleted of all funds */ public lowBalanceNoticeDays: number; @@ -98,17 +92,13 @@ export default class BitcoinProcessor { private sidetreeTransactionParser: SidetreeTransactionParser; - private bitcoinDataDirectory: string | undefined; - /** at least 100 blocks per page unless reaching the last block */ private static readonly pageSizeInBlocks = 100; - public constructor (config: IBitcoinConfig, versionModels: VersionModel[]) { + public constructor (private config: IBitcoinConfig, versionModels: VersionModel[]) { this.versionManager = new VersionManager(versionModels); - this.sidetreePrefix = config.sidetreeTransactionPrefix; this.genesisBlockNumber = config.genesisBlockNumber; - this.bitcoinDataDirectory = config.bitcoinDataDirectory; this.serviceStateStore = new MongoDbServiceStateStore(config.mongoDbConnectionString, config.databaseName); this.blockMetadataStore = new MongoDbBlockMetadataStore(config.mongoDbConnectionString, config.databaseName); @@ -118,7 +108,6 @@ export default class BitcoinProcessor { BitcoinClient.convertBtcToSatoshis(config.bitcoinFeeSpendingCutoff), this.transactionStore); - this.pollPeriod = config.transactionPollPeriodInSeconds || 60; this.lowBalanceNoticeDays = config.lowBalanceNoticeInDays || 28; this.serviceInfoProvider = new ServiceInfoProvider('bitcoin'); @@ -133,7 +122,7 @@ export default class BitcoinProcessor { config.sidetreeTransactionFeeMarkupPercentage || 0, config.defaultTransactionFeeInSatoshisPerKB); - this.sidetreeTransactionParser = new SidetreeTransactionParser(this.bitcoinClient, this.sidetreePrefix); + this.sidetreeTransactionParser = new SidetreeTransactionParser(this.bitcoinClient, this.config.sidetreeTransactionPrefix); this.lockResolver = new LockResolver( @@ -170,34 +159,39 @@ export default class BitcoinProcessor { await this.bitcoinClient.initialize(); await this.mongoDbLockTransactionStore.initialize(); - await this.upgradeDatabaseIfNeeded(); + // Only observe transactions if polling is enabled. + if (this.config.transactionPollPeriodInSeconds > 0) { + await this.upgradeDatabaseIfNeeded(); - // Current implementation records processing progress at block increments using `this.lastProcessedBlock`, - // so we need to trim the databases back to the last fully processed block. - this.lastProcessedBlock = await this.blockMetadataStore.getLast(); + // Current implementation records processing progress at block increments using `this.lastProcessedBlock`, + // so we need to trim the databases back to the last fully processed block. + this.lastProcessedBlock = await this.blockMetadataStore.getLast(); - const startingBlock = await this.getStartingBlockForPeriodicPoll(); + const startingBlock = await this.getStartingBlockForPeriodicPoll(); - if (startingBlock === undefined) { - console.info('Bitcoin processor state is ahead of bitcoind: skipping initialization'); - } else { - console.debug('Synchronizing blocks for sidetree transactions...'); - console.info(`Starting block: ${startingBlock.height} (${startingBlock.hash})`); - if (this.bitcoinDataDirectory) { - // This reads into the raw block files and parse to speed up the initial startup instead of rpc - await this.fastProcessTransactions(startingBlock); + if (startingBlock === undefined) { + console.info('Bitcoin processor state is ahead of Bitcoin Core, skipping initialization...'); } else { - await this.processTransactions(startingBlock); + console.debug('Synchronizing blocks for sidetree transactions...'); + console.info(`Starting block: ${startingBlock.height} (${startingBlock.hash})`); + if (this.config.bitcoinDataDirectory) { + // This reads into the raw block files and parse to speed up the initial startup instead of rpc + await this.fastProcessTransactions(startingBlock); + } else { + await this.processTransactions(startingBlock); + } } + + // Intentionally not await on the promise. + this.periodicPoll(); + } else { + console.warn(LogColor.yellow(`Transaction observer is disabled.`)); } - // NOTE: important to this initialization after we have processed all the blocks - // this is because that the lock monitor needs the normalized fee calculator to - // have all the data. + // NOTE: important to start lock monitor polling AFTER we have processed all the blocks above (for the case that this node is observing transactions), + // this is because that the lock monitor depends on lock resolver, and lock resolver currently needs the normalized fee calculator, + // even though lock monitor itself does not depend on normalized fee calculator. await this.lockMonitor.startPeriodicProcessing(); - - // Intentionally not await on the promise. - this.periodicPoll(); } private async upgradeDatabaseIfNeeded () { @@ -230,7 +224,7 @@ export default class BitcoinProcessor { * @param startingBlock the starting block to begin processing */ private async fastProcessTransactions (startingBlock: IBlockInfo) { - const bitcoinBlockDataIterator = new BitcoinBlockDataIterator(this.bitcoinDataDirectory!); + const bitcoinBlockDataIterator = new BitcoinBlockDataIterator(this.config.bitcoinDataDirectory!); const lastBlockHeight = await this.bitcoinClient.getCurrentBlockHeight(); const lastBlockInfo = await this.bitcoinClient.getBlockInfoFromHeight(lastBlockHeight); @@ -428,19 +422,28 @@ export default class BitcoinProcessor { } console.info(`Returning transactions since ${since ? 'block ' + TransactionNumber.getBlockNumber(since) : 'beginning'}...`); - // deep copy last processed block - const currentLastProcessedBlock = Object.assign({}, this.lastProcessedBlock!); - const [transactions, lastBlockSeen] = await this.getTransactionsSince(since, currentLastProcessedBlock.height); + + // We get the last processed block directly from DB because if this service has observer turned off, + // it would not have the last processed block cached in memory. + const lastProcessedBlock = await this.blockMetadataStore.getLast(); + if (lastProcessedBlock === undefined) { + return { + moreTransactions: false, + transactions: [] + }; + } + + const [transactions, lastBlockSeen] = await this.getTransactionsSince(since, lastProcessedBlock.height); // make sure the last processed block hasn't changed since before getting transactions // if changed, then a block reorg happened. - if (!await this.verifyBlock(currentLastProcessedBlock.height, currentLastProcessedBlock.hash)) { + if (!await this.verifyBlock(lastProcessedBlock.height, lastProcessedBlock.hash)) { console.info('Requested transactions hash mismatched blockchain'); throw new RequestError(ResponseStatus.BadRequest, SharedErrorCode.InvalidTransactionNumberOrTimeHash); } // if last processed block has not been seen, then there are more transactions - const moreTransactions = lastBlockSeen < currentLastProcessedBlock.height; + const moreTransactions = lastBlockSeen < lastProcessedBlock.height; return { transactions, @@ -487,7 +490,7 @@ export default class BitcoinProcessor { * @param minimumFee The minimum fee to be paid for this transaction. */ public async writeTransaction (anchorString: string, minimumFee: number) { - const sidetreeTransactionString = `${this.sidetreePrefix}${anchorString}`; + const sidetreeTransactionString = `${this.config.sidetreeTransactionPrefix}${anchorString}`; const sidetreeTransaction = await this.bitcoinClient.createSidetreeTransaction(sidetreeTransactionString, minimumFee); const transactionFee = sidetreeTransaction.transactionFee; console.info(`Fee: ${transactionFee}. Anchoring string ${anchorString}`); @@ -598,7 +601,7 @@ export default class BitcoinProcessor { * Will process transactions every interval seconds. * @param interval Number of seconds between each query */ - private async periodicPoll (interval: number = this.pollPeriod) { + private async periodicPoll (interval: number = this.config.transactionPollPeriodInSeconds) { try { // Defensive programming to prevent multiple polling loops even if this method is externally called multiple times. @@ -814,7 +817,7 @@ export default class BitcoinProcessor { * @returns a tuple of [transactions, lastBlockSeen] */ private async getTransactionsSince (since: number | undefined, maxBlockHeight: number): Promise<[TransactionModel[], number]> { - // test against undefined because 0 is falsey and this helps differenciate the behavior between 0 and undefined + // test against undefined because 0 is falsy and this helps differentiate the behavior between 0 and undefined let inclusiveBeginTransactionTime = since === undefined ? this.genesisBlockNumber : TransactionNumber.getBlockNumber(since); const transactionsToReturn: TransactionModel[] = []; diff --git a/lib/bitcoin/IBitcoinConfig.ts b/lib/bitcoin/IBitcoinConfig.ts index 05edbb8cd..d52f1c198 100644 --- a/lib/bitcoin/IBitcoinConfig.ts +++ b/lib/bitcoin/IBitcoinConfig.ts @@ -20,7 +20,7 @@ export default interface IBitcoinConfig { requestMaxRetries: number | undefined; sidetreeTransactionFeeMarkupPercentage: number; sidetreeTransactionPrefix: string; - transactionPollPeriodInSeconds: number | undefined; + transactionPollPeriodInSeconds: number; valueTimeLockUpdateEnabled: boolean; valueTimeLockAmountInBitcoins: number; valueTimeLockPollPeriodInSeconds: number; diff --git a/tests/bitcoin/BitcoinProcessor.spec.ts b/tests/bitcoin/BitcoinProcessor.spec.ts index 602a51216..eae41786b 100644 --- a/tests/bitcoin/BitcoinProcessor.spec.ts +++ b/tests/bitcoin/BitcoinProcessor.spec.ts @@ -66,7 +66,7 @@ describe('BitcoinProcessor', () => { let bitcoinProcessor: BitcoinProcessor; - // DB related spys. + // DB related spies. let blockMetadataStoreInitializeSpy: jasmine.Spy; let blockMetadataStoreAddSpy: jasmine.Spy; let blockMetadataStoreGetLastSpy: jasmine.Spy; @@ -84,7 +84,8 @@ describe('BitcoinProcessor', () => { let lockMonitorSpy: jasmine.Spy; beforeEach(() => { - bitcoinProcessor = new BitcoinProcessor(testConfig, versionModels); + const config = Object.assign({}, testConfig); // Clone the test config so that tests don't share the same object when it is being modified. + bitcoinProcessor = new BitcoinProcessor(config, versionModels); blockMetadataStoreInitializeSpy = spyOn(bitcoinProcessor['blockMetadataStore'], 'initialize'); serviceStateStoreInitializeSpy = spyOn(bitcoinProcessor['serviceStateStore'], 'initialize'); @@ -153,7 +154,7 @@ describe('BitcoinProcessor', () => { lowBalanceNoticeInDays: undefined, requestTimeoutInMilliseconds: undefined, requestMaxRetries: undefined, - transactionPollPeriodInSeconds: undefined, + transactionPollPeriodInSeconds: 60, sidetreeTransactionFeeMarkupPercentage: 0, valueTimeLockUpdateEnabled: true, valueTimeLockPollPeriodInSeconds: 60, @@ -164,8 +165,8 @@ describe('BitcoinProcessor', () => { const bitcoinProcessor = new BitcoinProcessor(config, versionModels); expect(bitcoinProcessor.genesisBlockNumber).toEqual(config.genesisBlockNumber); expect(bitcoinProcessor.lowBalanceNoticeDays).toEqual(28); - expect(bitcoinProcessor.pollPeriod).toEqual(60); - expect(bitcoinProcessor.sidetreePrefix).toEqual(config.sidetreeTransactionPrefix); + expect((bitcoinProcessor as any).config.transactionPollPeriodInSeconds).toEqual(60); + expect((bitcoinProcessor as any).config.sidetreeTransactionPrefix).toEqual(config.sidetreeTransactionPrefix); expect(bitcoinProcessor['transactionStore'].databaseName).toEqual(config.databaseName); expect(bitcoinProcessor['transactionStore']['serverUrl']).toEqual(config.mongoDbConnectionString); expect(bitcoinProcessor['bitcoinClient']['sidetreeTransactionFeeMarkupPercentage']).toEqual(0); @@ -200,6 +201,13 @@ describe('BitcoinProcessor', () => { done(); }); + it('should not start transaction observer polling if polling is turned off.', async () => { + (bitcoinProcessor as any).config.transactionPollPeriodInSeconds = 0; + + await bitcoinProcessor.initialize(); + expect(periodicPollSpy).not.toHaveBeenCalled(); + }); + it('should skip initialization if unable to find a starting block.', async (done) => { getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve(undefined)); @@ -239,7 +247,7 @@ describe('BitcoinProcessor', () => { }); it('should process all the blocks since its last known With fastProcessTransactions', async (done) => { - bitcoinProcessor['bitcoinDataDirectory'] = 'somePath'; + (bitcoinProcessor as any).config.bitcoinDataDirectory = 'somePath'; const fromNumber = randomNumber(); const fromHash = randomString(); @@ -306,16 +314,20 @@ describe('BitcoinProcessor', () => { }); }); - describe('transactions', () => { + describe('transactions()', () => { it('should get transactions since genesis capped by page size in blocks', async (done) => { const verifyMock = spyOn(bitcoinProcessor, 'verifyBlock' as any).and.returnValue(Promise.resolve(true)); + // return as many as page size const transactions: TransactionModel[] = createTransactions(BitcoinProcessor['pageSizeInBlocks'], bitcoinProcessor['genesisBlockNumber'], true); - bitcoinProcessor['lastProcessedBlock'] = { + + const mockLastProcessedBlock = { height: transactions[transactions.length - 1].transactionTime + 1, hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const getTransactionsStartingFromSpy = spyOn(bitcoinProcessor['transactionStore'], 'getTransactionsStartingFrom').and.callFake(() => { return Promise.resolve(transactions); }); @@ -333,11 +345,13 @@ describe('BitcoinProcessor', () => { const transactions = createTransactions(BitcoinProcessor['pageSizeInBlocks'], bitcoinProcessor['genesisBlockNumber'], true); // This makes the last transaction in the array "processing" const lastProcessedBlockHeightLess = transactions[transactions.length - 1].transactionTime - 1; - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: lastProcessedBlockHeightLess, hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const getTransactionsStartingFromSpy = spyOn(bitcoinProcessor['transactionStore'], 'getTransactionsStartingFrom').and.callFake(() => { return Promise.resolve(transactions); }); @@ -350,16 +364,29 @@ describe('BitcoinProcessor', () => { done(); }); + it('should return no transaction if last processed block in DB is not found.', async () => { + const mockLastProcessedBlock = undefined + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + + const fetchedTransactions = await bitcoinProcessor.transactions(); + expect(fetchedTransactions).toEqual({ + moreTransactions: false, + transactions: [] + }); + }); + it('should get transactions since genesis and handle complete last block', async (done) => { const verifyMock = spyOn(bitcoinProcessor, 'verifyBlock' as any).and.returnValue(Promise.resolve(true)); const transactions = createTransactions(BitcoinProcessor['pageSizeInBlocks'], bitcoinProcessor['genesisBlockNumber'], true); // make the last transaction time to be the same as last processed block height const lastProcessedBlockHeight = transactions[transactions.length - 1].transactionTime; - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: lastProcessedBlockHeight, hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const getTransactionsStartingFromSpy = spyOn(bitcoinProcessor['transactionStore'], 'getTransactionsStartingFrom').and.callFake(() => { return Promise.resolve(transactions); }); @@ -377,11 +404,13 @@ describe('BitcoinProcessor', () => { const lastProcessedBlockHeight = bitcoinProcessor['genesisBlockNumber']; const transactions = createTransactions(BitcoinProcessor['pageSizeInBlocks'], lastProcessedBlockHeight + 1, false); // make the last transaction time genesis so the transactions will all be out of bound - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: lastProcessedBlockHeight, hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const getTransactionsStartingFromSpy = spyOn(bitcoinProcessor['transactionStore'], 'getTransactionsStartingFrom').and.callFake(() => { return Promise.resolve(transactions); }); @@ -398,11 +427,13 @@ describe('BitcoinProcessor', () => { const verifyMock = spyOn(bitcoinProcessor, 'verifyBlock' as any).and.returnValue(Promise.resolve(true)); // make the last transaction time genesis + 1000 so it needs to call getTransactionsStartingFrom multiple times const lastProcessedBlockHeight = bitcoinProcessor['genesisBlockNumber'] + 1000; - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: lastProcessedBlockHeight, hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const getTransactionsStartingFromSpy = spyOn(bitcoinProcessor['transactionStore'], 'getTransactionsStartingFrom').and.callFake((begin) => { if (begin === bitcoinProcessor['genesisBlockNumber'] + 500) { return Promise.resolve(createTransactions(1, begin + 5, false)); @@ -420,11 +451,13 @@ describe('BitcoinProcessor', () => { it('should return default if transactions is empty array', async (done) => { const verifyMock = spyOn(bitcoinProcessor, 'verifyBlock' as any).and.returnValue(Promise.resolve(true)); - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: bitcoinProcessor['genesisBlockNumber'] + 99999, // loop past this number and get nothing hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const getTransactionsStartingFromSpy = spyOn(bitcoinProcessor['transactionStore'], 'getTransactionsStartingFrom').and.callFake(() => { return Promise.resolve([]); }); @@ -443,11 +476,13 @@ describe('BitcoinProcessor', () => { const expectedTransactionNumber = TransactionNumber.construct(expectedHeight, 0); const lastBlockHeight = BitcoinProcessor['pageSizeInBlocks'] + expectedHeight - 1; // caps the result to avoid 2 sets of transactions const lastBlockHash = 'last block hash'; - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: lastBlockHeight, hash: lastBlockHash, previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const verifyMock = spyOn(bitcoinProcessor, 'verifyBlock' as any).and.callFake((height: number, hash: string) => { expect(height === expectedHeight || height === lastBlockHeight).toBeTruthy(); expect(hash === expectedHash || hash === lastBlockHash).toBeTruthy(); @@ -502,11 +537,13 @@ describe('BitcoinProcessor', () => { const expectedTransactionNumber = TransactionNumber.construct(expectedHeight, 0); const verifyMock = spyOn(bitcoinProcessor, 'verifyBlock' as any).and.returnValues(Promise.resolve(true), Promise.resolve(false)); const getTransactionsSinceMock = spyOn(bitcoinProcessor, 'getTransactionsSince' as any).and.returnValue([[], 0]); - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: 1234, hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + try { await bitcoinProcessor.transactions(expectedTransactionNumber, expectedHash); fail('expected to throw'); @@ -521,11 +558,13 @@ describe('BitcoinProcessor', () => { }); it('should make moreTransactions true when last block processed is not reached', async (done) => { - bitcoinProcessor['lastProcessedBlock'] = { + const mockLastProcessedBlock = { height: Number.MAX_SAFE_INTEGER, // this is unreachable hash: 'some hash', previousHash: 'previous hash' }; + blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock)); + const expectedHeight = randomNumber(); const expectedHash = randomString(); const expectedTransactionNumber = TransactionNumber.construct(expectedHeight, 0);