From 56a2d451fc9d1ffc6796219981a15579028439f8 Mon Sep 17 00:00:00 2001 From: "E. Lynette Rayle" Date: Thu, 12 Dec 2024 18:02:31 -0500 Subject: [PATCH 1/3] Test adding expiration config for queue messages --- config/cdConfig.js | 1 + .../queuing/storageQueueManagerTests.js | 25 +++++++++++++++++++ .../providers/queuing/storageQueueTests.js | 0 3 files changed, 26 insertions(+) create mode 100644 test/unit/providers/queuing/storageQueueManagerTests.js create mode 100644 test/unit/providers/queuing/storageQueueTests.js diff --git a/config/cdConfig.js b/config/cdConfig.js index 37346cab..1b21f596 100644 --- a/config/cdConfig.js +++ b/config/cdConfig.js @@ -129,6 +129,7 @@ module.exports = { weights: { immediate: 3, soon: 2, normal: 3, later: 2 }, connectionString: config.get('CRAWLER_QUEUE_AZURE_CONNECTION_STRING') || cd_azblob.connection, queueName: config.get('CRAWLER_QUEUE_PREFIX') || 'cdcrawlerdev', + messageTimeToLive: config.get('CRAWLER_QUEUE_MESSAGE_EXPIRATION') || 60 * 60 * 24 * 7, // Default to 7 days visibilityTimeout: 8 * 60 * 60, // 8 hours visibilityTimeout_remainLocal: fetchedCacheTtlSeconds, maxDequeueCount: 5, diff --git a/test/unit/providers/queuing/storageQueueManagerTests.js b/test/unit/providers/queuing/storageQueueManagerTests.js new file mode 100644 index 00000000..dac9e2a9 --- /dev/null +++ b/test/unit/providers/queuing/storageQueueManagerTests.js @@ -0,0 +1,25 @@ +const sinon = require('sinon') +const { expect } = require('chai') +const StorageQueueManager = require('../../../../providers/queuing/storageQueueManager') +const StorageQueue = require('../../../../providers/queuing/storageQueue') + +describe('StorageQueueManager', () => { + let manager, clientStub + + beforeEach(() => { + clientStub = { + createMessage: sinon.stub().yields(null, {}) + } + manager = new StorageQueueManager('connectionString', { expiration: 3600 }) // 1 hour expiration + manager.client = clientStub + }) + + it('should create a queue with the correct expiration', async () => { + const queue = manager.createQueue('testQueue') + await queue.push({ body: 'test message' }) + + expect(clientStub.createMessage.calledOnce).to.be.true + const args = clientStub.createMessage.getCall(0).args + expect(args[2].messageTimeToLive).to.equal(3600) + }) +}) \ No newline at end of file diff --git a/test/unit/providers/queuing/storageQueueTests.js b/test/unit/providers/queuing/storageQueueTests.js new file mode 100644 index 00000000..e69de29b From cb056158defd6495802663302062d42ef16e6f59 Mon Sep 17 00:00:00 2001 From: "E. Lynette Rayle" Date: Thu, 12 Dec 2024 18:15:07 -0500 Subject: [PATCH 2/3] add storageQueueTests --- .../queuing/storageQueueManagerTests.js | 6 ++--- .../providers/queuing/storageQueueTests.js | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/test/unit/providers/queuing/storageQueueManagerTests.js b/test/unit/providers/queuing/storageQueueManagerTests.js index dac9e2a9..bc9af247 100644 --- a/test/unit/providers/queuing/storageQueueManagerTests.js +++ b/test/unit/providers/queuing/storageQueueManagerTests.js @@ -1,7 +1,7 @@ const sinon = require('sinon') const { expect } = require('chai') -const StorageQueueManager = require('../../../../providers/queuing/storageQueueManager') -const StorageQueue = require('../../../../providers/queuing/storageQueue') +const StorageQueueManager = require('../../../../ghcrawler/providers/queuing/storageQueueManager') +const StorageQueue = require('../../../../ghcrawler/providers/queuing/storageQueue') describe('StorageQueueManager', () => { let manager, clientStub @@ -10,7 +10,7 @@ describe('StorageQueueManager', () => { clientStub = { createMessage: sinon.stub().yields(null, {}) } - manager = new StorageQueueManager('connectionString', { expiration: 3600 }) // 1 hour expiration + manager = new StorageQueueManager('connectionString', { messageTimeToLive: 3600 }) // 1 hour expiration manager.client = clientStub }) diff --git a/test/unit/providers/queuing/storageQueueTests.js b/test/unit/providers/queuing/storageQueueTests.js index e69de29b..ec21f253 100644 --- a/test/unit/providers/queuing/storageQueueTests.js +++ b/test/unit/providers/queuing/storageQueueTests.js @@ -0,0 +1,22 @@ +const sinon = require('sinon') +const { expect } = require('chai') +const StorageQueue = require('../../../../ghcrawler/providers/queuing/storageQueue') + +describe('StorageQueue', () => { + let queue, clientStub + + beforeEach(() => { + clientStub = { + createMessage: sinon.stub().yields(null, {}) + } + queue = new StorageQueue(clientStub, 'testQueue', 'testQueueName', message => message, { messageTimeToLive: 3600 }) // 1 hour expiration + }) + + it('should add messages with the correct expiration', async () => { + await queue.push({ body: 'test message' }) + + expect(clientStub.createMessage.calledOnce).to.be.true + const args = clientStub.createMessage.getCall(0).args + expect(args[2].messageTimeToLive).to.equal(3600) + }) +}) \ No newline at end of file From ef7ecfda44283e10e52788f3e1890caa543317e6 Mon Sep 17 00:00:00 2001 From: "E. Lynette Rayle" Date: Fri, 13 Dec 2024 08:52:44 -0500 Subject: [PATCH 3/3] log options being passed with createMessage This will tell me two things: 1) Is this the method being used to log messages? 2) What options are being passed with the createMessage call? --- ghcrawler/providers/queuing/storageQueue.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ghcrawler/providers/queuing/storageQueue.js b/ghcrawler/providers/queuing/storageQueue.js index c52e1257..4d876c66 100644 --- a/ghcrawler/providers/queuing/storageQueue.js +++ b/ghcrawler/providers/queuing/storageQueue.js @@ -37,6 +37,7 @@ class StorageQueue { qlimit(this.options.parallelPush || 1)(request => { const body = JSON.stringify(request) return new Promise((resolve, reject) => { + this.logger.info(`Sending createMessage to ${this.queueName} with options ${JSON.stringify(option)}`) this.client.createMessage(this.queueName, body, option, (error, queueMessageResult) => { if (error) { return reject(error)