From 72ea950ea251aa12f879ba19c0b5dfeb6a093da2 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Tue, 19 Nov 2024 16:53:14 +0100 Subject: [PATCH 01/52] feat(job-scheduler): add telemetry support to the job scheduler --- src/classes/job-scheduler.ts | 77 ++++++++++++++++++++----------- src/enums/telemetry-attributes.ts | 1 + tests/test_job_scheduler.ts | 6 +++ tests/test_telemetry_interface.ts | 50 ++++++++++++++++++++ 4 files changed, 108 insertions(+), 26 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index dee044e494..33e4765031 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -4,6 +4,7 @@ import { JobsOptions, RepeatStrategy } from '../types'; import { Job } from './job'; import { QueueBase } from './queue-base'; import { RedisConnection } from './redis-connection'; +import { SpanKind, TelemetryAttributes } from '../enums'; export interface JobSchedulerJson { key: string; // key is actually the job scheduler id @@ -24,6 +25,7 @@ export class JobScheduler extends QueueBase { opts: RepeatBaseOptions, Connection?: typeof RedisConnection, ) { + console.log('JobScheduler constructor', name, opts); super(name, opts, Connection); this.repeatStrategy = @@ -46,6 +48,12 @@ export class JobScheduler extends QueueBase { ); } + if (!pattern && !every) { + throw new Error( + 'Either .pattern or .every options must be defined for this repeatable job', + ); + } + if (repeatOpts.immediately && repeatOpts.startDate) { throw new Error( 'Both .immediately and .startDate options are defined for this repeatable job', @@ -77,7 +85,7 @@ export class JobScheduler extends QueueBase { now = startMillis > now ? startMillis : now; } - let nextMillis; + let nextMillis: number; if (every) { nextMillis = prevMillis + every; @@ -92,7 +100,7 @@ export class JobScheduler extends QueueBase { const multi = (await this.client).multi(); if (nextMillis) { if (override) { - await this.scripts.addJobScheduler( + this.scripts.addJobScheduler( (multi) as RedisClient, jobSchedulerId, nextMillis, @@ -105,37 +113,54 @@ export class JobScheduler extends QueueBase { }, ); } else { - await this.scripts.updateJobSchedulerNextMillis( + this.scripts.updateJobSchedulerNextMillis( (multi) as RedisClient, jobSchedulerId, nextMillis, ); } - const job = this.createNextJob( - (multi) as RedisClient, - jobName, - nextMillis, - jobSchedulerId, - { ...opts, repeat: filteredRepeatOpts }, - jobData, - iterationCount, + return this.trace>( + SpanKind.PRODUCER, + 'add', + `${this.name}.${jobName}`, + async (span, srcPropagationMedatada) => { + const job = this.createNextJob( + (multi) as RedisClient, + jobName, + nextMillis, + jobSchedulerId, + { + ...opts, + repeat: filteredRepeatOpts, + telemetryMetadata: srcPropagationMedatada, + }, + jobData, + iterationCount, + ); + + const results = await multi.exec(); // multi.exec returns an array of results [ err, result ][] + + // Check if there are any errors + const erroredResult = results.find(result => result[0]); + if (erroredResult) { + throw new Error( + `Error upserting job scheduler ${jobSchedulerId} - ${erroredResult[0]}`, + ); + } + + // Get last result with the job id + const lastResult = results.pop(); + job.id = lastResult[1] as string; + + span?.setAttributes({ + [TelemetryAttributes.JobSchedulerId]: jobSchedulerId, + [TelemetryAttributes.JobId]: job.id, + }); + + return job; + }, ); - - const results = await multi.exec(); // multi.exec returns an array of results [ err, result ][] - - // Check if there are any errors - const erroredResult = results.find(result => result[0]); - if (erroredResult) { - throw new Error( - `Error upserting job scheduler ${jobSchedulerId} - ${erroredResult[0]}`, - ); - } - - // Get last result with the job id - const lastResult = results.pop(); - job.id = lastResult[1] as string; - return job; } } diff --git a/src/enums/telemetry-attributes.ts b/src/enums/telemetry-attributes.ts index 3d5aa3d528..c28853df99 100644 --- a/src/enums/telemetry-attributes.ts +++ b/src/enums/telemetry-attributes.ts @@ -31,6 +31,7 @@ export enum TelemetryAttributes { JobResult = 'bullmq.job.result', JobFailedReason = 'bullmq.job.failed.reason', FlowName = 'bullmq.flow.name', + JobSchedulerId = 'bullmq.job.scheduler.id', } export enum SpanKind { diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 270ad20bdc..fa4e950c72 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1809,6 +1809,12 @@ describe('Job Scheduler', function () { ); }); + it('should throw an error when not specifying .pattern or .every', async function () { + await expect(queue.upsertJobScheduler('repeat', {})).to.be.rejectedWith( + 'Either .pattern or .every options must be defined for this repeatable job', + ); + }); + it('should throw an error when using .immediately and .startDate simultaneously', async function () { await expect( queue.upsertJobScheduler('repeat', { diff --git a/tests/test_telemetry_interface.ts b/tests/test_telemetry_interface.ts index 13e48cc77d..d0ffe5c6c0 100644 --- a/tests/test_telemetry_interface.ts +++ b/tests/test_telemetry_interface.ts @@ -234,6 +234,56 @@ describe('Telemetry', () => { }); }); + describe('Queue.upsertJobScheduler', async () => { + it('should correctly interact with telemetry when adding a job scheduler', async () => { + const jobSchedulerId = 'testJobScheduler'; + const data = { foo: 'bar' }; + + await queue.upsertJobScheduler( + jobSchedulerId, + { every: 1000, endDate: Date.now() + 1000 }, + { name: 'repeatable-job', data }, + ); + + const activeContext = telemetryClient.contextManager.active(); + const span = activeContext.getSpan?.() as MockSpan; + expect(span).to.be.an.instanceOf(MockSpan); + expect(span.name).to.equal(`add ${queueName}.repeatable-job`); + expect(span.options?.kind).to.equal(SpanKind.PRODUCER); + expect(span.attributes[TelemetryAttributes.JobSchedulerId]).to.equal( + jobSchedulerId, + ); + expect(span.attributes[TelemetryAttributes.JobId]).to.be.a('string'); + expect(span.attributes[TelemetryAttributes.JobId]).to.include( + `repeat:${jobSchedulerId}:`, + ); + }); + + it('should correctly handle errors and record them in telemetry for upsertJobScheduler', async () => { + const recordExceptionSpy = sinon.spy( + MockSpan.prototype, + 'recordException', + ); + + try { + await queue.upsertJobScheduler( + 'testJobScheduler', + { endDate: 0 }, + { data: { foo: 'bar' } }, + ); + } catch (e) { + assert(recordExceptionSpy.calledOnce); + const recordedError = recordExceptionSpy.firstCall.args[0]; + assert.equal( + recordedError.message, + 'End date must be greater than current timestamp', + ); + } finally { + recordExceptionSpy.restore(); + } + }); + }); + describe('Worker.processJob', async () => { it('should correctly interact with telemetry when processing a job', async () => { const job = await queue.add('testJob', { foo: 'bar' }); From 5bab0f13c4bde89d66c6a836f92f3394949e8587 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Tue, 19 Nov 2024 17:14:17 +0100 Subject: [PATCH 02/52] test(job-scheduler): fix broken test --- src/classes/job-scheduler.ts | 1 - tests/test_telemetry_interface.ts | 15 ++++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 33e4765031..8bba169f92 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -25,7 +25,6 @@ export class JobScheduler extends QueueBase { opts: RepeatBaseOptions, Connection?: typeof RedisConnection, ) { - console.log('JobScheduler constructor', name, opts); super(name, opts, Connection); this.repeatStrategy = diff --git a/tests/test_telemetry_interface.ts b/tests/test_telemetry_interface.ts index d0ffe5c6c0..828d701339 100644 --- a/tests/test_telemetry_interface.ts +++ b/tests/test_telemetry_interface.ts @@ -16,6 +16,7 @@ import { } from '../src/interfaces'; import * as sinon from 'sinon'; import { SpanKind, TelemetryAttributes } from '../src/enums'; +import { JobScheduler } from '../src/classes/job-scheduler'; describe('Telemetry', () => { type ExtendedException = Exception & { @@ -265,19 +266,23 @@ describe('Telemetry', () => { 'recordException', ); + const errMessage = 'Error creating job'; + + // Force an exception on the job schedulers private method createNextJob + (JobScheduler).prototype.createNextJob = () => { + throw new Error(errMessage); + }; + try { await queue.upsertJobScheduler( 'testJobScheduler', - { endDate: 0 }, + { every: 1000 }, { data: { foo: 'bar' } }, ); } catch (e) { assert(recordExceptionSpy.calledOnce); const recordedError = recordExceptionSpy.firstCall.args[0]; - assert.equal( - recordedError.message, - 'End date must be greater than current timestamp', - ); + assert.equal(recordedError.message, errMessage); } finally { recordExceptionSpy.restore(); } From 6ac3d0936e2a5cde803666c79604cb614444e5f8 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Tue, 19 Nov 2024 17:24:50 +0100 Subject: [PATCH 03/52] test(job-scheduler): add test to verify returned job --- tests/test_job_scheduler.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index fa4e950c72..f0f8ad5e4d 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1827,6 +1827,20 @@ describe('Job Scheduler', function () { ); }); + it("should return a valid job with the job's options and data passed as the job template", async function () { + const repeatOpts = { + every: 1000, + }; + + const job = await queue.upsertJobScheduler('test', repeatOpts, { + data: { foo: 'bar' }, + }); + + expect(job).to.be.ok; + expect(job!.data.foo).to.be.eql('bar'); + expect(job!.opts.repeat!.every).to.be.eql(1000); + }); + it('should emit a waiting event when adding a repeatable job to the waiting list', async function () { const date = new Date('2017-02-07 9:24:00'); this.clock.setSystemTime(date); From 26b2bc6386d639a35f87ab226dbeda3204a5ddd7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 19 Nov 2024 17:53:37 +0000 Subject: [PATCH 04/52] chore(release): 5.28.0 [skip ci] # [5.28.0](https://github.com/taskforcesh/bullmq/compare/v5.27.0...v5.28.0) (2024-11-19) ### Features * **job-scheduler:** add telemetry support to the job scheduler ([72ea950](https://github.com/taskforcesh/bullmq/commit/72ea950ea251aa12f879ba19c0b5dfeb6a093da2)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 75d831e5eb..6909a35a56 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +# [5.28.0](https://github.com/taskforcesh/bullmq/compare/v5.27.0...v5.28.0) (2024-11-19) + + +### Features + +* **job-scheduler:** add telemetry support to the job scheduler ([72ea950](https://github.com/taskforcesh/bullmq/commit/72ea950ea251aa12f879ba19c0b5dfeb6a093da2)) + # [5.27.0](https://github.com/taskforcesh/bullmq/compare/v5.26.2...v5.27.0) (2024-11-19) diff --git a/package.json b/package.json index c258ba6e9e..f8c1578bc1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.27.0", + "version": "5.28.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 192be5e052..492bb6cd43 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.27.0'; +export const version = '5.28.0'; From 5fbb075dd4abd51cc84a59575261de84e56633d8 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Wed, 20 Nov 2024 07:01:02 -0600 Subject: [PATCH 05/52] fix(scheduler): use Job class from getter for extension (#2917) --- src/classes/job-scheduler.ts | 2 +- src/classes/queue-getters.ts | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 8bba169f92..63c0f8199c 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -194,7 +194,7 @@ export class JobScheduler extends QueueBase { mergedOpts.repeat = { ...opts.repeat, count: currentCount }; - const job = new Job(this, name, data, mergedOpts, jobId); + const job = new this.Job(this, name, data, mergedOpts, jobId); job.addJob(client); return job; diff --git a/src/classes/queue-getters.ts b/src/classes/queue-getters.ts index 8740cf5df4..29e0e9d1c3 100644 --- a/src/classes/queue-getters.ts +++ b/src/classes/queue-getters.ts @@ -45,13 +45,6 @@ export class QueueGetters extends QueueBase { }); } - /** - * Helper to easily extend Job class calls. - */ - protected get Job(): typeof Job { - return Job; - } - private sanitizeJobTypes(types: JobType[] | JobType | undefined): JobType[] { const currentTypes = typeof types === 'string' ? [types] : types; From a9ed0c1cbfc96a0363a6ad51de7de7991293e251 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 20 Nov 2024 13:02:10 +0000 Subject: [PATCH 06/52] chore(release): 5.28.1 [skip ci] ## [5.28.1](https://github.com/taskforcesh/bullmq/compare/v5.28.0...v5.28.1) (2024-11-20) ### Bug Fixes * **scheduler:** use Job class from getter for extension ([#2917](https://github.com/taskforcesh/bullmq/issues/2917)) ([5fbb075](https://github.com/taskforcesh/bullmq/commit/5fbb075dd4abd51cc84a59575261de84e56633d8)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 6909a35a56..14727a473a 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.28.1](https://github.com/taskforcesh/bullmq/compare/v5.28.0...v5.28.1) (2024-11-20) + + +### Bug Fixes + +* **scheduler:** use Job class from getter for extension ([#2917](https://github.com/taskforcesh/bullmq/issues/2917)) ([5fbb075](https://github.com/taskforcesh/bullmq/commit/5fbb075dd4abd51cc84a59575261de84e56633d8)) + # [5.28.0](https://github.com/taskforcesh/bullmq/compare/v5.27.0...v5.28.0) (2024-11-19) diff --git a/package.json b/package.json index f8c1578bc1..324686bd68 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.28.0", + "version": "5.28.1", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 492bb6cd43..b2d0a9710e 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.28.0'; +export const version = '5.28.1'; From 34c23485bcb32b3c69046b2fb37e5db8927561ce Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Thu, 21 Nov 2024 19:19:39 -0600 Subject: [PATCH 07/52] fix(queue): change _jobScheduler from private to protected for extension (#2920) --- src/classes/index.ts | 1 + src/classes/queue.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/classes/index.ts b/src/classes/index.ts index ad2158913c..88c348d845 100644 --- a/src/classes/index.ts +++ b/src/classes/index.ts @@ -5,6 +5,7 @@ export * from './child-processor'; export * from './errors'; export * from './flow-producer'; export * from './job'; +export * from './job-scheduler'; // export * from './main'; this file must not be exported // export * from './main-worker'; this file must not be exported export * from './queue-base'; diff --git a/src/classes/queue.ts b/src/classes/queue.ts index b527c73509..2042b180eb 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -153,7 +153,7 @@ export class Queue< protected libName = 'bullmq'; private _repeat?: Repeat; // To be deprecated in v6 in favor of JobScheduler - private _jobScheduler?: JobScheduler; + protected _jobScheduler?: JobScheduler; constructor( name: string, From 4d3eb88f64e59c5d0cf80895c0af528700788771 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 22 Nov 2024 01:20:45 +0000 Subject: [PATCH 08/52] chore(release): 5.28.2 [skip ci] ## [5.28.2](https://github.com/taskforcesh/bullmq/compare/v5.28.1...v5.28.2) (2024-11-22) ### Bug Fixes * **queue:** change _jobScheduler from private to protected for extension ([#2920](https://github.com/taskforcesh/bullmq/issues/2920)) ([34c2348](https://github.com/taskforcesh/bullmq/commit/34c23485bcb32b3c69046b2fb37e5db8927561ce)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 14727a473a..e54c683c55 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.28.2](https://github.com/taskforcesh/bullmq/compare/v5.28.1...v5.28.2) (2024-11-22) + + +### Bug Fixes + +* **queue:** change _jobScheduler from private to protected for extension ([#2920](https://github.com/taskforcesh/bullmq/issues/2920)) ([34c2348](https://github.com/taskforcesh/bullmq/commit/34c23485bcb32b3c69046b2fb37e5db8927561ce)) + ## [5.28.1](https://github.com/taskforcesh/bullmq/compare/v5.28.0...v5.28.1) (2024-11-20) diff --git a/package.json b/package.json index 324686bd68..db4e0ff1cf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.28.1", + "version": "5.28.2", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index b2d0a9710e..4f194c9c8f 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.28.1'; +export const version = '5.28.2'; From 09f257196f6d5a6690edbf55f12d585cec86ee8f Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Thu, 21 Nov 2024 15:54:25 +0100 Subject: [PATCH 09/52] feat(queue): refactor a protected addJob method allowing telemetry extensions --- src/classes/queue.ts | 91 ++++++++++++++++++++++++++----------------- src/classes/worker.ts | 2 +- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 2042b180eb..89373010ae 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -311,47 +311,66 @@ export class Queue< opts = { ...opts, telemetryMetadata: srcPropagationMedatada }; } - if (opts && opts.repeat) { - if (opts.repeat.endDate) { - if (+new Date(opts.repeat.endDate) < Date.now()) { - throw new Error( - 'End date must be greater than current timestamp', - ); - } - } + const job = await this.addJob(name, data, opts); - return (await this.repeat).updateRepeatableJob< - DataType, - ResultType, - NameType - >(name, data, { ...this.jobsOpts, ...opts }, { override: true }); - } else { - const jobId = opts?.jobId; + span?.setAttributes({ + [TelemetryAttributes.JobName]: name, + [TelemetryAttributes.JobId]: job.id, + }); - if (jobId == '0' || jobId?.startsWith('0:')) { - throw new Error("JobId cannot be '0' or start with 0:"); - } + return job; + }, + ); + } - const job = await this.Job.create( - this as MinimalQueue, - name, - data, - { - ...this.jobsOpts, - ...opts, - jobId, - }, - ); - this.emit('waiting', job as JobBase); + /** + * addJob is a telemetry free version of the add method, useful in order to wrap it + * with custom telemetry on subclasses. + * + * @param name + * @param data + * @param opts + * + * @returns Job + */ + protected async addJob( + name: NameType, + data: DataType, + opts?: JobsOptions, + ): Promise> { + if (opts && opts.repeat) { + if (opts.repeat.endDate) { + if (+new Date(opts.repeat.endDate) < Date.now()) { + throw new Error('End date must be greater than current timestamp'); + } + } - span?.setAttributes({ - [TelemetryAttributes.JobId]: job.id, - }); + return (await this.repeat).updateRepeatableJob< + DataType, + ResultType, + NameType + >(name, data, { ...this.jobsOpts, ...opts }, { override: true }); + } else { + const jobId = opts?.jobId; - return job; - } - }, - ); + if (jobId == '0' || jobId?.startsWith('0:')) { + throw new Error("JobId cannot be '0' or start with 0:"); + } + + const job = await this.Job.create( + this as MinimalQueue, + name, + data, + { + ...this.jobsOpts, + ...opts, + jobId, + }, + ); + this.emit('waiting', job as JobBase); + + return job; + } } /** diff --git a/src/classes/worker.ts b/src/classes/worker.ts index e5092ee772..58fc702f9d 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -995,7 +995,7 @@ will never work with more accuracy than 1ms. */ * This method waits for current jobs to finalize before returning. * * @param force - Use force boolean parameter if you do not want to wait for - * current jobs to be processed. When using telemetry, be mindful that it can + * current jobs to be processed. When using telemetry, be mindful that it can * interfere with the proper closure of spans, potentially preventing them from being exported. * * @returns Promise that resolves when the worker has been closed. From 88a720c8f712ec717ba074b79cc5525b3fb537d1 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Thu, 21 Nov 2024 17:31:39 +0100 Subject: [PATCH 10/52] test(worker): improve flaky test --- tests/test_worker.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 75539d0c5b..c8f7e38f11 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -510,9 +510,17 @@ describe('workers', function () { await worker.close(); }); - it('do not call moveToActive more than number of jobs + 1', async () => { + it('do not call moveToActive more than number of jobs + 2', async () => { const numJobs = 50; let completedJobs = 0; + + const jobs: Promise[] = []; + for (let i = 0; i < numJobs; i++) { + jobs.push(queue.add('test', { foo: 'bar' })); + } + + await Promise.all(jobs); + const worker = new Worker( queueName, async job => { @@ -521,7 +529,6 @@ describe('workers', function () { }, { connection, prefix, concurrency: 100 }, ); - await worker.waitUntilReady(); // Add spy to worker.moveToActive const spy = sinon.spy(worker, 'moveToActive'); @@ -529,14 +536,9 @@ describe('workers', function () { await worker.blockingConnection.client, 'bzpopmin', ); + await worker.waitUntilReady(); - for (let i = 0; i < numJobs; i++) { - const job = await queue.add('test', { foo: 'bar' }); - expect(job.id).to.be.ok; - expect(job.data.foo).to.be.eql('bar'); - } - - expect(bclientSpy.callCount).to.be.equal(1); + expect(bclientSpy.callCount).to.be.equal(0); await new Promise((resolve, reject) => { worker.on('completed', (job: Job, result: any) => { @@ -547,9 +549,11 @@ describe('workers', function () { }); }); + expect(completedJobs).to.be.equal(numJobs); + expect(bclientSpy.callCount).to.be.equal(2); + // Check moveToActive was called numJobs + 2 times expect(spy.callCount).to.be.equal(numJobs + 2); - expect(bclientSpy.callCount).to.be.equal(3); await worker.close(); }); From bf03f864fb64c86bf75d52dad2c894a5e5e39e44 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 22 Nov 2024 09:11:19 +0000 Subject: [PATCH 11/52] chore(release): 5.29.0 [skip ci] # [5.29.0](https://github.com/taskforcesh/bullmq/compare/v5.28.2...v5.29.0) (2024-11-22) ### Features * **queue:** refactor a protected addJob method allowing telemetry extensions ([09f2571](https://github.com/taskforcesh/bullmq/commit/09f257196f6d5a6690edbf55f12d585cec86ee8f)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index e54c683c55..e483290cde 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +# [5.29.0](https://github.com/taskforcesh/bullmq/compare/v5.28.2...v5.29.0) (2024-11-22) + + +### Features + +* **queue:** refactor a protected addJob method allowing telemetry extensions ([09f2571](https://github.com/taskforcesh/bullmq/commit/09f257196f6d5a6690edbf55f12d585cec86ee8f)) + ## [5.28.2](https://github.com/taskforcesh/bullmq/compare/v5.28.1...v5.28.2) (2024-11-22) diff --git a/package.json b/package.json index db4e0ff1cf..e9a733ea4d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.28.2", + "version": "5.29.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 4f194c9c8f..41e46fd8ca 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.28.2'; +export const version = '5.29.0'; From 14ca7f44f31a393a8b6d0ce4ed244e0063198879 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Fri, 22 Nov 2024 23:47:20 -0600 Subject: [PATCH 12/52] fix(scheduler): remove deprecation warning on immediately option (#2923) --- src/classes/queue-base.ts | 2 +- src/classes/worker.ts | 2 +- src/interfaces/repeat-options.ts | 3 --- tests/test_telemetry_interface.ts | 3 +-- tests/test_worker.ts | 14 +++++++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index 2760a7b108..9e2008f958 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -12,7 +12,7 @@ import { RedisConnection } from './redis-connection'; import { Job } from './job'; import { KeysMap, QueueKeys } from './queue-keys'; import { Scripts } from './scripts'; -import { TelemetryAttributes, SpanKind } from '../enums'; +import { SpanKind } from '../enums'; /** * @class QueueBase diff --git a/src/classes/worker.ts b/src/classes/worker.ts index 58fc702f9d..acc6f1d533 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -191,7 +191,7 @@ export class Worker< private waiting: Promise | null = null; private _repeat: Repeat; // To be deprecated in v6 in favor of Job Scheduler - private _jobScheduler: JobScheduler; + protected _jobScheduler: JobScheduler; protected paused: Promise; protected processFn: Processor; diff --git a/src/interfaces/repeat-options.ts b/src/interfaces/repeat-options.ts index 67d9450a96..fe71d27d12 100644 --- a/src/interfaces/repeat-options.ts +++ b/src/interfaces/repeat-options.ts @@ -33,9 +33,6 @@ export interface RepeatOptions extends Omit { /** * Repeated job should start right now * ( work only with every settings) - * - * @deprecated - * */ immediately?: boolean; diff --git a/tests/test_telemetry_interface.ts b/tests/test_telemetry_interface.ts index 828d701339..b598475d86 100644 --- a/tests/test_telemetry_interface.ts +++ b/tests/test_telemetry_interface.ts @@ -2,7 +2,7 @@ import { expect, assert } from 'chai'; import { default as IORedis } from 'ioredis'; import { after, beforeEach, describe, it, before } from 'mocha'; import { v4 } from 'uuid'; -import { FlowProducer, Queue, Worker } from '../src/classes'; +import { FlowProducer, JobScheduler, Queue, Worker } from '../src/classes'; import { removeAllQueueData } from '../src/utils'; import { Telemetry, @@ -16,7 +16,6 @@ import { } from '../src/interfaces'; import * as sinon from 'sinon'; import { SpanKind, TelemetryAttributes } from '../src/enums'; -import { JobScheduler } from '../src/classes/job-scheduler'; describe('Telemetry', () => { type ExtendedException = Exception & { diff --git a/tests/test_worker.ts b/tests/test_worker.ts index c8f7e38f11..ad9e8354a4 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -482,16 +482,20 @@ describe('workers', function () { // Add spy to worker.moveToActive const spy = sinon.spy(worker, 'moveToActive'); const bclientSpy = sinon.spy( - await worker.blockingConnection.client, + await (worker as any).blockingConnection.client, 'bzpopmin', ); - for (let i = 0; i < numJobs; i++) { - const job = await queue.add('test', { foo: 'bar' }); - expect(job.id).to.be.ok; - expect(job.data.foo).to.be.eql('bar'); + const jobsData: { name: string; data: any }[] = []; + for (let j = 0; j < numJobs; j++) { + jobsData.push({ + name: 'test', + data: { foo: 'bar' }, + }); } + await queue.addBulk(jobsData); + expect(bclientSpy.callCount).to.be.equal(1); await new Promise((resolve, reject) => { From cc6cfc2231dbcdc55f14c92ebe05243b99eebb09 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 23 Nov 2024 05:48:30 +0000 Subject: [PATCH 13/52] chore(release): 5.29.1 [skip ci] ## [5.29.1](https://github.com/taskforcesh/bullmq/compare/v5.29.0...v5.29.1) (2024-11-23) ### Bug Fixes * **scheduler:** remove deprecation warning on immediately option ([#2923](https://github.com/taskforcesh/bullmq/issues/2923)) ([14ca7f4](https://github.com/taskforcesh/bullmq/commit/14ca7f44f31a393a8b6d0ce4ed244e0063198879)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index e483290cde..2b5ab07162 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.29.1](https://github.com/taskforcesh/bullmq/compare/v5.29.0...v5.29.1) (2024-11-23) + + +### Bug Fixes + +* **scheduler:** remove deprecation warning on immediately option ([#2923](https://github.com/taskforcesh/bullmq/issues/2923)) ([14ca7f4](https://github.com/taskforcesh/bullmq/commit/14ca7f44f31a393a8b6d0ce4ed244e0063198879)) + # [5.29.0](https://github.com/taskforcesh/bullmq/compare/v5.28.2...v5.29.0) (2024-11-22) diff --git a/package.json b/package.json index e9a733ea4d..37e30fa43a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.29.0", + "version": "5.29.1", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 41e46fd8ca..f7e6775c35 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.29.0'; +export const version = '5.29.1'; From 71ce75c04b096b5593da0986c41a771add1a81ce Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Mon, 25 Nov 2024 21:33:49 -0600 Subject: [PATCH 14/52] feat(queue): add getDelayedCount method [python] (#2934) --- python/bullmq/queue.py | 3 +++ python/tests/queue_tests.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/bullmq/queue.py b/python/bullmq/queue.py index 0bcffc1fae..2672750ad6 100644 --- a/python/bullmq/queue.py +++ b/python/bullmq/queue.py @@ -254,6 +254,9 @@ def getJobState(self, job_id: str): def getCompletedCount(self): return self.getJobCountByTypes('completed') + def getDelayedCount(self): + return self.getJobCountByTypes('delayed') + def getFailedCount(self): return self.getJobCountByTypes('failed') diff --git a/python/tests/queue_tests.py b/python/tests/queue_tests.py index 94bdb7da6a..5ffeaf5f38 100644 --- a/python/tests/queue_tests.py +++ b/python/tests/queue_tests.py @@ -54,7 +54,7 @@ async def test_get_job_state(self): async def test_add_job_with_options(self): queue = Queue(queueName) data = {"foo": "bar"} - attempts = 3, + attempts = 3 delay = 1000 job = await queue.add("test-job", data=data, opts={"attempts": attempts, "delay": delay}) @@ -133,6 +133,18 @@ async def test_trim_events_manually_with_custom_prefix(self): await queue.obliterate() await queue.close() + async def test_get_delayed_count(self): + queue = Queue(queueName) + data = {"foo": "bar"} + delay = 1000 + await queue.add("test-job", data=data, opts={"delay": delay}) + await queue.add("test-job", data=data, opts={"delay": delay * 2}) + + count = await queue.getDelayedCount() + self.assertEqual(count, 2) + + await queue.close() + async def test_retry_failed_jobs(self): queue = Queue(queueName) job_count = 8 From 4ada3d5c318fd550ebb466e49203217fb6f1450b Mon Sep 17 00:00:00 2001 From: semantic-release Date: Tue, 26 Nov 2024 03:35:06 +0000 Subject: [PATCH 15/52] 2.11.0 Automatically generated by python-semantic-release --- docs/gitbook/python/changelog.md | 37 ++++++++++++++++++++++++++++++++ python/bullmq/__init__.py | 2 +- python/pyproject.toml | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/python/changelog.md b/docs/gitbook/python/changelog.md index 6bdea9577d..f06ba8ec17 100644 --- a/docs/gitbook/python/changelog.md +++ b/docs/gitbook/python/changelog.md @@ -2,6 +2,43 @@ +## v2.11.0 (2024-11-26) +### Feature +* **queue:** Add getDelayedCount method [python] ([#2934](https://github.com/taskforcesh/bullmq/issues/2934)) ([`71ce75c`](https://github.com/taskforcesh/bullmq/commit/71ce75c04b096b5593da0986c41a771add1a81ce)) +* **queue:** Refactor a protected addJob method allowing telemetry extensions ([`09f2571`](https://github.com/taskforcesh/bullmq/commit/09f257196f6d5a6690edbf55f12d585cec86ee8f)) +* **job-scheduler:** Add telemetry support to the job scheduler ([`72ea950`](https://github.com/taskforcesh/bullmq/commit/72ea950ea251aa12f879ba19c0b5dfeb6a093da2)) +* **queue:** Add rateLimit method ([#2896](https://github.com/taskforcesh/bullmq/issues/2896)) ([`db84ad5`](https://github.com/taskforcesh/bullmq/commit/db84ad51a945c754c3cd03e5e718cd8d0341a8b4)) +* **queue:** Add removeRateLimitKey method ([#2806](https://github.com/taskforcesh/bullmq/issues/2806)) ([`ff70613`](https://github.com/taskforcesh/bullmq/commit/ff706131bf642fb7544b9d15994d75b1edcb27dc)) +* Improve queue getters to use generic job type ([#2905](https://github.com/taskforcesh/bullmq/issues/2905)) ([`c9531ec`](https://github.com/taskforcesh/bullmq/commit/c9531ec7a49126a017611eb2fd2eaea8fcb5ada5)) +* **queue-events:** Add QueueEventsProducer for publishing custom events ([#2844](https://github.com/taskforcesh/bullmq/issues/2844)) ([`5eb03cd`](https://github.com/taskforcesh/bullmq/commit/5eb03cd7f27027191eb4bc4ed7386755fd9be1fb)) +* **flows:** Add telemetry support ([#2879](https://github.com/taskforcesh/bullmq/issues/2879)) ([`5ed154b`](https://github.com/taskforcesh/bullmq/commit/5ed154ba240dbe9eb5c22e27ad02e851c0f3cf69)) +* **scheduler:** Add getJobScheduler method (#2877) ref #2875 ([`956d98c`](https://github.com/taskforcesh/bullmq/commit/956d98c6890484742bb080919c70692234f28c69)) +* **queue:** Add a telemetry interface ([#2721](https://github.com/taskforcesh/bullmq/issues/2721)) ([`273b574`](https://github.com/taskforcesh/bullmq/commit/273b574e6b5628680990eb02e1930809c9cba5bb)) + +### Fix +* **scheduler:** Remove deprecation warning on immediately option ([#2923](https://github.com/taskforcesh/bullmq/issues/2923)) ([`14ca7f4`](https://github.com/taskforcesh/bullmq/commit/14ca7f44f31a393a8b6d0ce4ed244e0063198879)) +* **queue:** Change _jobScheduler from private to protected for extension ([#2920](https://github.com/taskforcesh/bullmq/issues/2920)) ([`34c2348`](https://github.com/taskforcesh/bullmq/commit/34c23485bcb32b3c69046b2fb37e5db8927561ce)) +* **scheduler:** Use Job class from getter for extension ([#2917](https://github.com/taskforcesh/bullmq/issues/2917)) ([`5fbb075`](https://github.com/taskforcesh/bullmq/commit/5fbb075dd4abd51cc84a59575261de84e56633d8)) +* **telemetry:** Do not set span on parent context if undefined ([`c417a23`](https://github.com/taskforcesh/bullmq/commit/c417a23bb28d9effa42115e954b18cc41c1fc043)) +* **queue:** Fix generics to be able to properly be extended ([`f2495e5`](https://github.com/taskforcesh/bullmq/commit/f2495e5ee9ecdb26492da510dc38730718cb28c5)) +* **job-scheculer:** Avoid hazards when upserting job schedulers concurrently ([`022f7b7`](https://github.com/taskforcesh/bullmq/commit/022f7b7d0a0ce14387ed2b9fed791e1f56e34770)) +* **connection:** Do not allow to set blockingConnection option ([#2851](https://github.com/taskforcesh/bullmq/issues/2851)) ([`9391cc2`](https://github.com/taskforcesh/bullmq/commit/9391cc22200914ecc8958972ebc580862a70f63c)) +* **repeatable:** Only apply immediately in the first iteration ([`f69cfbc`](https://github.com/taskforcesh/bullmq/commit/f69cfbcbc5516a854adbbc29b259d08e65a19705)) +* **scripts:** Set package version by default for extension ([#2887](https://github.com/taskforcesh/bullmq/issues/2887)) ([`b955340`](https://github.com/taskforcesh/bullmq/commit/b955340b940e4c1e330445526cd572e0ab25daa9)) +* **worker:** Allow retrieving concurrency value (#2883) fixes #2880 ([`52f6317`](https://github.com/taskforcesh/bullmq/commit/52f6317ecd2080a5c9684a4fe384e20d86f21de4)) +* **connection:** Set packageVersion as protected attribute for extension ([#2884](https://github.com/taskforcesh/bullmq/issues/2884)) ([`411ccae`](https://github.com/taskforcesh/bullmq/commit/411ccae9419e008d916be6cf71c4d57dd2a07b2b)) +* **deps:** Bump msgpackr to 1.1.2 to resolve ERR_BUFFER_OUT_OF_BOUNDS error (#2882) ref #2747 ([`4d2136c`](https://github.com/taskforcesh/bullmq/commit/4d2136cc6ba340e511a539c130c9a739fe1055d0)) + +### Documentation +* **pro:** Update changelog to v7.20.1 ([#2903](https://github.com/taskforcesh/bullmq/issues/2903)) ([`d23bbef`](https://github.com/taskforcesh/bullmq/commit/d23bbef253f732500a4438e3390372dd02117ffc)) +* Add minimum redis version #227 ([`88fff2c`](https://github.com/taskforcesh/bullmq/commit/88fff2c957d6d4b58ef6399ec56123b6da825d88)) +* Telemetry note on close force in worker ([`bf72d3f`](https://github.com/taskforcesh/bullmq/commit/bf72d3fc0ec3bbf73457a7024d8853a78293facd)) +* Add BullMQ Guru (Gurubase.io) badge ([#2888](https://github.com/taskforcesh/bullmq/issues/2888)) ([`b70dbdf`](https://github.com/taskforcesh/bullmq/commit/b70dbdf8e4e8f5b69868f10e98a7c8abd5b2fb23)) +* **patterns:** Fix token type in process-step-jobs ([#2881](https://github.com/taskforcesh/bullmq/issues/2881)) ([`974d096`](https://github.com/taskforcesh/bullmq/commit/974d0960c5990c725a0f13483ab7a8e7e5403949)) + +### Performance +* **marker:** Add base markers while consuming jobs to get workers busy (#2904) fixes #2842 ([`1759c8b`](https://github.com/taskforcesh/bullmq/commit/1759c8bc111cab9e43d5fccb4d8d2dccc9c39fb4)) + ## v2.10.1 (2024-10-26) ### Fix * **commands:** Add missing build statement when releasing [python] (#2869) fixes #2868 ([`ff2a47b`](https://github.com/taskforcesh/bullmq/commit/ff2a47b37c6b36ee1a725f91de2c6e4bcf8b011a)) diff --git a/python/bullmq/__init__.py b/python/bullmq/__init__.py index 1db7674593..8a8ee15cf7 100644 --- a/python/bullmq/__init__.py +++ b/python/bullmq/__init__.py @@ -3,7 +3,7 @@ A background job processor and message queue for Python based on Redis. """ -__version__ = "2.10.1" +__version__ = "2.11.0" __author__ = 'Taskforce.sh Inc.' __credits__ = 'Taskforce.sh Inc.' diff --git a/python/pyproject.toml b/python/pyproject.toml index 68ad2bc0da..601b202b37 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bullmq" -version = "2.10.1" +version = "2.11.0" description='BullMQ for Python' readme="README.md" authors = [ From d10a2fe4e20bb17d16836bbabc364ad5c23703f1 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Mon, 25 Nov 2024 22:14:50 -0600 Subject: [PATCH 16/52] docs(changelog): remove nodejs change docs from python v2.11.0 (#2935) --- docs/gitbook/python/changelog.md | 36 -------------------------------- 1 file changed, 36 deletions(-) diff --git a/docs/gitbook/python/changelog.md b/docs/gitbook/python/changelog.md index f06ba8ec17..ec2a2753ea 100644 --- a/docs/gitbook/python/changelog.md +++ b/docs/gitbook/python/changelog.md @@ -5,36 +5,6 @@ ## v2.11.0 (2024-11-26) ### Feature * **queue:** Add getDelayedCount method [python] ([#2934](https://github.com/taskforcesh/bullmq/issues/2934)) ([`71ce75c`](https://github.com/taskforcesh/bullmq/commit/71ce75c04b096b5593da0986c41a771add1a81ce)) -* **queue:** Refactor a protected addJob method allowing telemetry extensions ([`09f2571`](https://github.com/taskforcesh/bullmq/commit/09f257196f6d5a6690edbf55f12d585cec86ee8f)) -* **job-scheduler:** Add telemetry support to the job scheduler ([`72ea950`](https://github.com/taskforcesh/bullmq/commit/72ea950ea251aa12f879ba19c0b5dfeb6a093da2)) -* **queue:** Add rateLimit method ([#2896](https://github.com/taskforcesh/bullmq/issues/2896)) ([`db84ad5`](https://github.com/taskforcesh/bullmq/commit/db84ad51a945c754c3cd03e5e718cd8d0341a8b4)) -* **queue:** Add removeRateLimitKey method ([#2806](https://github.com/taskforcesh/bullmq/issues/2806)) ([`ff70613`](https://github.com/taskforcesh/bullmq/commit/ff706131bf642fb7544b9d15994d75b1edcb27dc)) -* Improve queue getters to use generic job type ([#2905](https://github.com/taskforcesh/bullmq/issues/2905)) ([`c9531ec`](https://github.com/taskforcesh/bullmq/commit/c9531ec7a49126a017611eb2fd2eaea8fcb5ada5)) -* **queue-events:** Add QueueEventsProducer for publishing custom events ([#2844](https://github.com/taskforcesh/bullmq/issues/2844)) ([`5eb03cd`](https://github.com/taskforcesh/bullmq/commit/5eb03cd7f27027191eb4bc4ed7386755fd9be1fb)) -* **flows:** Add telemetry support ([#2879](https://github.com/taskforcesh/bullmq/issues/2879)) ([`5ed154b`](https://github.com/taskforcesh/bullmq/commit/5ed154ba240dbe9eb5c22e27ad02e851c0f3cf69)) -* **scheduler:** Add getJobScheduler method (#2877) ref #2875 ([`956d98c`](https://github.com/taskforcesh/bullmq/commit/956d98c6890484742bb080919c70692234f28c69)) -* **queue:** Add a telemetry interface ([#2721](https://github.com/taskforcesh/bullmq/issues/2721)) ([`273b574`](https://github.com/taskforcesh/bullmq/commit/273b574e6b5628680990eb02e1930809c9cba5bb)) - -### Fix -* **scheduler:** Remove deprecation warning on immediately option ([#2923](https://github.com/taskforcesh/bullmq/issues/2923)) ([`14ca7f4`](https://github.com/taskforcesh/bullmq/commit/14ca7f44f31a393a8b6d0ce4ed244e0063198879)) -* **queue:** Change _jobScheduler from private to protected for extension ([#2920](https://github.com/taskforcesh/bullmq/issues/2920)) ([`34c2348`](https://github.com/taskforcesh/bullmq/commit/34c23485bcb32b3c69046b2fb37e5db8927561ce)) -* **scheduler:** Use Job class from getter for extension ([#2917](https://github.com/taskforcesh/bullmq/issues/2917)) ([`5fbb075`](https://github.com/taskforcesh/bullmq/commit/5fbb075dd4abd51cc84a59575261de84e56633d8)) -* **telemetry:** Do not set span on parent context if undefined ([`c417a23`](https://github.com/taskforcesh/bullmq/commit/c417a23bb28d9effa42115e954b18cc41c1fc043)) -* **queue:** Fix generics to be able to properly be extended ([`f2495e5`](https://github.com/taskforcesh/bullmq/commit/f2495e5ee9ecdb26492da510dc38730718cb28c5)) -* **job-scheculer:** Avoid hazards when upserting job schedulers concurrently ([`022f7b7`](https://github.com/taskforcesh/bullmq/commit/022f7b7d0a0ce14387ed2b9fed791e1f56e34770)) -* **connection:** Do not allow to set blockingConnection option ([#2851](https://github.com/taskforcesh/bullmq/issues/2851)) ([`9391cc2`](https://github.com/taskforcesh/bullmq/commit/9391cc22200914ecc8958972ebc580862a70f63c)) -* **repeatable:** Only apply immediately in the first iteration ([`f69cfbc`](https://github.com/taskforcesh/bullmq/commit/f69cfbcbc5516a854adbbc29b259d08e65a19705)) -* **scripts:** Set package version by default for extension ([#2887](https://github.com/taskforcesh/bullmq/issues/2887)) ([`b955340`](https://github.com/taskforcesh/bullmq/commit/b955340b940e4c1e330445526cd572e0ab25daa9)) -* **worker:** Allow retrieving concurrency value (#2883) fixes #2880 ([`52f6317`](https://github.com/taskforcesh/bullmq/commit/52f6317ecd2080a5c9684a4fe384e20d86f21de4)) -* **connection:** Set packageVersion as protected attribute for extension ([#2884](https://github.com/taskforcesh/bullmq/issues/2884)) ([`411ccae`](https://github.com/taskforcesh/bullmq/commit/411ccae9419e008d916be6cf71c4d57dd2a07b2b)) -* **deps:** Bump msgpackr to 1.1.2 to resolve ERR_BUFFER_OUT_OF_BOUNDS error (#2882) ref #2747 ([`4d2136c`](https://github.com/taskforcesh/bullmq/commit/4d2136cc6ba340e511a539c130c9a739fe1055d0)) - -### Documentation -* **pro:** Update changelog to v7.20.1 ([#2903](https://github.com/taskforcesh/bullmq/issues/2903)) ([`d23bbef`](https://github.com/taskforcesh/bullmq/commit/d23bbef253f732500a4438e3390372dd02117ffc)) -* Add minimum redis version #227 ([`88fff2c`](https://github.com/taskforcesh/bullmq/commit/88fff2c957d6d4b58ef6399ec56123b6da825d88)) -* Telemetry note on close force in worker ([`bf72d3f`](https://github.com/taskforcesh/bullmq/commit/bf72d3fc0ec3bbf73457a7024d8853a78293facd)) -* Add BullMQ Guru (Gurubase.io) badge ([#2888](https://github.com/taskforcesh/bullmq/issues/2888)) ([`b70dbdf`](https://github.com/taskforcesh/bullmq/commit/b70dbdf8e4e8f5b69868f10e98a7c8abd5b2fb23)) -* **patterns:** Fix token type in process-step-jobs ([#2881](https://github.com/taskforcesh/bullmq/issues/2881)) ([`974d096`](https://github.com/taskforcesh/bullmq/commit/974d0960c5990c725a0f13483ab7a8e7e5403949)) ### Performance * **marker:** Add base markers while consuming jobs to get workers busy (#2904) fixes #2842 ([`1759c8b`](https://github.com/taskforcesh/bullmq/commit/1759c8bc111cab9e43d5fccb4d8d2dccc9c39fb4)) @@ -43,19 +13,13 @@ ### Fix * **commands:** Add missing build statement when releasing [python] (#2869) fixes #2868 ([`ff2a47b`](https://github.com/taskforcesh/bullmq/commit/ff2a47b37c6b36ee1a725f91de2c6e4bcf8b011a)) -### Documentation -* **job:** Clarify per-queue scoping of job ids ([#2864](https://github.com/taskforcesh/bullmq/issues/2864)) ([`6c2b80f`](https://github.com/taskforcesh/bullmq/commit/6c2b80f490a0ab4afe502fc8415d6549e0022367)) -* **v4:** Update changelog with v4.18.2 ([#2867](https://github.com/taskforcesh/bullmq/issues/2867)) ([`7ba452e`](https://github.com/taskforcesh/bullmq/commit/7ba452ec3e6d4658e357b2ca810893172c1e0b25)) - ## v2.10.0 (2024-10-24) ### Feature * **job:** Add getChildrenValues method [python] ([#2853](https://github.com/taskforcesh/bullmq/issues/2853)) ([`0f25213`](https://github.com/taskforcesh/bullmq/commit/0f25213b28900a1c35922bd33611701629d83184)) * **queue:** Add option to skip metas update ([`b7dd925`](https://github.com/taskforcesh/bullmq/commit/b7dd925e7f2a4468c98a05f3a3ca1a476482b6c0)) * **queue:** Add queue version support ([#2822](https://github.com/taskforcesh/bullmq/issues/2822)) ([`3a4781b`](https://github.com/taskforcesh/bullmq/commit/3a4781bf7cadf04f6a324871654eed8f01cdadae)) -* **repeat:** Deprecate immediately on job scheduler ([`ed047f7`](https://github.com/taskforcesh/bullmq/commit/ed047f7ab69ebdb445343b6cb325e90b95ee9dc5)) * **job:** Expose priority value ([#2804](https://github.com/taskforcesh/bullmq/issues/2804)) ([`9abec3d`](https://github.com/taskforcesh/bullmq/commit/9abec3dbc4c69f2496c5ff6b5d724f4d1a5ca62f)) * **job:** Add deduplication logic ([#2796](https://github.com/taskforcesh/bullmq/issues/2796)) ([`0a4982d`](https://github.com/taskforcesh/bullmq/commit/0a4982d05d27c066248290ab9f59349b802d02d5)) -* **queue:** Add new upsertJobScheduler, getJobSchedulers and removeJobSchedulers methods ([`dd6b6b2`](https://github.com/taskforcesh/bullmq/commit/dd6b6b2263badd8f29db65d1fa6bcdf5a1e9f6e2)) * **queue:** Add getDebounceJobId method ([#2717](https://github.com/taskforcesh/bullmq/issues/2717)) ([`a68ead9`](https://github.com/taskforcesh/bullmq/commit/a68ead95f32a7d9dabba602895d05c22794b2c02)) ### Fix From 36b99739a428cabdbd292fc38caf9780e286d84b Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Fri, 22 Nov 2024 10:16:18 +0100 Subject: [PATCH 17/52] test(obliterate): add case for empty queue --- tests/test_obliterate.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_obliterate.ts b/tests/test_obliterate.ts index 48967bd5df..2358bcf790 100644 --- a/tests/test_obliterate.ts +++ b/tests/test_obliterate.ts @@ -47,6 +47,19 @@ describe('Obliterate', function () { expect(keys.length).to.be.eql(0); }); + it('should obliterate a queue which is empty but has had jobs in the past', async () => { + await queue.waitUntilReady(); + + const job = await queue.add('test', { foo: 'bar' }); + await job.remove(); + + await queue.obliterate(); + + const client = await queue.client; + const keys = await client.keys(`${prefix}:${queue.name}:*`); + expect(keys.length).to.be.eql(0); + }); + it('should obliterate a queue with jobs in different statuses', async () => { await queue.waitUntilReady(); From b56c3b45a87e52f5faf25406a2b992d1bfed4900 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Thu, 28 Nov 2024 22:44:31 -0600 Subject: [PATCH 18/52] fix(job-scheduler): upsert template when same pattern options are provided (#2943) ref #2940 --- src/commands/addJobScheduler-2.lua | 6 +- tests/test_job_scheduler.ts | 89 +++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 21 deletions(-) diff --git a/src/commands/addJobScheduler-2.lua b/src/commands/addJobScheduler-2.lua index 4583e6b223..8b7a8084eb 100644 --- a/src/commands/addJobScheduler-2.lua +++ b/src/commands/addJobScheduler-2.lua @@ -63,10 +63,12 @@ end local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId) if prevMillis ~= false then local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis - local nextDelayedJobId = repeatKey .. ":" .. jobSchedulerId .. ":" .. nextMillis + local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis + local nextDelayedJobKey = repeatKey .. ":" .. jobSchedulerId .. ":" .. nextMillis if rcall("ZSCORE", delayedKey, delayedJobId) ~= false - and rcall("EXISTS", nextDelayedJobId) ~= 1 then + and (rcall("EXISTS", nextDelayedJobKey) ~= 1 + or delayedJobId == nextDelayedJobId) then removeJob(delayedJobId, true, prefixKey, true --[[remove debounce key]]) rcall("ZREM", delayedKey, delayedJobId) end diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index f0f8ad5e4d..7e6407a660 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1403,35 +1403,86 @@ describe('Job Scheduler', function () { }); }); - it('should keep only one delayed job if adding a new repeatable job with the same id', async function () { - const date = new Date('2017-02-07 9:24:00'); - const key = 'mykey'; + describe('when every option is provided', function () { + it('should keep only one delayed job if adding a new repeatable job with the same id', async function () { + const date = new Date('2017-02-07 9:24:00'); + const key = 'mykey'; - this.clock.setSystemTime(date); + this.clock.setSystemTime(date); - const nextTick = 2 * ONE_SECOND; + const nextTick = 2 * ONE_SECOND; - await queue.upsertJobScheduler(key, { - every: 10_000, - }); + await queue.upsertJobScheduler(key, { + every: 10_000, + }); - this.clock.tick(nextTick); + this.clock.tick(nextTick); - let jobs = await queue.getJobSchedulers(); - expect(jobs).to.have.length(1); + let jobs = await queue.getJobSchedulers(); + expect(jobs).to.have.length(1); - let delayedJobs = await queue.getDelayed(); - expect(delayedJobs).to.have.length(1); + let delayedJobs = await queue.getDelayed(); + expect(delayedJobs).to.have.length(1); - await queue.upsertJobScheduler(key, { - every: 35_160, + await queue.upsertJobScheduler(key, { + every: 35_160, + }); + + jobs = await queue.getJobSchedulers(); + expect(jobs).to.have.length(1); + + delayedJobs = await queue.getDelayed(); + expect(delayedJobs).to.have.length(1); }); + }); + + describe('when pattern option is provided', function () { + it('should keep only one delayed job if adding a new repeatable job with the same id', async function () { + const date = new Date('2017-02-07 9:24:00'); + const key = 'mykey'; + + this.clock.setSystemTime(date); + + const nextTick = 2 * ONE_SECOND; - jobs = await queue.getJobSchedulers(); - expect(jobs).to.have.length(1); + await queue.upsertJobScheduler( + key, + { + pattern: '0 * 1 * *', + }, + { name: 'test1', data: { foo: 'bar' }, opts: { priority: 1 } }, + ); + + this.clock.tick(nextTick); + + let jobs = await queue.getJobSchedulers(); + expect(jobs).to.have.length(1); + + let delayedJobs = await queue.getDelayed(); + expect(delayedJobs).to.have.length(1); + + await queue.upsertJobScheduler( + key, + { + pattern: '0 * 1 * *', + }, + { name: 'test2', data: { foo: 'baz' }, opts: { priority: 2 } }, + ); + + jobs = await queue.getJobSchedulers(); + expect(jobs).to.have.length(1); - delayedJobs = await queue.getDelayed(); - expect(delayedJobs).to.have.length(1); + delayedJobs = await queue.getDelayed(); + expect(delayedJobs).to.have.length(1); + + expect(delayedJobs[0].name).to.be.equal('test2'); + expect(delayedJobs[0].data).to.deep.equal({ + foo: 'baz', + }); + expect(delayedJobs[0].opts).to.deep.include({ + priority: 2, + }); + }); }); // This test is flaky and too complex we need something simpler that tests the same thing From 38820dc8c267c616ada9931198e9e3e9d2f0d536 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Fri, 29 Nov 2024 05:44:55 +0100 Subject: [PATCH 19/52] feat(queue): add getJobSchedulersCount method (#2945) --- src/classes/job-scheduler.ts | 11 +++++------ src/classes/queue.ts | 10 ++++++++++ tests/test_job_scheduler.ts | 3 +++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 63c0f8199c..1cec4caa5c 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -278,12 +278,11 @@ export class JobScheduler extends QueueBase { return Promise.all(jobs); } - async getSchedulersCount( - client: RedisClient, - prefix: string, - queueName: string, - ): Promise { - return client.zcard(`${prefix}:${queueName}:repeat`); + async getSchedulersCount(): Promise { + const jobSchedulersKey = this.keys.repeat; + const client = await this.client; + + return client.zcard(jobSchedulersKey); } private getSchedulerNextJobId({ diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 89373010ae..f642233ac3 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -591,6 +591,16 @@ export class Queue< return (await this.jobScheduler).getJobSchedulers(start, end, asc); } + /** + * + * Get the number of job schedulers. + * + * @returns The number of job schedulers. + */ + async getJobSchedulersCount(): Promise { + return (await this.jobScheduler).getSchedulersCount(); + } + /** * Removes a repeatable job. * diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 7e6407a660..89d81bad90 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -183,6 +183,9 @@ describe('Job Scheduler', function () { const delayed = await queue.getDelayed(); expect(delayed).to.have.length(3); + + const jobSchedulersCount = await queue.getJobSchedulersCount(); + expect(jobSchedulersCount).to.be.eql(3); }); }); From d0c473f90670b95fd98a63995dbb9481d0f6742d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 29 Nov 2024 04:46:01 +0000 Subject: [PATCH 20/52] chore(release): 5.30.0 [skip ci] # [5.30.0](https://github.com/taskforcesh/bullmq/compare/v5.29.1...v5.30.0) (2024-11-29) ### Bug Fixes * **job-scheduler:** upsert template when same pattern options are provided ([#2943](https://github.com/taskforcesh/bullmq/issues/2943)) ref [#2940](https://github.com/taskforcesh/bullmq/issues/2940) ([b56c3b4](https://github.com/taskforcesh/bullmq/commit/b56c3b45a87e52f5faf25406a2b992d1bfed4900)) ### Features * **queue:** add getDelayedCount method [python] ([#2934](https://github.com/taskforcesh/bullmq/issues/2934)) ([71ce75c](https://github.com/taskforcesh/bullmq/commit/71ce75c04b096b5593da0986c41a771add1a81ce)) * **queue:** add getJobSchedulersCount method ([#2945](https://github.com/taskforcesh/bullmq/issues/2945)) ([38820dc](https://github.com/taskforcesh/bullmq/commit/38820dc8c267c616ada9931198e9e3e9d2f0d536)) --- docs/gitbook/changelog.md | 13 +++++++++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 2b5ab07162..67b4cbe17d 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,16 @@ +# [5.30.0](https://github.com/taskforcesh/bullmq/compare/v5.29.1...v5.30.0) (2024-11-29) + + +### Bug Fixes + +* **job-scheduler:** upsert template when same pattern options are provided ([#2943](https://github.com/taskforcesh/bullmq/issues/2943)) ref [#2940](https://github.com/taskforcesh/bullmq/issues/2940) ([b56c3b4](https://github.com/taskforcesh/bullmq/commit/b56c3b45a87e52f5faf25406a2b992d1bfed4900)) + + +### Features + +* **queue:** add getDelayedCount method [python] ([#2934](https://github.com/taskforcesh/bullmq/issues/2934)) ([71ce75c](https://github.com/taskforcesh/bullmq/commit/71ce75c04b096b5593da0986c41a771add1a81ce)) +* **queue:** add getJobSchedulersCount method ([#2945](https://github.com/taskforcesh/bullmq/issues/2945)) ([38820dc](https://github.com/taskforcesh/bullmq/commit/38820dc8c267c616ada9931198e9e3e9d2f0d536)) + ## [5.29.1](https://github.com/taskforcesh/bullmq/compare/v5.29.0...v5.29.1) (2024-11-23) diff --git a/package.json b/package.json index 37e30fa43a..514a602673 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.29.1", + "version": "5.30.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index f7e6775c35..88b94d8347 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.29.1'; +export const version = '5.30.0'; From 85f6f6f181003fafbf75304a268170f0d271ccc3 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Sat, 30 Nov 2024 08:45:49 -0600 Subject: [PATCH 21/52] fix(flow): allow using removeOnFail and failParentOnFailure in parents (#2947) fixes #2229 --- .../moveParentFromWaitingChildrenToFailed.lua | 12 +- src/commands/includes/removeJobsOnFail.lua | 36 +++++ src/commands/moveStalledJobsToWait-9.lua | 29 +--- tests/test_flow.ts | 137 +++++++++++++++++- 4 files changed, 184 insertions(+), 30 deletions(-) create mode 100644 src/commands/includes/removeJobsOnFail.lua diff --git a/src/commands/includes/moveParentFromWaitingChildrenToFailed.lua b/src/commands/includes/moveParentFromWaitingChildrenToFailed.lua index 4ad4723506..a87e4187b9 100644 --- a/src/commands/includes/moveParentFromWaitingChildrenToFailed.lua +++ b/src/commands/includes/moveParentFromWaitingChildrenToFailed.lua @@ -5,16 +5,19 @@ -- Includes --- @include "moveParentToWaitIfNeeded" --- @include "removeDeduplicationKeyIfNeeded" +--- @include "removeJobsOnFail" local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp) if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then - rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId) + local parentQueuePrefix = parentQueueKey .. ":" + local parentFailedKey = parentQueueKey .. ":failed" + rcall("ZADD", parentFailedKey, timestamp, parentId) local failedReason = "child " .. jobIdKey .. " failed" rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp) rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason", failedReason, "prev", "waiting-children") - local jobAttributes = rcall("HMGET", parentKey, "parent", "deid") + local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts") removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2]) @@ -41,5 +44,10 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, end end end + + local parentRawOpts = jobAttributes[3] + local parentOpts = cjson.decode(parentRawOpts) + + removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp) end end diff --git a/src/commands/includes/removeJobsOnFail.lua b/src/commands/includes/removeJobsOnFail.lua new file mode 100644 index 0000000000..a7fa14aad9 --- /dev/null +++ b/src/commands/includes/removeJobsOnFail.lua @@ -0,0 +1,36 @@ +--[[ + Functions to remove jobs when removeOnFail option is provided. +]] + +-- Includes +--- @include "removeJob" +--- @include "removeJobsByMaxAge" +--- @include "removeJobsByMaxCount" + +local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp) + local removeOnFailType = type(opts["removeOnFail"]) + if removeOnFailType == "number" then + removeJobsByMaxCount(opts["removeOnFail"], + failedKey, queueKeyPrefix) + elseif removeOnFailType == "boolean" then + if opts["removeOnFail"] then + removeJob(jobId, false, queueKeyPrefix, + false --[[remove debounce key]]) + rcall("ZREM", failedKey, jobId) + end + elseif removeOnFailType ~= "nil" then + local maxAge = opts["removeOnFail"]["age"] + local maxCount = opts["removeOnFail"]["count"] + + if maxAge ~= nil then + removeJobsByMaxAge(timestamp, maxAge, + failedKey, queueKeyPrefix) + end + + if maxCount ~= nil and maxCount > 0 then + removeJobsByMaxCount(maxCount, failedKey, + queueKeyPrefix) + end + end +end + \ No newline at end of file diff --git a/src/commands/moveStalledJobsToWait-9.lua b/src/commands/moveStalledJobsToWait-9.lua index 2e6161ebee..83a7af1cb2 100644 --- a/src/commands/moveStalledJobsToWait-9.lua +++ b/src/commands/moveStalledJobsToWait-9.lua @@ -30,9 +30,7 @@ local rcall = redis.call --- @include "includes/moveParentFromWaitingChildrenToFailed" --- @include "includes/moveParentToWaitIfNeeded" --- @include "includes/removeDeduplicationKeyIfNeeded" ---- @include "includes/removeJob" ---- @include "includes/removeJobsByMaxAge" ---- @include "includes/removeJobsByMaxCount" +--- @include "includes/removeJobsOnFail" --- @include "includes/trimEvents" local stalledKey = KEYS[1] @@ -86,7 +84,6 @@ if (#stalling > 0) then local rawOpts = jobAttributes[1] local rawParentData = jobAttributes[2] local opts = cjson.decode(rawOpts) - local removeOnFailType = type(opts["removeOnFail"]) rcall("ZADD", failedKey, timestamp, jobId) removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3]) @@ -123,29 +120,7 @@ if (#stalling > 0) then end end - if removeOnFailType == "number" then - removeJobsByMaxCount(opts["removeOnFail"], - failedKey, queueKeyPrefix) - elseif removeOnFailType == "boolean" then - if opts["removeOnFail"] then - removeJob(jobId, false, queueKeyPrefix, - false --[[remove debounce key]]) - rcall("ZREM", failedKey, jobId) - end - elseif removeOnFailType ~= "nil" then - local maxAge = opts["removeOnFail"]["age"] - local maxCount = opts["removeOnFail"]["count"] - - if maxAge ~= nil then - removeJobsByMaxAge(timestamp, maxAge, - failedKey, queueKeyPrefix) - end - - if maxCount ~= nil and maxCount > 0 then - removeJobsByMaxCount(maxCount, failedKey, - queueKeyPrefix) - end - end + removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp) table.insert(failed, jobId) else diff --git a/tests/test_flow.ts b/tests/test_flow.ts index 0369a39988..75554abe01 100644 --- a/tests/test_flow.ts +++ b/tests/test_flow.ts @@ -2239,7 +2239,142 @@ describe('flows', () => { await removeAllQueueData(new IORedis(redisHost), parentQueueName); await removeAllQueueData(new IORedis(redisHost), grandChildrenQueueName); - }).timeout(8000); + }); + + describe('when removeOnFail option is provided', async () => { + it('should remove parent when child is moved to failed', async () => { + const name = 'child-job'; + + const parentQueueName = `parent-queue-${v4()}`; + const grandChildrenQueueName = `grand-children-queue-${v4()}`; + + const parentQueue = new Queue(parentQueueName, { + connection, + prefix, + }); + const grandChildrenQueue = new Queue(grandChildrenQueueName, { + connection, + prefix, + }); + const queueEvents = new QueueEvents(parentQueueName, { + connection, + prefix, + }); + await queueEvents.waitUntilReady(); + + let grandChildrenProcessor, + processedGrandChildren = 0; + const processingChildren = new Promise(resolve => { + grandChildrenProcessor = async () => { + processedGrandChildren++; + + if (processedGrandChildren === 2) { + return resolve(); + } + + await delay(200); + + throw new Error('failed'); + }; + }); + + const grandChildrenWorker = new Worker( + grandChildrenQueueName, + grandChildrenProcessor, + { connection, prefix }, + ); + + await grandChildrenWorker.waitUntilReady(); + + const flow = new FlowProducer({ connection, prefix }); + const tree = await flow.add({ + name: 'parent-job', + queueName: parentQueueName, + data: {}, + children: [ + { + name, + data: { foo: 'bar' }, + queueName, + }, + { + name, + data: { foo: 'qux' }, + queueName, + opts: { failParentOnFailure: true, removeOnFail: true }, + children: [ + { + name, + data: { foo: 'bar' }, + queueName: grandChildrenQueueName, + opts: { failParentOnFailure: true }, + }, + { + name, + data: { foo: 'baz' }, + queueName: grandChildrenQueueName, + }, + ], + }, + ], + }); + + const failed = new Promise(resolve => { + queueEvents.on('failed', async ({ jobId, failedReason, prev }) => { + if (jobId === tree.job.id) { + expect(prev).to.be.equal('waiting-children'); + expect(failedReason).to.be.equal( + `child ${prefix}:${queueName}:${tree.children[1].job.id} failed`, + ); + resolve(); + } + }); + }); + + expect(tree).to.have.property('job'); + expect(tree).to.have.property('children'); + + const { children, job } = tree; + const parentState = await job.getState(); + + expect(parentState).to.be.eql('waiting-children'); + + await processingChildren; + await failed; + + const { children: grandChildren } = children[1]; + const updatedGrandchildJob = await grandChildrenQueue.getJob( + grandChildren[0].job.id, + ); + const grandChildState = await updatedGrandchildJob.getState(); + + expect(grandChildState).to.be.eql('failed'); + expect(updatedGrandchildJob.failedReason).to.be.eql('failed'); + + const updatedParentJob = await queue.getJob(children[1].job.id); + expect(updatedParentJob).to.be.undefined; + + const updatedGrandparentJob = await parentQueue.getJob(job.id); + const updatedGrandparentState = await updatedGrandparentJob.getState(); + + expect(updatedGrandparentState).to.be.eql('failed'); + expect(updatedGrandparentJob.failedReason).to.be.eql( + `child ${prefix}:${queueName}:${children[1].job.id} failed`, + ); + + await parentQueue.close(); + await grandChildrenQueue.close(); + await grandChildrenWorker.close(); + await flow.close(); + await queueEvents.close(); + + await removeAllQueueData(new IORedis(redisHost), parentQueueName); + await removeAllQueueData( + new IORedis(redisHost), + grandChildrenQueueName, + ); + }); + }); describe('when removeDependencyOnFailure is provided', async () => { it('moves parent to wait after children fail', async () => { From 9b1f30c8377ee56b9247e953765892ddf794e4b2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 30 Nov 2024 14:46:59 +0000 Subject: [PATCH 22/52] chore(release): 5.30.1 [skip ci] ## [5.30.1](https://github.com/taskforcesh/bullmq/compare/v5.30.0...v5.30.1) (2024-11-30) ### Bug Fixes * **flow:** allow using removeOnFail and failParentOnFailure in parents ([#2947](https://github.com/taskforcesh/bullmq/issues/2947)) fixes [#2229](https://github.com/taskforcesh/bullmq/issues/2229) ([85f6f6f](https://github.com/taskforcesh/bullmq/commit/85f6f6f181003fafbf75304a268170f0d271ccc3)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 67b4cbe17d..a633151335 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.30.1](https://github.com/taskforcesh/bullmq/compare/v5.30.0...v5.30.1) (2024-11-30) + + +### Bug Fixes + +* **flow:** allow using removeOnFail and failParentOnFailure in parents ([#2947](https://github.com/taskforcesh/bullmq/issues/2947)) fixes [#2229](https://github.com/taskforcesh/bullmq/issues/2229) ([85f6f6f](https://github.com/taskforcesh/bullmq/commit/85f6f6f181003fafbf75304a268170f0d271ccc3)) + # [5.30.0](https://github.com/taskforcesh/bullmq/compare/v5.29.1...v5.30.0) (2024-11-29) diff --git a/package.json b/package.json index 514a602673..07b534b195 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.30.0", + "version": "5.30.1", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 88b94d8347..df1d0818a2 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.30.0'; +export const version = '5.30.1'; From c94e2bd7b1c34486fe6cd8ca0b84738b1a4f5503 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Mon, 2 Dec 2024 17:47:04 +0100 Subject: [PATCH 23/52] test(job-scheduler): add tests covering job retries and stalls (#2941) --- src/classes/worker.ts | 2 +- tests/test_job_scheduler.ts | 249 ++++++++++++++++++++++++++++++++++++ tests/test_queue.ts | 15 +++ 3 files changed, 265 insertions(+), 1 deletion(-) diff --git a/src/classes/worker.ts b/src/classes/worker.ts index acc6f1d533..f9a2ce34ce 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -1042,7 +1042,6 @@ will never work with more accuracy than 1ms. */ } clearTimeout(this.extendLocksTimer); - //clearTimeout(this.stalledCheckTimer); this.stalledCheckStopper?.(); this.closed = true; @@ -1249,6 +1248,7 @@ will never work with more accuracy than 1ms. */ this.emit('stalled', jobId, 'active'); }); + // Todo: check if there any listeners on failed event const jobPromises: Promise>[] = []; for (let i = 0; i < failed.length; i++) { jobPromises.push( diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 89d81bad90..a1093895fd 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1406,6 +1406,255 @@ describe('Job Scheduler', function () { }); }); + describe('when repeatable job fails', function () { + it('should continue repeating', async function () { + const repeatOpts = { + pattern: '0 * 1 * *', + }; + + const worker = new Worker( + queueName, + async () => { + throw new Error('failed'); + }, + { + connection, + prefix, + }, + ); + + const failing = new Promise(resolve => { + worker.on('failed', () => { + resolve(); + }); + }); + + const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.equal(1); + + await repeatableJob!.promote(); + await failing; + + const failedCount = await queue.getFailedCount(); + expect(failedCount).to.be.equal(1); + + const delayedCount2 = await queue.getDelayedCount(); + expect(delayedCount2).to.be.equal(1); + + const jobSchedulers = await queue.getJobSchedulers(); + + const count = await queue.count(); + expect(count).to.be.equal(1); + expect(jobSchedulers).to.have.length(1); + await worker.close(); + }); + + it('should not create a new delayed job if the failed job is retried with retryJobs', async function () { + const repeatOpts = { + every: 579, + }; + + let isFirstRun = true; + + const worker = new Worker( + queueName, + async () => { + this.clock.tick(177); + if (isFirstRun) { + isFirstRun = false; + throw new Error('failed'); + } + }, + { + connection, + prefix, + }, + ); + + const failing = new Promise(resolve => { + worker.on('failed', async () => { + resolve(); + }); + }); + + const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.equal(1); + + await repeatableJob!.promote(); + await failing; + + const failedCount = await queue.getFailedCount(); + expect(failedCount).to.be.equal(1); + + // Retry the failed job + this.clock.tick(1143); + await queue.retryJobs({ state: 'failed' }); + const failedCountAfterRetry = await queue.getFailedCount(); + expect(failedCountAfterRetry).to.be.equal(0); + + const delayedCount2 = await queue.getDelayedCount(); + expect(delayedCount2).to.be.equal(1); + }); + + it('should not create a new delayed job if the failed job is retried with Job.retry()', async function () { + const repeatOpts = { + every: 477, + }; + + let isFirstRun = true; + + const worker = new Worker( + queueName, + async () => { + this.clock.tick(177); + + if (isFirstRun) { + isFirstRun = false; + throw new Error('failed'); + } + }, + { + connection, + prefix, + }, + ); + + const failing = new Promise(resolve => { + worker.on('failed', async () => { + resolve(); + }); + }); + + const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.equal(1); + + await repeatableJob!.promote(); + + this.clock.tick(177); + + await failing; + + this.clock.tick(177); + + const failedJobs = await queue.getFailed(); + expect(failedJobs.length).to.be.equal(1); + + // Retry the failed job + const failedJob = await queue.getJob(failedJobs[0].id); + await failedJob!.retry(); + const failedCountAfterRetry = await queue.getFailedCount(); + expect(failedCountAfterRetry).to.be.equal(0); + + const delayedCount2 = await queue.getDelayedCount(); + expect(delayedCount2).to.be.equal(1); + }); + + it('should not create a new delayed job if the failed job is stalled and moved back to wait', async function () { + // Note, this test is expected to throw an exception like this: + // "Error: Missing lock for job repeat:test:1486455840000. moveToFinished" + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + + const repeatOpts = { + every: 2000, + }; + + const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); + expect(repeatableJob).to.be.ok; + + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.equal(1); + + await repeatableJob!.promote(); + + let resolveCompleting: () => void; + const complettingJob = new Promise(resolve => { + resolveCompleting = resolve; + }); + + let worker: Worker; + const processing = new Promise(resolve => { + worker = new Worker( + queueName, + async () => { + resolve(); + return complettingJob; + }, + { + connection, + prefix, + skipLockRenewal: true, + skipStalledCheck: true, + }, + ); + }); + + await processing; + + // force remove the lock + const client = await queue.client; + const lockKey = `${prefix}:${queueName}:${repeatableJob!.id}:lock`; + await client.del(lockKey); + + const stalledCheckerKey = `${prefix}:${queueName}:stalled-check`; + await client.del(stalledCheckerKey); + + const scripts = (worker!).scripts; + let [failed, stalled] = await scripts.moveStalledJobsToWait(); + + await client.del(stalledCheckerKey); + + [failed, stalled] = await scripts.moveStalledJobsToWait(); + + const waitingJobs = await queue.getWaiting(); + expect(waitingJobs.length).to.be.equal(1); + + await this.clock.tick(500); + + resolveCompleting!(); + await worker!.close(); + + await this.clock.tick(500); + + const delayedCount2 = await queue.getDelayedCount(); + expect(delayedCount2).to.be.equal(1); + + let completedJobs = await queue.getCompleted(); + expect(completedJobs.length).to.be.equal(0); + + const processing2 = new Promise(resolve => { + worker = new Worker( + queueName, + async () => { + resolve(); + }, + { + connection, + prefix, + skipLockRenewal: true, + skipStalledCheck: true, + }, + ); + }); + + await processing2; + + await worker!.close(); + + completedJobs = await queue.getCompleted(); + expect(completedJobs.length).to.be.equal(1); + + const waitingJobs2 = await queue.getWaiting(); + expect(waitingJobs2.length).to.be.equal(0); + + const delayedCount3 = await queue.getDelayedCount(); + expect(delayedCount3).to.be.equal(1); + }); + }); + describe('when every option is provided', function () { it('should keep only one delayed job if adding a new repeatable job with the same id', async function () { const date = new Date('2017-02-07 9:24:00'); diff --git a/tests/test_queue.ts b/tests/test_queue.ts index 80bd41a92d..446e6121a1 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -37,6 +37,21 @@ describe('queues', function () { await connection.quit(); }); + describe('use generics', function () { + it('should be able to use generics', async function () { + const queue = new Queue<{ foo: string; bar: number }>(queueName, { + prefix, + connection, + }); + + const job = await queue.add(queueName, { foo: 'bar', bar: 1 }); + const job2 = await queue.getJob(job.id!); + expect(job2?.data.foo).to.be.eql('bar'); + expect(job2?.data.bar).to.be.eql(1); + await queue.close(); + }); + }); + it('should return the queue version', async () => { const queue = new Queue(queueName, { connection }); const version = await queue.getVersion(); From cb990808db19dd79b5048ee99308fa7d1eaa2e9f Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Mon, 2 Dec 2024 14:32:30 -0600 Subject: [PATCH 24/52] feat(queue): enhance getJobScheduler method to include template information (#2929) ref #2875 --- src/classes/job-scheduler.ts | 56 ++++++++++++++++++---------- src/classes/job.ts | 45 ++-------------------- src/classes/queue.ts | 3 +- src/classes/scripts.ts | 12 +++++- src/commands/addJobScheduler-2.lua | 26 ++++++++++--- src/interfaces/index.ts | 1 + src/interfaces/job-scheduler-json.ts | 18 +++++++++ src/utils.ts | 52 ++++++++++++++++++++++++++ tests/test_job_scheduler.ts | 16 ++++++++ 9 files changed, 161 insertions(+), 68 deletions(-) create mode 100644 src/interfaces/job-scheduler-json.ts diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 1cec4caa5c..344c96014e 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -2,20 +2,11 @@ import { parseExpression } from 'cron-parser'; import { RedisClient, RepeatBaseOptions, RepeatOptions } from '../interfaces'; import { JobsOptions, RepeatStrategy } from '../types'; import { Job } from './job'; +import { JobSchedulerJson, JobSchedulerTemplateJson } from '../interfaces'; import { QueueBase } from './queue-base'; import { RedisConnection } from './redis-connection'; import { SpanKind, TelemetryAttributes } from '../enums'; - -export interface JobSchedulerJson { - key: string; // key is actually the job scheduler id - name: string; - id?: string | null; - endDate: number | null; - tz: string | null; - pattern: string | null; - every?: string | null; - next?: number; -} +import { optsAsJSON, optsFromJSON } from '../utils'; export class JobScheduler extends QueueBase { private repeatStrategy: RepeatStrategy; @@ -103,6 +94,8 @@ export class JobScheduler extends QueueBase { (multi) as RedisClient, jobSchedulerId, nextMillis, + JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), + optsAsJSON(opts), { name: jobName, endDate: endDate ? new Date(endDate).getTime() : undefined, @@ -241,22 +234,47 @@ export class JobScheduler extends QueueBase { }; } - async getJobScheduler(id: string): Promise { + async getJobScheduler(id: string): Promise> { const client = await this.client; - const jobData = await client.hgetall(this.toKey('repeat:' + id)); + const schedulerAttributes = await client.hgetall( + this.toKey('repeat:' + id), + ); - if (jobData) { + if (schedulerAttributes) { return { key: id, - name: jobData.name, - endDate: parseInt(jobData.endDate) || null, - tz: jobData.tz || null, - pattern: jobData.pattern || null, - every: jobData.every || null, + name: schedulerAttributes.name, + endDate: parseInt(schedulerAttributes.endDate) || null, + tz: schedulerAttributes.tz || null, + pattern: schedulerAttributes.pattern || null, + every: schedulerAttributes.every || null, + ...(schedulerAttributes.data || schedulerAttributes.opts + ? { + template: this.getTemplateFromJSON( + schedulerAttributes.data, + schedulerAttributes.opts, + ), + } + : {}), }; } } + private getTemplateFromJSON( + rawData?: string, + rawOpts?: string, + ): JobSchedulerTemplateJson { + console.log(typeof rawOpts); + const template: JobSchedulerTemplateJson = {}; + if (rawData) { + template.data = JSON.parse(rawData); + } + if (rawOpts) { + template.opts = optsFromJSON(rawOpts); + } + return template; + } + async getJobSchedulers( start = 0, end = -1, diff --git a/src/classes/job.ts b/src/classes/job.ts index b63e5cdf49..21a73cff81 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -30,6 +30,8 @@ import { parseObjectValues, tryCatch, removeUndefinedFields, + optsAsJSON, + optsFromJSON, } from '../utils'; import { Backoffs } from './backoffs'; import { Scripts, raw2NextJobData } from './scripts'; @@ -324,7 +326,7 @@ export class Job< jobId?: string, ): Job { const data = JSON.parse(json.data || '{}'); - const opts = Job.optsFromJSON(json.opts); + const opts = optsFromJSON(json.opts); const job = new this( queue, @@ -388,27 +390,6 @@ export class Job< this.scripts = new Scripts(this.queue); } - private static optsFromJSON(rawOpts?: string): JobsOptions { - const opts = JSON.parse(rawOpts || '{}'); - - const optionEntries = Object.entries(opts) as Array< - [keyof RedisJobOptions, any] - >; - - const options: Partial> = {}; - for (const item of optionEntries) { - const [attributeName, value] = item; - if ((optsDecodeMap as Record)[attributeName]) { - options[(optsDecodeMap as Record)[attributeName]] = - value; - } else { - options[attributeName] = value; - } - } - - return options as JobsOptions; - } - /** * Fetches a Job from the queue given the passed job id. * @@ -469,7 +450,7 @@ export class Job< id: this.id, name: this.name, data: JSON.stringify(typeof this.data === 'undefined' ? {} : this.data), - opts: removeUndefinedFields(this.optsAsJSON(this.opts)), + opts: optsAsJSON(this.opts), parent: this.parent ? { ...this.parent } : undefined, parentKey: this.parentKey, progress: this.progress, @@ -487,24 +468,6 @@ export class Job< }); } - private optsAsJSON(opts: JobsOptions = {}): RedisJobOptions { - const optionEntries = Object.entries(opts) as Array< - [keyof JobsOptions, any] - >; - const options: Partial> = {}; - for (const item of optionEntries) { - const [attributeName, value] = item; - if ((optsEncodeMap as Record)[attributeName]) { - options[(optsEncodeMap as Record)[attributeName]] = - value; - } else { - options[attributeName] = value; - } - } - - return options as RedisJobOptions; - } - /** * Prepares a job to be passed to Sandbox. * @returns diff --git a/src/classes/queue.ts b/src/classes/queue.ts index f642233ac3..3afa5691c0 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -3,6 +3,7 @@ import { BaseJobOptions, BulkJobOptions, IoredisListener, + JobSchedulerJson, QueueOptions, RepeatableJob, RepeatOptions, @@ -571,7 +572,7 @@ export class Queue< * * @param id - identifier of scheduler. */ - async getJobScheduler(id: string): Promise { + async getJobScheduler(id: string): Promise> { return (await this.jobScheduler).getJobScheduler(id); } diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index c9156b291d..0897d905a9 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -311,6 +311,8 @@ export class Scripts { client: RedisClient, jobSchedulerId: string, nextMillis: number, + templateData: string, + templateOpts: RedisJobOptions, opts: RepeatableOptions, ): Promise { const queueKeys = this.queue.keys; @@ -319,7 +321,15 @@ export class Scripts { queueKeys.repeat, queueKeys.delayed, ]; - const args = [nextMillis, pack(opts), jobSchedulerId, queueKeys['']]; + + const args = [ + nextMillis, + pack(opts), + jobSchedulerId, + templateData, + pack(templateOpts), + queueKeys[''], + ]; return this.execCommand(client, 'addJobScheduler', keys.concat(args)); } diff --git a/src/commands/addJobScheduler-2.lua b/src/commands/addJobScheduler-2.lua index 8b7a8084eb..c5a84a641c 100644 --- a/src/commands/addJobScheduler-2.lua +++ b/src/commands/addJobScheduler-2.lua @@ -13,7 +13,9 @@ [4] endDate? [5] every? ARGV[3] jobs scheduler id - ARGV[4] prefix key + ARGV[4] Json stringified template data + ARGV[5] mspacked template opts + ARGV[6] prefix key Output: repeatableKey - OK @@ -24,13 +26,14 @@ local delayedKey = KEYS[2] local nextMillis = ARGV[1] local jobSchedulerId = ARGV[3] -local prefixKey = ARGV[4] +local templateOpts = cmsgpack.unpack(ARGV[5]) +local prefixKey = ARGV[6] -- Includes --- @include "includes/removeJob" -local function storeRepeatableJob(repeatKey, nextMillis, rawOpts) - rcall("ZADD", repeatKey, nextMillis, jobSchedulerId) +local function storeRepeatableJob(schedulerId, repeatKey, nextMillis, rawOpts, templateData, templateOpts) + rcall("ZADD", repeatKey, nextMillis, schedulerId) local opts = cmsgpack.unpack(rawOpts) local optionalValues = {} @@ -54,7 +57,18 @@ local function storeRepeatableJob(repeatKey, nextMillis, rawOpts) table.insert(optionalValues, opts['every']) end - rcall("HMSET", repeatKey .. ":" .. jobSchedulerId, "name", opts['name'], + local jsonTemplateOpts = cjson.encode(templateOpts) + if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then + table.insert(optionalValues, "opts") + table.insert(optionalValues, jsonTemplateOpts) + end + + if templateData and templateData ~= '{}' then + table.insert(optionalValues, "data") + table.insert(optionalValues, templateData) + end + + rcall("HMSET", repeatKey .. ":" .. schedulerId, "name", opts['name'], unpack(optionalValues)) end @@ -74,4 +88,4 @@ if prevMillis ~= false then end end -return storeRepeatableJob(repeatKey, nextMillis, ARGV[2]) +return storeRepeatableJob(jobSchedulerId, repeatKey, nextMillis, ARGV[2], ARGV[4], templateOpts) diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts index bf8a6ac0be..44a4945006 100644 --- a/src/interfaces/index.ts +++ b/src/interfaces/index.ts @@ -7,6 +7,7 @@ export * from './debounce-options'; export * from './flow-job'; export * from './ioredis-events'; export * from './job-json'; +export * from './job-scheduler-json'; export * from './keep-jobs'; export * from './metrics-options'; export * from './metrics'; diff --git a/src/interfaces/job-scheduler-json.ts b/src/interfaces/job-scheduler-json.ts new file mode 100644 index 0000000000..90796f5dd5 --- /dev/null +++ b/src/interfaces/job-scheduler-json.ts @@ -0,0 +1,18 @@ +import { JobsOptions } from '../types'; + +export interface JobSchedulerTemplateJson { + data?: D; + opts?: Omit; +} + +export interface JobSchedulerJson { + key: string; // key is actually the job scheduler id + name: string; + id?: string | null; + endDate: number | null; + tz: string | null; + pattern: string | null; + every?: string | null; + next?: number; + template?: JobSchedulerTemplateJson; +} diff --git a/src/utils.ts b/src/utils.ts index 4efa2e3467..1bf2eac69c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -17,6 +17,7 @@ import { EventEmitter } from 'events'; import * as semver from 'semver'; import { SpanKind, TelemetryAttributes } from './enums'; +import { JobsOptions, RedisJobOptions } from './types'; export const errorObject: { [index: string]: any } = { value: null }; @@ -270,6 +271,57 @@ export const toString = (value: any): string => { export const QUEUE_EVENT_SUFFIX = ':qe'; +const optsDecodeMap = { + de: 'deduplication', + fpof: 'failParentOnFailure', + idof: 'ignoreDependencyOnFailure', + kl: 'keepLogs', + rdof: 'removeDependencyOnFailure', + tm: 'telemetryMetadata', +}; + +const optsEncodeMap = invertObject(optsDecodeMap); +optsEncodeMap.debounce = 'de'; + +export function optsAsJSON(opts: JobsOptions = {}): RedisJobOptions { + const optionEntries = Object.entries(opts) as Array<[keyof JobsOptions, any]>; + const options: Partial> = {}; + for (const item of optionEntries) { + const [attributeName, value] = item; + if (value !== undefined) { + if ((optsEncodeMap as Record)[attributeName]) { + options[(optsEncodeMap as Record)[attributeName]] = + value; + } else { + options[attributeName] = value; + } + } + } + + return options as RedisJobOptions; +} + +export function optsFromJSON(rawOpts?: string): JobsOptions { + const opts = JSON.parse(rawOpts || '{}'); + + const optionEntries = Object.entries(opts) as Array< + [keyof RedisJobOptions, any] + >; + + const options: Partial> = {}; + for (const item of optionEntries) { + const [attributeName, value] = item; + if ((optsDecodeMap as Record)[attributeName]) { + options[(optsDecodeMap as Record)[attributeName]] = + value; + } else { + options[attributeName] = value; + } + } + + return options as JobsOptions; +} + export function removeUndefinedFields>( obj: Record, ) { diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index a1093895fd..f0f3244af8 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -360,6 +360,11 @@ describe('Job Scheduler', function () { tz: null, pattern: '*/2 * * * * *', every: null, + template: { + data: { + foo: 'bar', + }, + }, }); this.clock.tick(nextTick); @@ -687,6 +692,17 @@ describe('Job Scheduler', function () { name: 'rrule', }); + const scheduler = await queue.getJobScheduler('rrule'); + + expect(scheduler).to.deep.equal({ + key: 'rrule', + name: 'rrule', + endDate: null, + tz: null, + pattern: 'RRULE:FREQ=SECONDLY;INTERVAL=2;WKST=MO', + every: null, + }); + this.clock.tick(nextTick); let prev: any; From 9ae31bac5bb5c1f35d59db94d2af3d6eaaec1ad3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 2 Dec 2024 20:33:37 +0000 Subject: [PATCH 25/52] chore(release): 5.31.0 [skip ci] # [5.31.0](https://github.com/taskforcesh/bullmq/compare/v5.30.1...v5.31.0) (2024-12-02) ### Features * **queue:** enhance getJobScheduler method to include template information ([#2929](https://github.com/taskforcesh/bullmq/issues/2929)) ref [#2875](https://github.com/taskforcesh/bullmq/issues/2875) ([cb99080](https://github.com/taskforcesh/bullmq/commit/cb990808db19dd79b5048ee99308fa7d1eaa2e9f)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index a633151335..4d8438de35 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +# [5.31.0](https://github.com/taskforcesh/bullmq/compare/v5.30.1...v5.31.0) (2024-12-02) + + +### Features + +* **queue:** enhance getJobScheduler method to include template information ([#2929](https://github.com/taskforcesh/bullmq/issues/2929)) ref [#2875](https://github.com/taskforcesh/bullmq/issues/2875) ([cb99080](https://github.com/taskforcesh/bullmq/commit/cb990808db19dd79b5048ee99308fa7d1eaa2e9f)) + ## [5.30.1](https://github.com/taskforcesh/bullmq/compare/v5.30.0...v5.30.1) (2024-11-30) diff --git a/package.json b/package.json index 07b534b195..1a4cbb7f4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.30.1", + "version": "5.31.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index df1d0818a2..da5589bbac 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.30.1'; +export const version = '5.31.0'; From 3402bfe0d01e5e5205db74d2106cd19d7df53fcb Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Wed, 4 Dec 2024 00:16:32 -0600 Subject: [PATCH 26/52] fix(scheduler-template): remove console.log when getting template information (#2950) --- src/classes/job-scheduler.ts | 10 +++++++--- src/classes/job.ts | 14 -------------- src/classes/queue-base.ts | 2 +- src/commands/addParentJob-4.lua | 6 +++--- src/commands/includes/removeDeduplicationKey.lua | 1 - src/interfaces/base-job-options.ts | 4 +++- src/interfaces/parent.ts | 1 + src/types/job-json-sandbox.ts | 2 +- tests/test_bulk.ts | 13 +++++-------- tests/test_job_scheduler.ts | 4 ++-- tests/test_repeat.ts | 4 ++-- tests/test_worker.ts | 9 +++++---- 12 files changed, 30 insertions(+), 40 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 344c96014e..1f3493c6e6 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -1,8 +1,13 @@ import { parseExpression } from 'cron-parser'; -import { RedisClient, RepeatBaseOptions, RepeatOptions } from '../interfaces'; +import { + JobSchedulerJson, + JobSchedulerTemplateJson, + RedisClient, + RepeatBaseOptions, + RepeatOptions, +} from '../interfaces'; import { JobsOptions, RepeatStrategy } from '../types'; import { Job } from './job'; -import { JobSchedulerJson, JobSchedulerTemplateJson } from '../interfaces'; import { QueueBase } from './queue-base'; import { RedisConnection } from './redis-connection'; import { SpanKind, TelemetryAttributes } from '../enums'; @@ -264,7 +269,6 @@ export class JobScheduler extends QueueBase { rawData?: string, rawOpts?: string, ): JobSchedulerTemplateJson { - console.log(typeof rawOpts); const template: JobSchedulerTemplateJson = {}; if (rawData) { template.data = JSON.parse(rawData); diff --git a/src/classes/job.ts b/src/classes/job.ts index 21a73cff81..5ac56fea75 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -19,11 +19,9 @@ import { JobState, JobJsonSandbox, MinimalQueue, - RedisJobOptions, } from '../types'; import { errorObject, - invertObject, isEmpty, getParentKey, lengthInUtf8Bytes, @@ -41,18 +39,6 @@ import { SpanKind } from '../enums'; const logger = debuglog('bull'); -const optsDecodeMap = { - de: 'deduplication', - fpof: 'failParentOnFailure', - idof: 'ignoreDependencyOnFailure', - kl: 'keepLogs', - rdof: 'removeDependencyOnFailure', - tm: 'telemetryMetadata', -}; - -const optsEncodeMap = invertObject(optsDecodeMap); -optsEncodeMap.debounce = 'de'; - export const PRIORITY_LIMIT = 2 ** 21; /** diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index 9e2008f958..b355f90014 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'events'; -import { QueueBaseOptions, RedisClient, Span, Tracer } from '../interfaces'; +import { QueueBaseOptions, RedisClient, Span } from '../interfaces'; import { MinimalQueue } from '../types'; import { delay, diff --git a/src/commands/addParentJob-4.lua b/src/commands/addParentJob-4.lua index 0840037e5f..bde421fca8 100644 --- a/src/commands/addParentJob-4.lua +++ b/src/commands/addParentJob-4.lua @@ -3,13 +3,13 @@ - Increases the job counter if needed. - Creates a new job key with the job data. - adds the job to the waiting-children zset - + Input: KEYS[1] 'meta' KEYS[2] 'id' KEYS[3] 'completed' KEYS[4] events stream key - + ARGV[1] msgpacked arguments array [1] key prefix, [2] custom id (will not generate one automatically) @@ -21,7 +21,7 @@ [8] parent? {id, queueKey} [9] repeat job key [10] deduplication key - + ARGV[2] Json stringified job data ARGV[3] msgpacked options diff --git a/src/commands/includes/removeDeduplicationKey.lua b/src/commands/includes/removeDeduplicationKey.lua index b93e9e29f8..a4b2a461d7 100644 --- a/src/commands/includes/removeDeduplicationKey.lua +++ b/src/commands/includes/removeDeduplicationKey.lua @@ -9,4 +9,3 @@ local function removeDeduplicationKey(prefixKey, jobKey) rcall("DEL", deduplicationKey) end end - \ No newline at end of file diff --git a/src/interfaces/base-job-options.ts b/src/interfaces/base-job-options.ts index bb10f1caa3..269309263b 100644 --- a/src/interfaces/base-job-options.ts +++ b/src/interfaces/base-job-options.ts @@ -1,4 +1,6 @@ -import { RepeatOptions, KeepJobs, BackoffOptions } from './'; +import { BackoffOptions } from './backoff-options'; +import { KeepJobs } from './keep-jobs'; +import { RepeatOptions } from './repeat-options'; export interface DefaultJobOptions { /** diff --git a/src/interfaces/parent.ts b/src/interfaces/parent.ts index 8dd1fc0a92..ea00482321 100644 --- a/src/interfaces/parent.ts +++ b/src/interfaces/parent.ts @@ -1,4 +1,5 @@ import { JobsOptions } from '../types'; + /** * Describes the parent for a Job. */ diff --git a/src/types/job-json-sandbox.ts b/src/types/job-json-sandbox.ts index c1a96d96a6..3181fbc3f8 100644 --- a/src/types/job-json-sandbox.ts +++ b/src/types/job-json-sandbox.ts @@ -1,4 +1,4 @@ -import { JobJson, ParentKeys } from '../interfaces'; +import { JobJson } from '../interfaces'; export type JobJsonSandbox = JobJson & { queueName: string; diff --git a/tests/test_bulk.ts b/tests/test_bulk.ts index 1d6b8eb4ec..a3729885bc 100644 --- a/tests/test_bulk.ts +++ b/tests/test_bulk.ts @@ -85,7 +85,7 @@ describe('bulk jobs', () => { data: { idx: 0, foo: 'bar' }, opts: { parent: { - id: parent.id, + id: parent.id!, queue: `${prefix}:${parentQueueName}`, }, }, @@ -95,7 +95,7 @@ describe('bulk jobs', () => { data: { idx: 1, foo: 'baz' }, opts: { parent: { - id: parent.id, + id: parent.id!, queue: `${prefix}:${parentQueueName}`, }, }, @@ -122,22 +122,20 @@ describe('bulk jobs', () => { it('should keep workers busy', async () => { const numJobs = 6; - const queue2 = new Queue(queueName, { connection, markerCount: 2, prefix }); - const queueEvents = new QueueEvents(queueName, { connection, prefix }); await queueEvents.waitUntilReady(); const worker = new Worker( queueName, async () => { - await delay(1000); + await delay(900); }, { connection, prefix }, ); const worker2 = new Worker( queueName, async () => { - await delay(1000); + await delay(900); }, { connection, prefix }, ); @@ -153,10 +151,9 @@ describe('bulk jobs', () => { data: { index }, })); - await queue2.addBulk(jobs); + await queue.addBulk(jobs); await completed; - await queue2.close(); await worker.close(); await worker2.close(); await queueEvents.close(); diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index f0f3244af8..2066679fe0 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1211,7 +1211,7 @@ describe('Job Scheduler', function () { }); it('should repeat 7:th day every month at 9:25', async function () { - this.timeout(15000); + this.timeout(8000); const date = new Date('2017-02-02 7:21:42'); this.clock.setSystemTime(date); @@ -1260,7 +1260,7 @@ describe('Job Scheduler', function () { worker.run(); - await queue.upsertJobScheduler('repeat', { pattern: '* 25 9 7 * *' }); + await queue.upsertJobScheduler('repeat', { pattern: '25 9 7 * *' }); nextTick(); await completing; diff --git a/tests/test_repeat.ts b/tests/test_repeat.ts index ed6ee6e301..44d7125e5b 100644 --- a/tests/test_repeat.ts +++ b/tests/test_repeat.ts @@ -1110,7 +1110,7 @@ describe('repeat', function () { }); it('should repeat 7:th day every month at 9:25', async function () { - this.timeout(15000); + this.timeout(8000); const date = new Date('2017-02-02 7:21:42'); this.clock.setSystemTime(date); @@ -1162,7 +1162,7 @@ describe('repeat', function () { await queue.add( 'repeat', { foo: 'bar' }, - { repeat: { pattern: '* 25 9 7 * *' } }, + { repeat: { pattern: '25 9 7 * *' } }, ); nextTick(); diff --git a/tests/test_worker.ts b/tests/test_worker.ts index ad9e8354a4..0b0fecc633 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -480,7 +480,7 @@ describe('workers', function () { await worker.waitUntilReady(); // Add spy to worker.moveToActive - const spy = sinon.spy(worker, 'moveToActive'); + const spy = sinon.spy(worker as any, 'moveToActive'); const bclientSpy = sinon.spy( await (worker as any).blockingConnection.client, 'bzpopmin', @@ -496,7 +496,8 @@ describe('workers', function () { await queue.addBulk(jobsData); - expect(bclientSpy.callCount).to.be.equal(1); + expect(bclientSpy.callCount).to.be.gte(0); + expect(bclientSpy.callCount).to.be.lte(1); await new Promise((resolve, reject) => { worker.on('completed', (_job: Job, _result: any) => { @@ -535,9 +536,9 @@ describe('workers', function () { ); // Add spy to worker.moveToActive - const spy = sinon.spy(worker, 'moveToActive'); + const spy = sinon.spy(worker as any, 'moveToActive'); const bclientSpy = sinon.spy( - await worker.blockingConnection.client, + await (worker as any).blockingConnection.client, 'bzpopmin', ); await worker.waitUntilReady(); From cf49d37f5119a033f630aa781f2e9757831e5873 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 4 Dec 2024 06:17:34 +0000 Subject: [PATCH 27/52] chore(release): 5.31.1 [skip ci] ## [5.31.1](https://github.com/taskforcesh/bullmq/compare/v5.31.0...v5.31.1) (2024-12-04) ### Bug Fixes * **scheduler-template:** remove console.log when getting template information ([#2950](https://github.com/taskforcesh/bullmq/issues/2950)) ([3402bfe](https://github.com/taskforcesh/bullmq/commit/3402bfe0d01e5e5205db74d2106cd19d7df53fcb)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 4d8438de35..a09edb1399 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.31.1](https://github.com/taskforcesh/bullmq/compare/v5.31.0...v5.31.1) (2024-12-04) + + +### Bug Fixes + +* **scheduler-template:** remove console.log when getting template information ([#2950](https://github.com/taskforcesh/bullmq/issues/2950)) ([3402bfe](https://github.com/taskforcesh/bullmq/commit/3402bfe0d01e5e5205db74d2106cd19d7df53fcb)) + # [5.31.0](https://github.com/taskforcesh/bullmq/compare/v5.30.1...v5.31.0) (2024-12-02) diff --git a/package.json b/package.json index 1a4cbb7f4f..32a369eda7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.31.0", + "version": "5.31.1", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index da5589bbac..94cab2f2c8 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.31.0'; +export const version = '5.31.1'; From 0df85e1b1f042032bd6bea39a528a58bf05eae09 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Thu, 5 Dec 2024 11:08:24 +0000 Subject: [PATCH 28/52] GITBOOK-211: change request with no subject merged in GitBook --- docs/gitbook/SUMMARY.md | 237 ++++++++++++++------------- docs/gitbook/bullmq-pro/telemetry.md | 49 ++++++ 2 files changed, 168 insertions(+), 118 deletions(-) create mode 100644 docs/gitbook/bullmq-pro/telemetry.md diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 32944bac16..73b1e37a81 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -1,135 +1,136 @@ # Table of contents -- [What is BullMQ](README.md) -- [Quick Start]() -- [API Reference](https://api.docs.bullmq.io) -- [Changelogs](changelog.md) - - [v4](changelogs/changelog-v4.md) - - [v3](changelogs/changelog-v3.md) - - [v2](changelogs/changelog-v2.md) - - [v1](changelogs/changelog-v1.md) +* [What is BullMQ](README.md) +* [Quick Start]() +* [API Reference](https://api.docs.bullmq.io) +* [Changelogs](changelog.md) + * [v4](changelogs/changelog-v4.md) + * [v3](changelogs/changelog-v3.md) + * [v2](changelogs/changelog-v2.md) + * [v1](changelogs/changelog-v1.md) ## Guide -- [Introduction](guide/introduction.md) -- [Connections](guide/connections.md) -- [Queues](guide/queues/README.md) - - [Auto-removal of jobs](guide/queues/auto-removal-of-jobs.md) - - [Adding jobs in bulk](guide/queues/adding-bulks.md) - - [Global Concurrency](guide/queues/global-concurrency.md) - - [Removing Jobs](guide/queues/removing-jobs.md) -- [Workers](guide/workers/README.md) - - [Auto-removal of jobs](guide/workers/auto-removal-of-jobs.md) - - [Concurrency](guide/workers/concurrency.md) - - [Graceful shutdown](guide/workers/graceful-shutdown.md) - - [Stalled Jobs](guide/workers/stalled-jobs.md) - - [Sandboxed processors](guide/workers/sandboxed-processors.md) - - [Pausing queues](guide/workers/pausing-queues.md) -- [Jobs](guide/jobs/README.md) - - [FIFO](guide/jobs/fifo.md) - - [LIFO](guide/jobs/lifo.md) - - [Job Ids](guide/jobs/job-ids.md) - - [Job Data](guide/jobs/job-data.md) - - [Deduplication](guide/jobs/deduplication.md) - - [Delayed](guide/jobs/delayed.md) - - [Repeatable](guide/jobs/repeatable.md) - - [Prioritized](guide/jobs/prioritized.md) - - [Removing jobs](guide/jobs/removing-job.md) - - [Stalled](guide/jobs/stalled.md) - - [Getters](guide/jobs/getters.md) -- [Job Schedulers](guide/job-schedulers/README.md) - - [Repeat Strategies](guide/job-schedulers/repeat-strategies.md) - - [Repeat options](guide/job-schedulers/repeat-options.md) - - [Manage Job Schedulers](guide/job-schedulers/manage-job-schedulers.md) -- [Flows](guide/flows/README.md) - - [Adding flows in bulk](guide/flows/adding-bulks.md) - - [Get Flow Tree](guide/flows/get-flow-tree.md) - - [Fail Parent](guide/flows/fail-parent.md) - - [Remove Dependency](guide/flows/remove-dependency.md) - - [Ignore Dependency](guide/flows/ignore-dependency.md) - - [Remove Child Dependency](guide/flows/remove-child-dependency.md) -- [Metrics](guide/metrics/metrics.md) -- [Rate limiting](guide/rate-limiting.md) -- [Parallelism and Concurrency](guide/parallelism-and-concurrency.md) -- [Retrying failing jobs](guide/retrying-failing-jobs.md) -- [Returning job data](guide/returning-job-data.md) -- [Events](guide/events/README.md) - - [Create Custom Events](guide/events/create-custom-events.md) -- [Telemetry](guide/telemetry/README.md) - - [Getting started](guide/telemetry/getting-started.md) - - [Running Jaeger](guide/telemetry/running-jaeger.md) - - [Running a simple example](guide/telemetry/running-a-simple-example.md) -- [QueueScheduler](guide/queuescheduler.md) -- [Redis™ Compatibility](guide/redis-tm-compatibility/README.md) - - [Dragonfly](guide/redis-tm-compatibility/dragonfly.md) -- [Redis™ hosting](guide/redis-tm-hosting/README.md) - - [AWS MemoryDB](guide/redis-tm-hosting/aws-memorydb.md) - - [AWS Elasticache](guide/redis-tm-hosting/aws-elasticache.md) -- [Architecture](guide/architecture.md) -- [NestJs](guide/nestjs/README.md) - - [Producers](guide/nestjs/producers.md) - - [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) -- [Going to production](guide/going-to-production.md) -- [Migration to newer versions](guide/migration-to-newer-versions.md) -- [Troubleshooting](guide/troubleshooting.md) +* [Introduction](guide/introduction.md) +* [Connections](guide/connections.md) +* [Queues](guide/queues/README.md) + * [Auto-removal of jobs](guide/queues/auto-removal-of-jobs.md) + * [Adding jobs in bulk](guide/queues/adding-bulks.md) + * [Global Concurrency](guide/queues/global-concurrency.md) + * [Removing Jobs](guide/queues/removing-jobs.md) +* [Workers](guide/workers/README.md) + * [Auto-removal of jobs](guide/workers/auto-removal-of-jobs.md) + * [Concurrency](guide/workers/concurrency.md) + * [Graceful shutdown](guide/workers/graceful-shutdown.md) + * [Stalled Jobs](guide/workers/stalled-jobs.md) + * [Sandboxed processors](guide/workers/sandboxed-processors.md) + * [Pausing queues](guide/workers/pausing-queues.md) +* [Jobs](guide/jobs/README.md) + * [FIFO](guide/jobs/fifo.md) + * [LIFO](guide/jobs/lifo.md) + * [Job Ids](guide/jobs/job-ids.md) + * [Job Data](guide/jobs/job-data.md) + * [Deduplication](guide/jobs/deduplication.md) + * [Delayed](guide/jobs/delayed.md) + * [Repeatable](guide/jobs/repeatable.md) + * [Prioritized](guide/jobs/prioritized.md) + * [Removing jobs](guide/jobs/removing-job.md) + * [Stalled](guide/jobs/stalled.md) + * [Getters](guide/jobs/getters.md) +* [Job Schedulers](guide/job-schedulers/README.md) + * [Repeat Strategies](guide/job-schedulers/repeat-strategies.md) + * [Repeat options](guide/job-schedulers/repeat-options.md) + * [Manage Job Schedulers](guide/job-schedulers/manage-job-schedulers.md) +* [Flows](guide/flows/README.md) + * [Adding flows in bulk](guide/flows/adding-bulks.md) + * [Get Flow Tree](guide/flows/get-flow-tree.md) + * [Fail Parent](guide/flows/fail-parent.md) + * [Remove Dependency](guide/flows/remove-dependency.md) + * [Ignore Dependency](guide/flows/ignore-dependency.md) + * [Remove Child Dependency](guide/flows/remove-child-dependency.md) +* [Metrics](guide/metrics/metrics.md) +* [Rate limiting](guide/rate-limiting.md) +* [Parallelism and Concurrency](guide/parallelism-and-concurrency.md) +* [Retrying failing jobs](guide/retrying-failing-jobs.md) +* [Returning job data](guide/returning-job-data.md) +* [Events](guide/events/README.md) + * [Create Custom Events](guide/events/create-custom-events.md) +* [Telemetry](guide/telemetry/README.md) + * [Getting started](guide/telemetry/getting-started.md) + * [Running Jaeger](guide/telemetry/running-jaeger.md) + * [Running a simple example](guide/telemetry/running-a-simple-example.md) +* [QueueScheduler](guide/queuescheduler.md) +* [Redis™ Compatibility](guide/redis-tm-compatibility/README.md) + * [Dragonfly](guide/redis-tm-compatibility/dragonfly.md) +* [Redis™ hosting](guide/redis-tm-hosting/README.md) + * [AWS MemoryDB](guide/redis-tm-hosting/aws-memorydb.md) + * [AWS Elasticache](guide/redis-tm-hosting/aws-elasticache.md) +* [Architecture](guide/architecture.md) +* [NestJs](guide/nestjs/README.md) + * [Producers](guide/nestjs/producers.md) + * [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) +* [Going to production](guide/going-to-production.md) +* [Migration to newer versions](guide/migration-to-newer-versions.md) +* [Troubleshooting](guide/troubleshooting.md) ## Patterns -- [Adding jobs in bulk across different queues](patterns/adding-bulks.md) -- [Manually processing jobs](patterns/manually-fetching-jobs.md) -- [Named Processor](patterns/named-processor.md) -- [Flows](patterns/flows.md) -- [Idempotent jobs](patterns/idempotent-jobs.md) -- [Throttle jobs](patterns/throttle-jobs.md) -- [Process Step Jobs](patterns/process-step-jobs.md) -- [Failing fast when Redis is down](patterns/failing-fast-when-redis-is-down.md) -- [Stop retrying jobs](patterns/stop-retrying-jobs.md) -- [Timeout jobs](patterns/timeout-jobs.md) -- [Redis Cluster](patterns/redis-cluster.md) +* [Adding jobs in bulk across different queues](patterns/adding-bulks.md) +* [Manually processing jobs](patterns/manually-fetching-jobs.md) +* [Named Processor](patterns/named-processor.md) +* [Flows](patterns/flows.md) +* [Idempotent jobs](patterns/idempotent-jobs.md) +* [Throttle jobs](patterns/throttle-jobs.md) +* [Process Step Jobs](patterns/process-step-jobs.md) +* [Failing fast when Redis is down](patterns/failing-fast-when-redis-is-down.md) +* [Stop retrying jobs](patterns/stop-retrying-jobs.md) +* [Timeout jobs](patterns/timeout-jobs.md) +* [Redis Cluster](patterns/redis-cluster.md) ## BullMQ Pro -- [Introduction](bullmq-pro/introduction.md) -- [Install](bullmq-pro/install.md) -- [Observables](bullmq-pro/observables/README.md) - - [Cancelation](bullmq-pro/observables/cancelation.md) -- [Groups](bullmq-pro/groups/README.md) - - [Getters](bullmq-pro/groups/getters.md) - - [Rate limiting](bullmq-pro/groups/rate-limiting.md) - - [Concurrency](bullmq-pro/groups/concurrency.md) - - [Local group concurrency](bullmq-pro/groups/local-group-concurrency.md) - - [Max group size](bullmq-pro/groups/max-group-size.md) - - [Pausing groups](bullmq-pro/groups/pausing-groups.md) - - [Prioritized intra-groups](bullmq-pro/groups/prioritized.md) - - [Sandboxes for groups](bullmq-pro/groups/sandboxes-for-groups.md) -- [Batches](bullmq-pro/batches.md) -- [NestJs](bullmq-pro/nestjs/README.md) - - [Producers](bullmq-pro/nestjs/producers.md) - - [Queue Events Listeners](bullmq-pro/nestjs/queue-events-listeners.md) - - [API Reference](https://nestjs.bullmq.pro/) - - [Changelog](bullmq-pro/nestjs/changelog.md) -- [API Reference](https://api.bullmq.pro) -- [Changelog](bullmq-pro/changelog.md) -- [Support](bullmq-pro/support.md) +* [Introduction](bullmq-pro/introduction.md) +* [Install](bullmq-pro/install.md) +* [Observables](bullmq-pro/observables/README.md) + * [Cancelation](bullmq-pro/observables/cancelation.md) +* [Groups](bullmq-pro/groups/README.md) + * [Getters](bullmq-pro/groups/getters.md) + * [Rate limiting](bullmq-pro/groups/rate-limiting.md) + * [Concurrency](bullmq-pro/groups/concurrency.md) + * [Local group concurrency](bullmq-pro/groups/local-group-concurrency.md) + * [Max group size](bullmq-pro/groups/max-group-size.md) + * [Pausing groups](bullmq-pro/groups/pausing-groups.md) + * [Prioritized intra-groups](bullmq-pro/groups/prioritized.md) + * [Sandboxes for groups](bullmq-pro/groups/sandboxes-for-groups.md) +* [Telemetry](bullmq-pro/telemetry.md) +* [Batches](bullmq-pro/batches.md) +* [NestJs](bullmq-pro/nestjs/README.md) + * [Producers](bullmq-pro/nestjs/producers.md) + * [Queue Events Listeners](bullmq-pro/nestjs/queue-events-listeners.md) + * [API Reference](https://nestjs.bullmq.pro/) + * [Changelog](bullmq-pro/nestjs/changelog.md) +* [API Reference](https://api.bullmq.pro) +* [Changelog](bullmq-pro/changelog.md) +* [Support](bullmq-pro/support.md) ## Bull -- [Introduction](bull/introduction.md) -- [Install](bull/install.md) -- [Quick Guide](bull/quick-guide.md) -- [Important Notes](bull/important-notes.md) -- [Reference](https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md) -- [Patterns](bull/patterns/README.md) - - [Persistent connections](bull/patterns/persistent-connections.md) - - [Message queue](bull/patterns/message-queue.md) - - [Returning Job Completions](bull/patterns/returning-job-completions.md) - - [Reusing Redis Connections](bull/patterns/reusing-redis-connections.md) - - [Redis cluster](bull/patterns/redis-cluster.md) - - [Custom backoff strategy](bull/patterns/custom-backoff-strategy.md) - - [Debugging](bull/patterns/debugging.md) - - [Manually fetching jobs](bull/patterns/manually-fetching-jobs.md) +* [Introduction](bull/introduction.md) +* [Install](bull/install.md) +* [Quick Guide](bull/quick-guide.md) +* [Important Notes](bull/important-notes.md) +* [Reference](https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md) +* [Patterns](bull/patterns/README.md) + * [Persistent connections](bull/patterns/persistent-connections.md) + * [Message queue](bull/patterns/message-queue.md) + * [Returning Job Completions](bull/patterns/returning-job-completions.md) + * [Reusing Redis Connections](bull/patterns/reusing-redis-connections.md) + * [Redis cluster](bull/patterns/redis-cluster.md) + * [Custom backoff strategy](bull/patterns/custom-backoff-strategy.md) + * [Debugging](bull/patterns/debugging.md) + * [Manually fetching jobs](bull/patterns/manually-fetching-jobs.md) ## Python -- [Introduction](python/introduction.md) -- [Changelog](python/changelog.md) +* [Introduction](python/introduction.md) +* [Changelog](python/changelog.md) diff --git a/docs/gitbook/bullmq-pro/telemetry.md b/docs/gitbook/bullmq-pro/telemetry.md new file mode 100644 index 0000000000..3e40973f64 --- /dev/null +++ b/docs/gitbook/bullmq-pro/telemetry.md @@ -0,0 +1,49 @@ +# Telemetry + +In the same fashion we support telemetry in BullMQ open source edition, we also support telemetry for BullMQ Pro. It works basically the same, in fact you can just the same integrations available for BullMQ in the Pro version. So in order to enable it you would do something like this: + +```typescript +import { QueuePro } from "@taskforcesh/bullmq-pro"; +import { BullMQOtel } from "bullmq-otel"; + +// Initialize a Pro queue using BullMQ-Otel +const queue = new QueuePro("myProQueue", { + connection, + telemetry: new BullMQOtel("guide"), +}); + +await queue.add( + "myJob", + { data: "myData" }, + { + attempts: 2, + backoff: 1000, + group: { + id: "myGroupId", + }, + } +); +``` + +For the Worker we will do it in a similar way: + +```typescript +import { WorkerPro } from "@taskforcesh/bullmq-pro"; +import { BullMQOtel } from "bullmq-otel"; + +const worker = new WorkerPro( + "myProQueue", + async (job) => { + console.log("processing job", job.id); + }, + { + name: "myWorker", + connection, + telemetry: new BullMQOtel("guide"), + concurrency: 10, + batch: { size: 10 }, + } +); +``` + +For an introductury guide on how to integrate OpenTelemetry in you BullMQ applications take a look at this tutorial: [https://blog.taskforce.sh/how-to-integrate-bullmqs-telemetry-on-a-newsletters-subscription-application-2/](https://blog.taskforce.sh/how-to-integrate-bullmqs-telemetry-on-a-newsletters-subscription-application-2/) From 544fc7c9e4755e6b62b82216e25c0cb62734ed59 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Thu, 5 Dec 2024 21:04:57 -0600 Subject: [PATCH 29/52] fix(worker): catch connection error when moveToActive is called (#2952) --- package.json | 4 +- src/classes/worker.ts | 4 +- tests/test_job_scheduler.ts | 8 ++-- tests/test_repeat.ts | 4 +- yarn.lock | 87 ++++++++++++++++++++----------------- 5 files changed, 57 insertions(+), 50 deletions(-) diff --git a/package.json b/package.json index 32a369eda7..e411af5dec 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "@types/msgpack": "^0.0.31", "@types/node": "^12.20.25", "@types/semver": "^7.3.9", - "@types/sinon": "^7.5.2", + "@types/sinon": "^10.0.13", "@types/uuid": "^3.4.10", "@typescript-eslint/eslint-plugin": "^4.32.0", "@typescript-eslint/parser": "^5.33.0", @@ -112,7 +112,7 @@ "rimraf": "^3.0.2", "rrule": "^2.6.9", "semantic-release": "^19.0.3", - "sinon": "^15.1.0", + "sinon": "^18.0.1", "test-console": "^2.0.0", "ts-mocha": "^10.0.0", "ts-node": "^10.7.0", diff --git a/src/classes/worker.ts b/src/classes/worker.ts index f9a2ce34ce..e71d1f64fa 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -597,10 +597,10 @@ export class Worker< this.blockUntil = await this.waiting; if (this.blockUntil <= 0 || this.blockUntil - Date.now() < 1) { - return this.moveToActive(client, token, this.opts.name); + return await this.moveToActive(client, token, this.opts.name); } } catch (err) { - // Swallow error if locally paused or closing since we did force a disconnection + // Swallow error if locally not paused or not closing since we did not force a disconnection if ( !(this.paused || this.closing) && isNotConnectionError(err) diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 2066679fe0..ee329207aa 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -40,7 +40,7 @@ describe('Job Scheduler', function () { }); beforeEach(async function () { - this.clock = sinon.useFakeTimers(); + this.clock = sinon.useFakeTimers({ shouldClearNativeTimers: true }); queueName = `test-${v4()}`; queue = new Queue(queueName, { connection, prefix }); repeat = new Repeat(queueName, { connection, prefix }); @@ -516,7 +516,7 @@ describe('Job Scheduler', function () { const delay = 5 * ONE_SECOND + 500; const worker = new Worker( - queueName, + queueName2, async () => { this.clock.tick(nextTick); }, @@ -524,7 +524,7 @@ describe('Job Scheduler', function () { ); const delayStub = sinon.stub(worker, 'delay').callsFake(async () => {}); - await queue.upsertJobScheduler( + await queue2.upsertJobScheduler( 'test', { pattern: '*/2 * * * * *', @@ -1211,7 +1211,7 @@ describe('Job Scheduler', function () { }); it('should repeat 7:th day every month at 9:25', async function () { - this.timeout(8000); + this.timeout(12000); const date = new Date('2017-02-02 7:21:42'); this.clock.setSystemTime(date); diff --git a/tests/test_repeat.ts b/tests/test_repeat.ts index 44d7125e5b..4685c08803 100644 --- a/tests/test_repeat.ts +++ b/tests/test_repeat.ts @@ -45,7 +45,7 @@ describe('repeat', function () { }); beforeEach(async function () { - this.clock = sinon.useFakeTimers(); + this.clock = sinon.useFakeTimers({ shouldClearNativeTimers: true }); queueName = `test-${v4()}`; queue = new Queue(queueName, { connection, prefix }); repeat = new Repeat(queueName, { connection, prefix }); @@ -1110,7 +1110,7 @@ describe('repeat', function () { }); it('should repeat 7:th day every month at 9:25', async function () { - this.timeout(8000); + this.timeout(12000); const date = new Date('2017-02-02 7:21:42'); this.clock.setSystemTime(date); diff --git a/yarn.lock b/yarn.lock index f0742e0051..f68b48af72 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1002,27 +1002,27 @@ dependencies: type-detect "4.0.8" -"@sinonjs/commons@^3.0.0": +"@sinonjs/commons@^3.0.0", "@sinonjs/commons@^3.0.1": version "3.0.1" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" integrity sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ== dependencies: type-detect "4.0.8" -"@sinonjs/fake-timers@^10.3.0": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz#55fdff1ecab9f354019129daf4df0dd4d923ea66" - integrity sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA== - dependencies: - "@sinonjs/commons" "^3.0.0" - -"@sinonjs/fake-timers@^11.2.2": +"@sinonjs/fake-timers@11.2.2": version "11.2.2" resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz#50063cc3574f4a27bd8453180a04171c85cc9699" integrity sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw== dependencies: "@sinonjs/commons" "^3.0.0" +"@sinonjs/fake-timers@^13.0.1": + version "13.0.5" + resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz#36b9dbc21ad5546486ea9173d6bea063eb1717d5" + integrity sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw== + dependencies: + "@sinonjs/commons" "^3.0.1" + "@sinonjs/samsam@^8.0.0": version "8.0.0" resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-8.0.0.tgz#0d488c91efb3fa1442e26abea81759dfc8b5ac60" @@ -1032,10 +1032,10 @@ lodash.get "^4.4.2" type-detect "^4.0.8" -"@sinonjs/text-encoding@^0.7.2": - version "0.7.2" - resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz#5981a8db18b56ba38ef0efb7d995b12aa7b51918" - integrity sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ== +"@sinonjs/text-encoding@^0.7.3": + version "0.7.3" + resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.3.tgz#282046f03e886e352b2d5f5da5eb755e01457f3f" + integrity sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA== "@tootallnate/once@2": version "2.0.0" @@ -1143,10 +1143,17 @@ resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ== -"@types/sinon@^7.5.2": - version "7.5.2" - resolved "https://registry.yarnpkg.com/@types/sinon/-/sinon-7.5.2.tgz#5e2f1d120f07b9cda07e5dedd4f3bf8888fccdb9" - integrity sha512-T+m89VdXj/eidZyejvmoP9jivXgBDdkOSBVQjU9kF349NEx10QdPNGxHeZUaj1IlJ32/ewdyXJjnJxyxJroYwg== +"@types/sinon@^10.0.13": + version "10.0.20" + resolved "https://registry.yarnpkg.com/@types/sinon/-/sinon-10.0.20.tgz#f1585debf4c0d99f9938f4111e5479fb74865146" + integrity sha512-2APKKruFNCAZgx3daAyACGzWuJ028VVCUDk6o2rw/Z4PXT0ogwdV4KUegW0MwVs0Zu59auPXbbuBJHF12Sx1Eg== + dependencies: + "@types/sinonjs__fake-timers" "*" + +"@types/sinonjs__fake-timers@*": + version "8.1.5" + resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz#5fd3592ff10c1e9695d377020c033116cc2889f2" + integrity sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ== "@types/uuid@^3.4.10": version "3.4.13" @@ -2548,7 +2555,7 @@ diff@^4.0.1: resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== -diff@^5.1.0: +diff@^5.1.0, diff@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/diff/-/diff-5.2.0.tgz#26ded047cd1179b78b9537d5ef725503ce1ae531" integrity sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A== @@ -5296,16 +5303,16 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -nise@^5.1.4: - version "5.1.9" - resolved "https://registry.yarnpkg.com/nise/-/nise-5.1.9.tgz#0cb73b5e4499d738231a473cd89bd8afbb618139" - integrity sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww== +nise@^6.0.0: + version "6.1.1" + resolved "https://registry.yarnpkg.com/nise/-/nise-6.1.1.tgz#78ea93cc49be122e44cb7c8fdf597b0e8778b64a" + integrity sha512-aMSAzLVY7LyeM60gvBS423nBmIPP+Wy7St7hsb+8/fc1HmeoHJfLO8CKse4u3BtOZvQLJghYPI2i/1WZrEj5/g== dependencies: - "@sinonjs/commons" "^3.0.0" - "@sinonjs/fake-timers" "^11.2.2" - "@sinonjs/text-encoding" "^0.7.2" + "@sinonjs/commons" "^3.0.1" + "@sinonjs/fake-timers" "^13.0.1" + "@sinonjs/text-encoding" "^0.7.3" just-extend "^6.2.0" - path-to-regexp "^6.2.1" + path-to-regexp "^8.1.0" node-abort-controller@^3.1.1: version "3.1.1" @@ -5978,10 +5985,10 @@ path-parse@^1.0.6, path-parse@^1.0.7: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== -path-to-regexp@^6.2.1: - version "6.2.1" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.1.tgz#d54934d6798eb9e5ef14e7af7962c945906918e5" - integrity sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw== +path-to-regexp@^8.1.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-8.2.0.tgz#73990cc29e57a3ff2a0d914095156df5db79e8b4" + integrity sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ== path-type@^3.0.0: version "3.0.0" @@ -6816,17 +6823,17 @@ signale@^1.2.1: figures "^2.0.0" pkg-conf "^2.1.0" -sinon@^15.1.0: - version "15.2.0" - resolved "https://registry.yarnpkg.com/sinon/-/sinon-15.2.0.tgz#5e44d4bc5a9b5d993871137fd3560bebfac27565" - integrity sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw== +sinon@^18.0.1: + version "18.0.1" + resolved "https://registry.yarnpkg.com/sinon/-/sinon-18.0.1.tgz#464334cdfea2cddc5eda9a4ea7e2e3f0c7a91c5e" + integrity sha512-a2N2TDY1uGviajJ6r4D1CyRAkzE9NNVlYOV1wX5xQDuAk0ONgzgRl0EjCQuRCPxOwp13ghsMwt9Gdldujs39qw== dependencies: - "@sinonjs/commons" "^3.0.0" - "@sinonjs/fake-timers" "^10.3.0" + "@sinonjs/commons" "^3.0.1" + "@sinonjs/fake-timers" "11.2.2" "@sinonjs/samsam" "^8.0.0" - diff "^5.1.0" - nise "^5.1.4" - supports-color "^7.2.0" + diff "^5.2.0" + nise "^6.0.0" + supports-color "^7" slash@^3.0.0: version "3.0.0" @@ -7182,7 +7189,7 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" -supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0: +supports-color@^7, supports-color@^7.0.0, supports-color@^7.1.0: version "7.2.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== From 26f578e0db52ff0501c2a2f4960abac377171773 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 6 Dec 2024 03:06:10 +0000 Subject: [PATCH 30/52] chore(release): 5.31.2 [skip ci] ## [5.31.2](https://github.com/taskforcesh/bullmq/compare/v5.31.1...v5.31.2) (2024-12-06) ### Bug Fixes * **worker:** catch connection error when moveToActive is called ([#2952](https://github.com/taskforcesh/bullmq/issues/2952)) ([544fc7c](https://github.com/taskforcesh/bullmq/commit/544fc7c9e4755e6b62b82216e25c0cb62734ed59)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index a09edb1399..55f4ab024d 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.31.2](https://github.com/taskforcesh/bullmq/compare/v5.31.1...v5.31.2) (2024-12-06) + + +### Bug Fixes + +* **worker:** catch connection error when moveToActive is called ([#2952](https://github.com/taskforcesh/bullmq/issues/2952)) ([544fc7c](https://github.com/taskforcesh/bullmq/commit/544fc7c9e4755e6b62b82216e25c0cb62734ed59)) + ## [5.31.1](https://github.com/taskforcesh/bullmq/compare/v5.31.0...v5.31.1) (2024-12-04) diff --git a/package.json b/package.json index e411af5dec..c859383cbd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.31.1", + "version": "5.31.2", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 94cab2f2c8..e86f123c5a 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.31.1'; +export const version = '5.31.2'; From fb871dd87323246438521f997a77ac8ae2d22942 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Thu, 5 Dec 2024 22:14:12 -0600 Subject: [PATCH 31/52] docs(job-schedulers): add getJobScheduler documentation (#2953) --- docs/gitbook/guide/job-schedulers/README.md | 4 ++++ .../job-schedulers/manage-job-schedulers.md | 19 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/guide/job-schedulers/README.md b/docs/gitbook/guide/job-schedulers/README.md index f7bb35af6d..2320ae9e59 100644 --- a/docs/gitbook/guide/job-schedulers/README.md +++ b/docs/gitbook/guide/job-schedulers/README.md @@ -51,3 +51,7 @@ All jobs produced by this scheduler will use the given settings. Note that in th {% hint style="info" %} Since jobs produced by the Job Scheduler will get a special job ID in order to guarantee that jobs will never be created more often than the given repeat settings, you cannot choose a custom job id. However you can use the job's name if you need to discriminate these jobs from other jobs. {% endhint %} + +## Read more: + +- 💡 [Upsert Job Scheduler API Reference](https://api.docs.bullmq.io/classes/v5.Queue.html#upsertJobScheduler) diff --git a/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md b/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md index 4531a20173..164bab8191 100644 --- a/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md +++ b/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md @@ -4,7 +4,7 @@ In BullMQ, managing the lifecycle and inventory of job schedulers is crucial for #### Remove job scheduler -The removeJobScheduler method is designed to delete a specific job scheduler from the queue. This is particularly useful when a scheduled task is no longer needed or if you wish to clean up inactive or obsolete schedulers to optimize resource usage. +The **removeJobScheduler** method is designed to delete a specific job scheduler from the queue. This is particularly useful when a scheduled task is no longer needed or if you wish to clean up inactive or obsolete schedulers to optimize resource usage. ```typescript // Remove a job scheduler with ID 'scheduler-123' @@ -18,7 +18,7 @@ The method will return true if there was a Job Scheduler to remove with the give #### Get Job Schedulers -The getJobSchedulers method retrieves a list of all configured job schedulers within a specified range. This is invaluable for monitoring and managing multiple job schedulers, especially in systems where jobs are dynamically scheduled and require frequent reviews or adjustments. +The **getJobSchedulers** method retrieves a list of all configured job schedulers within a specified range. This is invaluable for monitoring and managing multiple job schedulers, especially in systems where jobs are dynamically scheduled and require frequent reviews or adjustments. ```typescript // Retrieve the first 10 job schedulers in ascending order of their next execution time @@ -27,3 +27,18 @@ console.log('Current job schedulers:', schedulers); ``` This method can be particularly useful for generating reports or dashboards that provide insights into when jobs are scheduled to run, aiding in system monitoring and troubleshooting. + +#### Get Job Scheduler + +The **getJobScheduler** method retrieves a job scheduler by id. This is invaluable for inspecting dedicated configurations. + +```typescript +const scheduler = await queue.getJobScheduler('test); +console.log('Current job scheduler:', scheduler); +``` + +## Read more: + +- 💡 [Remove Job Scheduler API Reference](https://api.docs.bullmq.io/classes/v5.Queue.html#removeJobScheduler) +- 💡 [Get Job Schedulers API Reference](https://api.docs.bullmq.io/classes/v5.Queue.html#getJobSchedulers) +- 💡 [Get Job Scheduler API Reference](https://api.docs.bullmq.io/classes/v5.Queue.html#getJobScheduler) From f1dfbad4b9c7cc74313004ea47d51bf910943d18 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Fri, 6 Dec 2024 14:07:08 +0100 Subject: [PATCH 32/52] docs: Update README.md --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 7c2a941c41..e4a40c20ad 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,6 @@ You can find tutorials and news in this blog: https://blog.taskforce.sh/ Do you need to work with BullMQ on platforms other than Node.js? If so, check out the [BullMQ Proxy](https://github.com/taskforcesh/bullmq-proxy) -## 🌟 Rediscover Scale Conference 2024 - -Discover the latest in in-memory and real-time data technologies at **Rediscover Scale 2024**. Ideal for engineers, architects, and technical leaders looking to push technological boundaries. Connect with experts and advance your skills at The Foundry SF, San Francisco. - -[Learn more and register here!](https://www.rediscoverscale.com/) - # Official FrontEnd [Taskforce.sh, Inc](https://taskforce.sh) From ba28e37d77676be8e359d828474cd5e351df47af Mon Sep 17 00:00:00 2001 From: fgozdz Date: Sat, 7 Dec 2024 05:49:58 +0100 Subject: [PATCH 33/52] docs(guide): provide connection details in getting started section (#2897) fixes #2838 --- docs/gitbook/README (1).md | 5 ++- docs/gitbook/guide/connections.md | 57 +++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/docs/gitbook/README (1).md b/docs/gitbook/README (1).md index 95f85fffbf..e849e2ce07 100644 --- a/docs/gitbook/README (1).md +++ b/docs/gitbook/README (1).md @@ -45,12 +45,15 @@ Jobs are added to the queue and can be processed at any time, with at least one ```typescript import { Worker } from 'bullmq'; +import IORedis from 'ioredis'; + +const connection = new IORedis({ maxRetriesPerRequest: null }); const worker = new Worker('foo', async job => { // Will print { foo: 'bar'} for the first job // and { qux: 'baz' } for the second. console.log(job.data); -}); +}, { connection }); ``` {% hint style="info" %} diff --git a/docs/gitbook/guide/connections.md b/docs/gitbook/guide/connections.md index 54e1a2bd53..40adadbc77 100644 --- a/docs/gitbook/guide/connections.md +++ b/docs/gitbook/guide/connections.md @@ -7,32 +7,59 @@ Every class will consume at least one Redis connection, but it is also possible Some examples: ```typescript -import { Queue, Worker } from 'bullmq' +import { Queue, Worker } from 'bullmq'; // Create a new connection in every instance -const myQueue = new Queue('myqueue', { connection: { - host: "myredis.taskforce.run", - port: 32856 -}}); - -const myWorker = new Worker('myqueue', async (job)=>{}, { connection: { - host: "myredis.taskforce.run", - port: 32856 -}}); +const myQueue = new Queue('myqueue', { + connection: { + host: 'myredis.taskforce.run', + port: 32856, + }, +}); + +const myWorker = new Worker('myqueue', async job => {}, { + connection: { + host: 'myredis.taskforce.run', + port: 32856, + }, +}); ``` ```typescript -import { Queue, Worker } from 'bullmq'; +import { Queue } from 'bullmq'; import IORedis from 'ioredis'; const connection = new IORedis(); -// Reuse the ioredis instance -const myQueue = new Queue('myqueue', { connection }); -const myWorker = new Worker('myqueue', async (job)=>{}, { connection }); +// Reuse the ioredis instance in 2 different producers +const myFirstQueue = new Queue('myFirstQueue', { connection }); +const mySecondQueue = new Queue('mySecondQueue', { connection }); ``` -Note that in the second example, even though the ioredis instance is being reused, the worker will create a duplicated connection that it needs internally to make blocking connections. Consult the [ioredis](https://github.com/luin/ioredis/blob/master/API.md) documentation to learn how to properly create an instance of `IORedis.` +```typescript +import { Worker } from 'bullmq'; +import IORedis from 'ioredis'; + +const connection = new IORedis({ maxRetriesPerRequest: null }); + +// Reuse the ioredis instance in 2 different consumers +const myFirstWorker = new Worker('myFirstWorker', async job => {}, { + connection, +}); +const mySecondWorker = new Worker('mySecondWorker', async job => {}, { + connection, +}); +``` + +Note that in the third example, even though the ioredis instance is being reused, the worker will create a duplicated connection that it needs internally to make blocking connections. Consult the [ioredis](https://github.com/luin/ioredis/blob/master/API.md) documentation to learn how to properly create an instance of `IORedis`. + +Also note that simple Queue instance used for managing the queue such as adding jobs, pausing, using getters, etc. usually has different requirements from the worker. + +For example, say that you are adding jobs to a queue as the result of a call to an HTTP endpoint - producer service. The caller of this endpoint cannot wait forever if the connection to Redis happens to be down when this call is made. Therefore the `maxRetriesPerRequest` setting should either be left at its default (which currently is 20) or set it to another value, maybe 1 so that the user gets an error quickly and can retry later. + +On the other hand, if you are adding jobs inside a Worker processor, this process is expected to happen in the background - consumer service. In this case you can share the same connection. + +For more details, refer to the [persistent connections](https://docs.bullmq.io/bull/patterns/persistent-connections) page. {% hint style="danger" %} When using ioredis connections, be careful not to use the "keyPrefix" option in [ioredis](https://redis.github.io/ioredis/interfaces/CommonRedisOptions.html#keyPrefix) as this option is not compatible with BullMQ, which provides its own key prefixing mechanism. From 6b5c3de4b0c5b4b1eb51f7f6c4cc006dfe18132a Mon Sep 17 00:00:00 2001 From: Gregor Date: Sat, 7 Dec 2024 12:38:38 +0300 Subject: [PATCH 34/52] docs: add missing closing quote (#2957) --- docs/gitbook/guide/job-schedulers/manage-job-schedulers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md b/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md index 164bab8191..d363b1c635 100644 --- a/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md +++ b/docs/gitbook/guide/job-schedulers/manage-job-schedulers.md @@ -33,7 +33,7 @@ This method can be particularly useful for generating reports or dashboards that The **getJobScheduler** method retrieves a job scheduler by id. This is invaluable for inspecting dedicated configurations. ```typescript -const scheduler = await queue.getJobScheduler('test); +const scheduler = await queue.getJobScheduler('test'); console.log('Current job scheduler:', scheduler); ``` From 5b005cd94ba0f98677bed4a44f8669c81f073f26 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Sun, 8 Dec 2024 11:56:32 -0600 Subject: [PATCH 35/52] feat(queue): enhance getJobSchedulers method to include template information (#2956) ref #2875 --- src/classes/job-scheduler.ts | 55 ++++++++++++++---------------- src/classes/queue.ts | 10 ++++-- src/classes/scripts.ts | 14 ++++++++ src/commands/getJobScheduler-1.lua | 19 +++++++++++ src/commands/updateJobOption-1.lua | 26 -------------- src/interfaces/telemetry.ts | 16 ++++----- tests/test_job_scheduler.ts | 34 ++++++++++++++++-- tests/test_metrics.ts | 2 +- 8 files changed, 106 insertions(+), 70 deletions(-) create mode 100644 src/commands/getJobScheduler-1.lua delete mode 100644 src/commands/updateJobOption-1.lua diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 1f3493c6e6..bd47466ce5 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -11,7 +11,7 @@ import { Job } from './job'; import { QueueBase } from './queue-base'; import { RedisConnection } from './redis-connection'; import { SpanKind, TelemetryAttributes } from '../enums'; -import { optsAsJSON, optsFromJSON } from '../utils'; +import { array2obj, optsAsJSON, optsFromJSON } from '../utils'; export class JobScheduler extends QueueBase { private repeatStrategy: RepeatStrategy; @@ -202,13 +202,21 @@ export class JobScheduler extends QueueBase { return this.scripts.removeJobScheduler(jobSchedulerId); } - private async getSchedulerData( + private async getSchedulerData( client: RedisClient, key: string, next?: number, - ): Promise { + ): Promise> { const jobData = await client.hgetall(this.toKey('repeat:' + key)); + return this.transformSchedulerData(key, jobData, next); + } + + private async transformSchedulerData( + key: string, + jobData: any, + next?: number, + ): Promise> { if (jobData) { return { key, @@ -217,6 +225,11 @@ export class JobScheduler extends QueueBase { tz: jobData.tz || null, pattern: jobData.pattern || null, every: jobData.every || null, + ...(jobData.data || jobData.opts + ? { + template: this.getTemplateFromJSON(jobData.data, jobData.opts), + } + : {}), next, }; } @@ -239,30 +252,14 @@ export class JobScheduler extends QueueBase { }; } - async getJobScheduler(id: string): Promise> { - const client = await this.client; - const schedulerAttributes = await client.hgetall( - this.toKey('repeat:' + id), - ); + async getScheduler(id: string): Promise> { + const [rawJobData, next] = await this.scripts.getJobScheduler(id); - if (schedulerAttributes) { - return { - key: id, - name: schedulerAttributes.name, - endDate: parseInt(schedulerAttributes.endDate) || null, - tz: schedulerAttributes.tz || null, - pattern: schedulerAttributes.pattern || null, - every: schedulerAttributes.every || null, - ...(schedulerAttributes.data || schedulerAttributes.opts - ? { - template: this.getTemplateFromJSON( - schedulerAttributes.data, - schedulerAttributes.opts, - ), - } - : {}), - }; - } + return this.transformSchedulerData( + id, + rawJobData ? array2obj(rawJobData) : null, + next ? parseInt(next) : null, + ); } private getTemplateFromJSON( @@ -279,11 +276,11 @@ export class JobScheduler extends QueueBase { return template; } - async getJobSchedulers( + async getJobSchedulers( start = 0, end = -1, asc = false, - ): Promise { + ): Promise[]> { const client = await this.client; const jobSchedulersKey = this.keys.repeat; @@ -294,7 +291,7 @@ export class JobScheduler extends QueueBase { const jobs = []; for (let i = 0; i < result.length; i += 2) { jobs.push( - this.getSchedulerData(client, result[i], parseInt(result[i + 1])), + this.getSchedulerData(client, result[i], parseInt(result[i + 1])), ); } return Promise.all(jobs); diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 3afa5691c0..1525d13254 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -573,7 +573,7 @@ export class Queue< * @param id - identifier of scheduler. */ async getJobScheduler(id: string): Promise> { - return (await this.jobScheduler).getJobScheduler(id); + return (await this.jobScheduler).getScheduler(id); } /** @@ -588,8 +588,12 @@ export class Queue< start?: number, end?: number, asc?: boolean, - ): Promise { - return (await this.jobScheduler).getJobSchedulers(start, end, asc); + ): Promise[]> { + return (await this.jobScheduler).getJobSchedulers( + start, + end, + asc, + ); } /** diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index 0897d905a9..dc487b42d1 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -1088,6 +1088,20 @@ export class Scripts { ]); } + getJobSchedulerArgs(id: string): string[] { + const keys: string[] = [this.queue.keys.repeat]; + + return keys.concat([id]); + } + + async getJobScheduler(id: string): Promise<[any, string | null]> { + const client = await this.queue.client; + + const args = this.getJobSchedulerArgs(id); + + return this.execCommand(client, 'getJobScheduler', args); + } + retryJobArgs( jobId: string, lifo: boolean, diff --git a/src/commands/getJobScheduler-1.lua b/src/commands/getJobScheduler-1.lua new file mode 100644 index 0000000000..324bdb58eb --- /dev/null +++ b/src/commands/getJobScheduler-1.lua @@ -0,0 +1,19 @@ +--[[ + Get job scheduler record. + + Input: + KEYS[1] 'repeat' key + + ARGV[1] id +]] + +local rcall = redis.call +local jobSchedulerKey = KEYS[1] .. ":" .. ARGV[1] + +local score = rcall("ZSCORE", KEYS[1], ARGV[1]) + +if score then + return {rcall("HGETALL", jobSchedulerKey), score} -- get job data +end + +return {nil, nil} diff --git a/src/commands/updateJobOption-1.lua b/src/commands/updateJobOption-1.lua deleted file mode 100644 index 03949faf29..0000000000 --- a/src/commands/updateJobOption-1.lua +++ /dev/null @@ -1,26 +0,0 @@ ---[[ - Update a job option - - Input: - KEYS[1] Job id key - - ARGV[1] field - ARGV[2] value - - Output: - 0 - OK - -1 - Missing job. -]] -local rcall = redis.call - -if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists - - local opts = rcall("HGET", KEYS[1], "opts") - local jsonOpts = cjson.decode(opts) - jsonOpts[ARGV[1]] = ARGV[2] - - rcall("HSET", KEYS[1], "opts", cjson.encode(jsonOpts)) - return 0 -else - return -1 -end diff --git a/src/interfaces/telemetry.ts b/src/interfaces/telemetry.ts index e55c0990dc..d39794d34a 100644 --- a/src/interfaces/telemetry.ts +++ b/src/interfaces/telemetry.ts @@ -36,8 +36,8 @@ export interface ContextManager { /** * Creates a new context and sets it as active for the fn passed as last argument * - * @param context - * @param fn + * @param context - + * @param fn - */ with any>( context: Context, @@ -54,7 +54,7 @@ export interface ContextManager { * is the mechanism used to propagate the context across a distributed * application. * - * @param context + * @param context - */ getMetadata(context: Context): string; @@ -62,8 +62,8 @@ export interface ContextManager { * Creates a new context from a serialized version effectively * linking the new context to the parent context. * - * @param activeContext - * @param metadata + * @param activeContext - + * @param metadata - */ fromMetadata(activeContext: Context, metadata: string): Context; } @@ -78,9 +78,9 @@ export interface Tracer { * context. If the context is not provided, the current active context should be * used. * - * @param name - * @param options - * @param context + * @param name - + * @param options - + * @param context - */ startSpan(name: string, options?: SpanOptions, context?: Context): Span; } diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index ee329207aa..427cb99cee 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -342,7 +342,7 @@ describe('Job Scheduler', function () { ); const delayStub = sinon.stub(worker, 'delay').callsFake(async () => {}); - const date = new Date('2017-02-07 9:24:00'); + const date = new Date('2017-02-07T15:24:00.000Z'); this.clock.setSystemTime(date); await queue.upsertJobScheduler( @@ -360,6 +360,7 @@ describe('Job Scheduler', function () { tz: null, pattern: '*/2 * * * * *', every: null, + next: 1486481042000, template: { data: { foo: 'bar', @@ -682,7 +683,7 @@ describe('Job Scheduler', function () { ); const delayStub = sinon.stub(worker, 'delay').callsFake(async () => {}); - const date = new Date('2017-02-07 9:24:00'); + const date = new Date('2017-02-07T15:24:00.000Z'); this.clock.setSystemTime(date); const repeat = { @@ -698,6 +699,7 @@ describe('Job Scheduler', function () { key: 'rrule', name: 'rrule', endDate: null, + next: 1486481042000, tz: null, pattern: 'RRULE:FREQ=SECONDLY;INTERVAL=2;WKST=MO', every: null, @@ -1424,8 +1426,11 @@ describe('Job Scheduler', function () { describe('when repeatable job fails', function () { it('should continue repeating', async function () { + const date = new Date('2017-02-07T15:24:00.000Z'); + this.clock.setSystemTime(date); const repeatOpts = { pattern: '0 * 1 * *', + tz: 'Asia/Calcutta', }; const worker = new Worker( @@ -1445,7 +1450,11 @@ describe('Job Scheduler', function () { }); }); - const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); + const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts, { + name: 'a', + data: { foo: 'bar' }, + opts: { priority: 1 }, + }); const delayedCount = await queue.getDelayedCount(); expect(delayedCount).to.be.equal(1); @@ -1463,6 +1472,25 @@ describe('Job Scheduler', function () { const count = await queue.count(); expect(count).to.be.equal(1); expect(jobSchedulers).to.have.length(1); + + expect(jobSchedulers[0]).to.deep.equal({ + key: 'test', + name: 'a', + endDate: null, + tz: 'Asia/Calcutta', + pattern: '0 * 1 * *', + every: null, + next: 1488310200000, + template: { + data: { + foo: 'bar', + }, + opts: { + priority: 1, + }, + }, + }); + await worker.close(); }); diff --git a/tests/test_metrics.ts b/tests/test_metrics.ts index 6433afc8b2..f59d4e44d7 100644 --- a/tests/test_metrics.ts +++ b/tests/test_metrics.ts @@ -28,7 +28,7 @@ describe('metrics', function () { }); beforeEach(function () { - this.clock = sinon.useFakeTimers(); + this.clock = sinon.useFakeTimers({ shouldClearNativeTimers: true }); }); beforeEach(async function () { From dfa3e8b477662a5a1e556946689a2d41a57f2776 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 8 Dec 2024 17:57:35 +0000 Subject: [PATCH 36/52] chore(release): 5.32.0 [skip ci] # [5.32.0](https://github.com/taskforcesh/bullmq/compare/v5.31.2...v5.32.0) (2024-12-08) ### Features * **queue:** enhance getJobSchedulers method to include template information ([#2956](https://github.com/taskforcesh/bullmq/issues/2956)) ref [#2875](https://github.com/taskforcesh/bullmq/issues/2875) ([5b005cd](https://github.com/taskforcesh/bullmq/commit/5b005cd94ba0f98677bed4a44f8669c81f073f26)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 55f4ab024d..82babe892b 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +# [5.32.0](https://github.com/taskforcesh/bullmq/compare/v5.31.2...v5.32.0) (2024-12-08) + + +### Features + +* **queue:** enhance getJobSchedulers method to include template information ([#2956](https://github.com/taskforcesh/bullmq/issues/2956)) ref [#2875](https://github.com/taskforcesh/bullmq/issues/2875) ([5b005cd](https://github.com/taskforcesh/bullmq/commit/5b005cd94ba0f98677bed4a44f8669c81f073f26)) + ## [5.31.2](https://github.com/taskforcesh/bullmq/compare/v5.31.1...v5.31.2) (2024-12-06) diff --git a/package.json b/package.json index c859383cbd..5c5d059dd4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.31.2", + "version": "5.32.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index e86f123c5a..6a8bdf1dcd 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.31.2'; +export const version = '5.32.0'; From c19c914969169c660a3e108126044c5152faf0cd Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Mon, 9 Dec 2024 10:23:26 +0100 Subject: [PATCH 37/52] feat: replace multi by lua scripts in moveToFailed (#2958) --- src/classes/job.ts | 135 +++++++++------------- src/classes/scripts.ts | 71 +++++++++--- src/classes/worker.ts | 30 ++--- src/commands/extendLocks-1.lua | 44 +++++++ src/commands/includes/updateJobFields.lua | 11 ++ src/commands/moveToDelayed-8.lua | 4 + src/commands/moveToFinished-14.lua | 4 + src/commands/saveStacktrace-1.lua | 3 - src/interfaces/minimal-job.ts | 1 + src/utils.ts | 14 +++ 10 files changed, 199 insertions(+), 118 deletions(-) create mode 100644 src/commands/extendLocks-1.lua create mode 100644 src/commands/includes/updateJobFields.lua diff --git a/src/classes/job.ts b/src/classes/job.ts index 5ac56fea75..9e71bb45e5 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -643,83 +643,64 @@ export class Job< token: string, fetchNext = false, ): Promise { - const client = await this.queue.client; - const message = err?.message; - - this.failedReason = message; - - let command: string; - const multi = client.multi(); - - this.saveStacktrace(multi, err); - - // - // Check if an automatic retry should be performed - // - let finishedOn: number; - const [shouldRetry, retryDelay] = await this.shouldRetryJob(err); - if (shouldRetry) { - if (retryDelay) { - const args = this.scripts.moveToDelayedArgs( - this.id, - Date.now(), - token, - retryDelay, - ); - this.scripts.execCommand(multi, 'moveToDelayed', args); - command = 'moveToDelayed'; - } else { - // Retry immediately - this.scripts.execCommand( - multi, - 'retryJob', - this.scripts.retryJobArgs(this.id, this.opts.lifo, token), - ); - command = 'retryJob'; - } - } else { - const args = this.scripts.moveToFailedArgs( - this, - message, - this.opts.removeOnFail, - token, - fetchNext, - ); - - this.scripts.execCommand(multi, 'moveToFinished', args); - finishedOn = args[this.scripts.moveToFinishedKeys.length + 1] as number; - command = 'moveToFinished'; - } + this.failedReason = err?.message; return this.queue.trace>( SpanKind.INTERNAL, - this.getSpanOperation(command), + this.getSpanOperation('moveToFailed'), this.queue.name, async (span, dstPropagationMedatadata) => { - if (dstPropagationMedatadata) { - this.scripts.execCommand(multi, 'updateJobOption', [ - this.toKey(this.id), - 'tm', - dstPropagationMedatadata, - ]); - } - - const results = await multi.exec(); - const anyError = results.find(result => result[0]); - if (anyError) { - throw new Error( - `Error "moveToFailed" with command ${command}: ${anyError}`, + let result; + + this.updateStacktrace(err); + + const fieldsToUpdate = { + failedReason: this.failedReason, + stacktrace: JSON.stringify(this.stacktrace), + tm: dstPropagationMedatadata, + }; + + // + // Check if an automatic retry should be performed + // + let finishedOn: number; + const [shouldRetry, retryDelay] = await this.shouldRetryJob(err); + + if (shouldRetry) { + if (retryDelay) { + // Retry with delay + result = await this.scripts.moveToDelayed( + this.id, + Date.now(), + retryDelay, + token, + { fieldsToUpdate }, + ); + } else { + // Retry immediately + result = await this.scripts.retryJob( + this.id, + this.opts.lifo, + token, + { + fieldsToUpdate, + }, + ); + } + } else { + const args = this.scripts.moveToFailedArgs( + this, + this.failedReason, + this.opts.removeOnFail, + token, + fetchNext, + fieldsToUpdate, ); - } - const result = results[results.length - 1][1] as number; - if (result < 0) { - throw this.scripts.finishedErrors({ - code: result, - jobId: this.id, - command, - state: 'active', - }); + result = await this.scripts.moveToFinished(this.id, args); + finishedOn = args[ + this.scripts.moveToFinishedKeys.length + 1 + ] as number; } if (finishedOn && typeof finishedOn === 'number') { @@ -732,9 +713,7 @@ export class Job< this.attemptsMade += 1; - if (Array.isArray(result)) { - return raw2NextJobData(result); - } + return result; }, ); } @@ -1249,7 +1228,7 @@ export class Job< } } - protected saveStacktrace(multi: ChainableCommander, err: Error): void { + protected updateStacktrace(err: Error) { this.stacktrace = this.stacktrace || []; if (err?.stack) { @@ -1260,14 +1239,6 @@ export class Job< this.stacktrace = this.stacktrace.slice(-this.opts.stackTraceLimit); } } - - const args = this.scripts.saveStacktraceArgs( - this.id, - JSON.stringify(this.stacktrace), - err?.message, - ); - - this.scripts.execCommand(multi, 'saveStacktrace', args); } } diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index dc487b42d1..e4ee4cbeae 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -34,7 +34,12 @@ import { RedisJobOptions, } from '../types'; import { ErrorCode } from '../enums'; -import { array2obj, getParentKey, isRedisVersionLowerThan } from '../utils'; +import { + array2obj, + getParentKey, + isRedisVersionLowerThan, + objectToFlatArray, +} from '../utils'; import { ChainableCommander } from 'ioredis'; import { version as packageVersion } from '../version'; export type JobData = [JobJsonRaw | number, string?]; @@ -457,6 +462,23 @@ export class Scripts { return this.execCommand(client, 'extendLock', args); } + async extendLocks( + jobIds: string[], + tokens: string[], + duration: number, + ): Promise { + const client = await this.queue.client; + + const args = [ + this.queue.keys.stalled, + this.queue.toKey(''), + pack(tokens), + pack(jobIds), + duration, + ]; + return this.execCommand(client, 'extendLocks', args); + } + async updateData( job: MinimalJob, data: T, @@ -547,6 +569,7 @@ export class Scripts { token: string, timestamp: number, fetchNext = true, + fieldsToUpdate?: Record, ): (string | number | boolean | Buffer)[] { const queueKeys = this.queue.keys; const opts: WorkerOptions = this.queue.opts; @@ -584,6 +607,7 @@ export class Scripts { idof: !!job.opts?.ignoreDependencyOnFailure, rdof: !!job.opts?.removeDependencyOnFailure, }), + fieldsToUpdate ? pack(objectToFlatArray(fieldsToUpdate)) : void 0, ]; return keys.concat(args); @@ -787,6 +811,7 @@ export class Scripts { removeOnFailed: boolean | number | KeepJobs, token: string, fetchNext = false, + fieldsToUpdate?: Record, ): (string | number | boolean | Buffer)[] { const timestamp = Date.now(); return this.moveToFinishedArgs( @@ -798,6 +823,7 @@ export class Scripts { token, timestamp, fetchNext, + fieldsToUpdate, ); } @@ -916,9 +942,9 @@ export class Scripts { token: string, delay: number, opts: MoveToDelayedOpts = {}, - ): (string | number)[] { + ): (string | number | Buffer)[] { const queueKeys = this.queue.keys; - const keys: (string | number)[] = [ + const keys: (string | number | Buffer)[] = [ queueKeys.marker, queueKeys.active, queueKeys.prioritized, @@ -936,19 +962,12 @@ export class Scripts { token, delay, opts.skipAttempt ? '1' : '0', + opts.fieldsToUpdate + ? pack(objectToFlatArray(opts.fieldsToUpdate)) + : void 0, ]); } - saveStacktraceArgs( - jobId: string, - stacktrace: string, - failedReason: string, - ): string[] { - const keys: string[] = [this.queue.toKey(jobId)]; - - return keys.concat([stacktrace, failedReason]); - } - moveToWaitingChildrenArgs( jobId: string, token: string, @@ -1106,8 +1125,9 @@ export class Scripts { jobId: string, lifo: boolean, token: string, - ): (string | number)[] { - const keys: (string | number)[] = [ + fieldsToUpdate?: Record, + ): (string | number | Buffer)[] { + const keys: (string | number | Buffer)[] = [ this.queue.keys.active, this.queue.keys.wait, this.queue.keys.paused, @@ -1129,9 +1149,30 @@ export class Scripts { pushCmd, jobId, token, + fieldsToUpdate ? pack(objectToFlatArray(fieldsToUpdate)) : void 0, ]); } + async retryJob( + jobId: string, + lifo: boolean, + token: string, + fieldsToUpdate?: Record, + ): Promise { + const client = await this.queue.client; + + const args = this.retryJobArgs(jobId, lifo, token, fieldsToUpdate); + const result = await this.execCommand(client, 'retryJob', args); + if (result < 0) { + throw this.finishedErrors({ + code: result, + jobId, + command: 'retryJob', + state: 'active', + }); + } + } + protected moveJobsToWaitArgs( state: FinishedStatus | 'delayed', count: number, diff --git a/src/classes/worker.ts b/src/classes/worker.ts index e71d1f64fa..0c2ae6d284 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -1198,26 +1198,20 @@ will never work with more accuracy than 1ms. */ }); try { - const pipeline = (await this.client).pipeline(); - for (const job of jobs) { - await this.scripts.extendLock( - job.id, - job.token, - this.opts.lockDuration, - pipeline, + const erroredJobIds = await this.scripts.extendLocks( + jobs.map(job => job.id), + jobs.map(job => job.token), + this.opts.lockDuration, + ); + + for (const jobId of erroredJobIds) { + // TODO: Send signal to process function that the job has been lost. + + this.emit( + 'error', + new Error(`could not renew lock for job ${jobId}`), ); } - const result = (await pipeline.exec()) as [Error, string][]; - - for (const [err, jobId] of result) { - if (err) { - // TODO: signal process function that the job has been lost. - this.emit( - 'error', - new Error(`could not renew lock for job ${jobId}`), - ); - } - } } catch (err) { this.emit('error', err); } diff --git a/src/commands/extendLocks-1.lua b/src/commands/extendLocks-1.lua new file mode 100644 index 0000000000..2756cfe2c6 --- /dev/null +++ b/src/commands/extendLocks-1.lua @@ -0,0 +1,44 @@ +--[[ + Extend locks for multiple jobs and remove them from the stalled set if successful. + Return the list of job IDs for which the operation failed. + + KEYS[1] = stalledKey + + ARGV[1] = baseKey + ARGV[2] = tokens + ARGV[3] = jobIds + ARGV[4] = lockDuration (ms) + + Output: + An array of failed job IDs. If empty, all succeeded. +]] +local rcall = redis.call + +local stalledKey = KEYS[1] +local baseKey = ARGV[1] +local tokens = cmsgpack.unpack(ARGV[2]) +local jobIds = cmsgpack.unpack(ARGV[3]) +local lockDuration = ARGV[4] + +local jobCount = #jobIds +local failedJobs = {} + +for i = 1, jobCount, 1 do + local lockKey = baseKey .. jobIds[i] .. ':lock' + local jobId = jobIds[i] + local token = tokens[i] + + local currentToken = rcall("GET", lockKey) + if currentToken == token then + local setResult = rcall("SET", lockKey, token, "PX", lockDuration) + if setResult then + rcall("SREM", stalledKey, jobId) + else + table.insert(failedJobs, jobId) + end + else + table.insert(failedJobs, jobId) + end +end + +return failedJobs diff --git a/src/commands/includes/updateJobFields.lua b/src/commands/includes/updateJobFields.lua new file mode 100644 index 0000000000..9d9057f21a --- /dev/null +++ b/src/commands/includes/updateJobFields.lua @@ -0,0 +1,11 @@ +--[[ + Function to update a bunch of fields in a job. +]] +local function updateJobFields(jobKey, msgpackedFields) + if msgpackedFields then + local fieldsToUpdate = cmsgpack.unpack(msgpackedFields) + if fieldsToUpdate then + redis.call("HMSET", jobKey, unpack(fieldsToUpdate)) + end + end +end diff --git a/src/commands/moveToDelayed-8.lua b/src/commands/moveToDelayed-8.lua index e48d3f235d..3237093b1a 100644 --- a/src/commands/moveToDelayed-8.lua +++ b/src/commands/moveToDelayed-8.lua @@ -17,6 +17,7 @@ ARGV[4] queue token ARGV[5] delay value ARGV[6] skip attempt + ARGV[7] optional job fields to update Output: 0 - OK @@ -33,6 +34,7 @@ local rcall = redis.call --- @include "includes/getDelayedScore" --- @include "includes/getOrSetMaxEvents" --- @include "includes/removeLock" +--- @include "includes/updateJobFields" local jobKey = KEYS[5] local metaKey = KEYS[7] @@ -43,6 +45,8 @@ if rcall("EXISTS", jobKey) == 1 then return errorCode end + updateJobFields(jobKey, ARGV[7]) + local delayedKey = KEYS[4] local jobId = ARGV[3] local delay = tonumber(ARGV[5]) diff --git a/src/commands/moveToFinished-14.lua b/src/commands/moveToFinished-14.lua index 616ee5dada..9316aff272 100644 --- a/src/commands/moveToFinished-14.lua +++ b/src/commands/moveToFinished-14.lua @@ -32,6 +32,7 @@ ARGV[6] fetch next? ARGV[7] keys prefix ARGV[8] opts + ARGV[9] job fields to update opts - token - lock token opts - keepJobs @@ -73,6 +74,7 @@ local rcall = redis.call --- @include "includes/removeParentDependencyKey" --- @include "includes/trimEvents" --- @include "includes/updateParentDepsIfNeeded" +--- @include "includes/updateJobFields" local jobIdKey = KEYS[12] if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists @@ -85,6 +87,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists return errorCode end + updateJobFields(jobIdKey, ARGV[9]); + local attempts = opts['attempts'] local maxMetricsSize = opts['maxMetricsSize'] local maxCount = opts['keepJobs']['count'] diff --git a/src/commands/saveStacktrace-1.lua b/src/commands/saveStacktrace-1.lua index 216e295a59..52c91b6091 100644 --- a/src/commands/saveStacktrace-1.lua +++ b/src/commands/saveStacktrace-1.lua @@ -1,12 +1,9 @@ --[[ Save stacktrace and failedReason. - Input: KEYS[1] job key - ARGV[1] stacktrace ARGV[2] failedReason - Output: 0 - OK -1 - Missing key diff --git a/src/interfaces/minimal-job.ts b/src/interfaces/minimal-job.ts index 60d4eb494a..63bf2feeb9 100644 --- a/src/interfaces/minimal-job.ts +++ b/src/interfaces/minimal-job.ts @@ -6,6 +6,7 @@ export type BulkJobOptions = Omit; export interface MoveToDelayedOpts { skipAttempt?: boolean; + fieldsToUpdate?: Record; } export interface MoveToWaitingChildrenOpts { diff --git a/src/utils.ts b/src/utils.ts index 1bf2eac69c..71c2d5e1da 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -60,6 +60,20 @@ export function array2obj(arr: string[]): Record { return obj; } +export function objectToFlatArray(obj: Record): string[] { + const arr = []; + for (const key in obj) { + if ( + Object.prototype.hasOwnProperty.call(obj, key) && + obj[key] !== undefined + ) { + arr[arr.length] = key; + arr[arr.length] = obj[key]; + } + } + return arr; +} + export function delay( ms: number, abortController?: AbortController, From ab437a74d11f18a49645c2c7ae59949c33dc05b9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 9 Dec 2024 09:24:30 +0000 Subject: [PATCH 38/52] chore(release): 5.33.0 [skip ci] # [5.33.0](https://github.com/taskforcesh/bullmq/compare/v5.32.0...v5.33.0) (2024-12-09) ### Features * replace multi by lua scripts in moveToFailed ([#2958](https://github.com/taskforcesh/bullmq/issues/2958)) ([c19c914](https://github.com/taskforcesh/bullmq/commit/c19c914969169c660a3e108126044c5152faf0cd)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 82babe892b..90f9518f87 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +# [5.33.0](https://github.com/taskforcesh/bullmq/compare/v5.32.0...v5.33.0) (2024-12-09) + + +### Features + +* replace multi by lua scripts in moveToFailed ([#2958](https://github.com/taskforcesh/bullmq/issues/2958)) ([c19c914](https://github.com/taskforcesh/bullmq/commit/c19c914969169c660a3e108126044c5152faf0cd)) + # [5.32.0](https://github.com/taskforcesh/bullmq/compare/v5.31.2...v5.32.0) (2024-12-08) diff --git a/package.json b/package.json index 5c5d059dd4..6b68c8517c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.32.0", + "version": "5.33.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 6a8bdf1dcd..442c670334 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.32.0'; +export const version = '5.33.0'; From b5fa6a3208a8f2a39777dc30c2db2f498addb907 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Mon, 9 Dec 2024 22:39:04 -0600 Subject: [PATCH 39/52] fix(job-scheduler): omit deduplication and debounce options from template options (#2960) --- src/classes/job-scheduler.ts | 8 ++++++-- src/classes/queue.ts | 9 +++++++-- src/interfaces/job-scheduler-json.ts | 4 ++-- src/types/index.ts | 1 + src/types/job-scheduler-template-options.ts | 6 ++++++ 5 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 src/types/job-scheduler-template-options.ts diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index bd47466ce5..c728f50834 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -6,7 +6,11 @@ import { RepeatBaseOptions, RepeatOptions, } from '../interfaces'; -import { JobsOptions, RepeatStrategy } from '../types'; +import { + JobSchedulerTemplateOptions, + JobsOptions, + RepeatStrategy, +} from '../types'; import { Job } from './job'; import { QueueBase } from './queue-base'; import { RedisConnection } from './redis-connection'; @@ -32,7 +36,7 @@ export class JobScheduler extends QueueBase { repeatOpts: Omit, jobName: N, jobData: T, - opts: Omit, + opts: JobSchedulerTemplateOptions, { override }: { override: boolean }, ): Promise | undefined> { const { every, pattern } = repeatOpts; diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 1525d13254..34cb1a4bd8 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -8,7 +8,12 @@ import { RepeatableJob, RepeatOptions, } from '../interfaces'; -import { FinishedStatus, JobsOptions, MinimalQueue } from '../types'; +import { + FinishedStatus, + JobsOptions, + JobSchedulerTemplateOptions, + MinimalQueue, +} from '../types'; import { Job } from './job'; import { QueueGetters } from './queue-getters'; import { Repeat } from './repeat'; @@ -433,7 +438,7 @@ export class Queue< jobTemplate?: { name?: NameType; data?: DataType; - opts?: Omit; + opts?: JobSchedulerTemplateOptions; }, ) { if (repeatOpts.endDate) { diff --git a/src/interfaces/job-scheduler-json.ts b/src/interfaces/job-scheduler-json.ts index 90796f5dd5..150d08873e 100644 --- a/src/interfaces/job-scheduler-json.ts +++ b/src/interfaces/job-scheduler-json.ts @@ -1,8 +1,8 @@ -import { JobsOptions } from '../types'; +import { JobSchedulerTemplateOptions } from '../types'; export interface JobSchedulerTemplateJson { data?: D; - opts?: Omit; + opts?: JobSchedulerTemplateOptions; } export interface JobSchedulerJson { diff --git a/src/types/index.ts b/src/types/index.ts index 7d30742ac9..c2ec6b49e8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -3,5 +3,6 @@ export * from './finished-status'; export * from './minimal-queue'; export * from './job-json-sandbox'; export * from './job-options'; +export * from './job-scheduler-template-options'; export * from './job-type'; export * from './repeat-strategy'; diff --git a/src/types/job-scheduler-template-options.ts b/src/types/job-scheduler-template-options.ts new file mode 100644 index 0000000000..57bbfd6b31 --- /dev/null +++ b/src/types/job-scheduler-template-options.ts @@ -0,0 +1,6 @@ +import { JobsOptions } from './job-options'; + +export type JobSchedulerTemplateOptions = Omit< + JobsOptions, + 'jobId' | 'repeat' | 'delay' | 'deduplication' | 'debounce' +>; From 0e3c2e5f345a1372eea1542ce2504a2f4ca6e6e0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 10 Dec 2024 04:40:11 +0000 Subject: [PATCH 40/52] chore(release): 5.33.1 [skip ci] ## [5.33.1](https://github.com/taskforcesh/bullmq/compare/v5.33.0...v5.33.1) (2024-12-10) ### Bug Fixes * **job-scheduler:** omit deduplication and debounce options from template options ([#2960](https://github.com/taskforcesh/bullmq/issues/2960)) ([b5fa6a3](https://github.com/taskforcesh/bullmq/commit/b5fa6a3208a8f2a39777dc30c2db2f498addb907)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 90f9518f87..5eee314976 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.33.1](https://github.com/taskforcesh/bullmq/compare/v5.33.0...v5.33.1) (2024-12-10) + + +### Bug Fixes + +* **job-scheduler:** omit deduplication and debounce options from template options ([#2960](https://github.com/taskforcesh/bullmq/issues/2960)) ([b5fa6a3](https://github.com/taskforcesh/bullmq/commit/b5fa6a3208a8f2a39777dc30c2db2f498addb907)) + # [5.33.0](https://github.com/taskforcesh/bullmq/compare/v5.32.0...v5.33.0) (2024-12-09) diff --git a/package.json b/package.json index 6b68c8517c..00e0d0a455 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.33.0", + "version": "5.33.1", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 442c670334..cb3e91677f 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.33.0'; +export const version = '5.33.1'; From 6514c335231cb6e727819cf5e0c56ed3f5132838 Mon Sep 17 00:00:00 2001 From: fgozdz Date: Tue, 10 Dec 2024 15:31:34 +0100 Subject: [PATCH 41/52] feat(telemetry): add option to omit context propagation on jobs (#2946) --- docs/gitbook/README (1).md | 14 +- docs/gitbook/SUMMARY.md | 238 +++++++++++++-------------- docs/gitbook/bullmq-pro/telemetry.md | 32 ++-- src/classes/flow-producer.ts | 22 ++- src/classes/job-scheduler.ts | 24 ++- src/classes/job.ts | 87 +++++++++- src/classes/queue-base.ts | 1 + src/classes/queue.ts | 44 +++-- src/classes/worker.ts | 4 +- src/interfaces/base-job-options.ts | 5 - src/types/job-options.ts | 35 +++- src/utils.ts | 73 ++------ tests/test_telemetry_interface.ts | 147 ++++++++++++++++- 13 files changed, 492 insertions(+), 234 deletions(-) diff --git a/docs/gitbook/README (1).md b/docs/gitbook/README (1).md index e849e2ce07..20de97fe4c 100644 --- a/docs/gitbook/README (1).md +++ b/docs/gitbook/README (1).md @@ -49,11 +49,15 @@ import IORedis from 'ioredis'; const connection = new IORedis({ maxRetriesPerRequest: null }); -const worker = new Worker('foo', async job => { - // Will print { foo: 'bar'} for the first job - // and { qux: 'baz' } for the second. - console.log(job.data); -}, { connection }); +const worker = new Worker( + 'foo', + async job => { + // Will print { foo: 'bar'} for the first job + // and { qux: 'baz' } for the second. + console.log(job.data); + }, + { connection }, +); ``` {% hint style="info" %} diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 73b1e37a81..dea69273f3 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -1,136 +1,136 @@ # Table of contents -* [What is BullMQ](README.md) -* [Quick Start]() -* [API Reference](https://api.docs.bullmq.io) -* [Changelogs](changelog.md) - * [v4](changelogs/changelog-v4.md) - * [v3](changelogs/changelog-v3.md) - * [v2](changelogs/changelog-v2.md) - * [v1](changelogs/changelog-v1.md) +- [What is BullMQ](README.md) +- [Quick Start]() +- [API Reference](https://api.docs.bullmq.io) +- [Changelogs](changelog.md) + - [v4](changelogs/changelog-v4.md) + - [v3](changelogs/changelog-v3.md) + - [v2](changelogs/changelog-v2.md) + - [v1](changelogs/changelog-v1.md) ## Guide -* [Introduction](guide/introduction.md) -* [Connections](guide/connections.md) -* [Queues](guide/queues/README.md) - * [Auto-removal of jobs](guide/queues/auto-removal-of-jobs.md) - * [Adding jobs in bulk](guide/queues/adding-bulks.md) - * [Global Concurrency](guide/queues/global-concurrency.md) - * [Removing Jobs](guide/queues/removing-jobs.md) -* [Workers](guide/workers/README.md) - * [Auto-removal of jobs](guide/workers/auto-removal-of-jobs.md) - * [Concurrency](guide/workers/concurrency.md) - * [Graceful shutdown](guide/workers/graceful-shutdown.md) - * [Stalled Jobs](guide/workers/stalled-jobs.md) - * [Sandboxed processors](guide/workers/sandboxed-processors.md) - * [Pausing queues](guide/workers/pausing-queues.md) -* [Jobs](guide/jobs/README.md) - * [FIFO](guide/jobs/fifo.md) - * [LIFO](guide/jobs/lifo.md) - * [Job Ids](guide/jobs/job-ids.md) - * [Job Data](guide/jobs/job-data.md) - * [Deduplication](guide/jobs/deduplication.md) - * [Delayed](guide/jobs/delayed.md) - * [Repeatable](guide/jobs/repeatable.md) - * [Prioritized](guide/jobs/prioritized.md) - * [Removing jobs](guide/jobs/removing-job.md) - * [Stalled](guide/jobs/stalled.md) - * [Getters](guide/jobs/getters.md) -* [Job Schedulers](guide/job-schedulers/README.md) - * [Repeat Strategies](guide/job-schedulers/repeat-strategies.md) - * [Repeat options](guide/job-schedulers/repeat-options.md) - * [Manage Job Schedulers](guide/job-schedulers/manage-job-schedulers.md) -* [Flows](guide/flows/README.md) - * [Adding flows in bulk](guide/flows/adding-bulks.md) - * [Get Flow Tree](guide/flows/get-flow-tree.md) - * [Fail Parent](guide/flows/fail-parent.md) - * [Remove Dependency](guide/flows/remove-dependency.md) - * [Ignore Dependency](guide/flows/ignore-dependency.md) - * [Remove Child Dependency](guide/flows/remove-child-dependency.md) -* [Metrics](guide/metrics/metrics.md) -* [Rate limiting](guide/rate-limiting.md) -* [Parallelism and Concurrency](guide/parallelism-and-concurrency.md) -* [Retrying failing jobs](guide/retrying-failing-jobs.md) -* [Returning job data](guide/returning-job-data.md) -* [Events](guide/events/README.md) - * [Create Custom Events](guide/events/create-custom-events.md) -* [Telemetry](guide/telemetry/README.md) - * [Getting started](guide/telemetry/getting-started.md) - * [Running Jaeger](guide/telemetry/running-jaeger.md) - * [Running a simple example](guide/telemetry/running-a-simple-example.md) -* [QueueScheduler](guide/queuescheduler.md) -* [Redis™ Compatibility](guide/redis-tm-compatibility/README.md) - * [Dragonfly](guide/redis-tm-compatibility/dragonfly.md) -* [Redis™ hosting](guide/redis-tm-hosting/README.md) - * [AWS MemoryDB](guide/redis-tm-hosting/aws-memorydb.md) - * [AWS Elasticache](guide/redis-tm-hosting/aws-elasticache.md) -* [Architecture](guide/architecture.md) -* [NestJs](guide/nestjs/README.md) - * [Producers](guide/nestjs/producers.md) - * [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) -* [Going to production](guide/going-to-production.md) -* [Migration to newer versions](guide/migration-to-newer-versions.md) -* [Troubleshooting](guide/troubleshooting.md) +- [Introduction](guide/introduction.md) +- [Connections](guide/connections.md) +- [Queues](guide/queues/README.md) + - [Auto-removal of jobs](guide/queues/auto-removal-of-jobs.md) + - [Adding jobs in bulk](guide/queues/adding-bulks.md) + - [Global Concurrency](guide/queues/global-concurrency.md) + - [Removing Jobs](guide/queues/removing-jobs.md) +- [Workers](guide/workers/README.md) + - [Auto-removal of jobs](guide/workers/auto-removal-of-jobs.md) + - [Concurrency](guide/workers/concurrency.md) + - [Graceful shutdown](guide/workers/graceful-shutdown.md) + - [Stalled Jobs](guide/workers/stalled-jobs.md) + - [Sandboxed processors](guide/workers/sandboxed-processors.md) + - [Pausing queues](guide/workers/pausing-queues.md) +- [Jobs](guide/jobs/README.md) + - [FIFO](guide/jobs/fifo.md) + - [LIFO](guide/jobs/lifo.md) + - [Job Ids](guide/jobs/job-ids.md) + - [Job Data](guide/jobs/job-data.md) + - [Deduplication](guide/jobs/deduplication.md) + - [Delayed](guide/jobs/delayed.md) + - [Repeatable](guide/jobs/repeatable.md) + - [Prioritized](guide/jobs/prioritized.md) + - [Removing jobs](guide/jobs/removing-job.md) + - [Stalled](guide/jobs/stalled.md) + - [Getters](guide/jobs/getters.md) +- [Job Schedulers](guide/job-schedulers/README.md) + - [Repeat Strategies](guide/job-schedulers/repeat-strategies.md) + - [Repeat options](guide/job-schedulers/repeat-options.md) + - [Manage Job Schedulers](guide/job-schedulers/manage-job-schedulers.md) +- [Flows](guide/flows/README.md) + - [Adding flows in bulk](guide/flows/adding-bulks.md) + - [Get Flow Tree](guide/flows/get-flow-tree.md) + - [Fail Parent](guide/flows/fail-parent.md) + - [Remove Dependency](guide/flows/remove-dependency.md) + - [Ignore Dependency](guide/flows/ignore-dependency.md) + - [Remove Child Dependency](guide/flows/remove-child-dependency.md) +- [Metrics](guide/metrics/metrics.md) +- [Rate limiting](guide/rate-limiting.md) +- [Parallelism and Concurrency](guide/parallelism-and-concurrency.md) +- [Retrying failing jobs](guide/retrying-failing-jobs.md) +- [Returning job data](guide/returning-job-data.md) +- [Events](guide/events/README.md) + - [Create Custom Events](guide/events/create-custom-events.md) +- [Telemetry](guide/telemetry/README.md) + - [Getting started](guide/telemetry/getting-started.md) + - [Running Jaeger](guide/telemetry/running-jaeger.md) + - [Running a simple example](guide/telemetry/running-a-simple-example.md) +- [QueueScheduler](guide/queuescheduler.md) +- [Redis™ Compatibility](guide/redis-tm-compatibility/README.md) + - [Dragonfly](guide/redis-tm-compatibility/dragonfly.md) +- [Redis™ hosting](guide/redis-tm-hosting/README.md) + - [AWS MemoryDB](guide/redis-tm-hosting/aws-memorydb.md) + - [AWS Elasticache](guide/redis-tm-hosting/aws-elasticache.md) +- [Architecture](guide/architecture.md) +- [NestJs](guide/nestjs/README.md) + - [Producers](guide/nestjs/producers.md) + - [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) +- [Going to production](guide/going-to-production.md) +- [Migration to newer versions](guide/migration-to-newer-versions.md) +- [Troubleshooting](guide/troubleshooting.md) ## Patterns -* [Adding jobs in bulk across different queues](patterns/adding-bulks.md) -* [Manually processing jobs](patterns/manually-fetching-jobs.md) -* [Named Processor](patterns/named-processor.md) -* [Flows](patterns/flows.md) -* [Idempotent jobs](patterns/idempotent-jobs.md) -* [Throttle jobs](patterns/throttle-jobs.md) -* [Process Step Jobs](patterns/process-step-jobs.md) -* [Failing fast when Redis is down](patterns/failing-fast-when-redis-is-down.md) -* [Stop retrying jobs](patterns/stop-retrying-jobs.md) -* [Timeout jobs](patterns/timeout-jobs.md) -* [Redis Cluster](patterns/redis-cluster.md) +- [Adding jobs in bulk across different queues](patterns/adding-bulks.md) +- [Manually processing jobs](patterns/manually-fetching-jobs.md) +- [Named Processor](patterns/named-processor.md) +- [Flows](patterns/flows.md) +- [Idempotent jobs](patterns/idempotent-jobs.md) +- [Throttle jobs](patterns/throttle-jobs.md) +- [Process Step Jobs](patterns/process-step-jobs.md) +- [Failing fast when Redis is down](patterns/failing-fast-when-redis-is-down.md) +- [Stop retrying jobs](patterns/stop-retrying-jobs.md) +- [Timeout jobs](patterns/timeout-jobs.md) +- [Redis Cluster](patterns/redis-cluster.md) ## BullMQ Pro -* [Introduction](bullmq-pro/introduction.md) -* [Install](bullmq-pro/install.md) -* [Observables](bullmq-pro/observables/README.md) - * [Cancelation](bullmq-pro/observables/cancelation.md) -* [Groups](bullmq-pro/groups/README.md) - * [Getters](bullmq-pro/groups/getters.md) - * [Rate limiting](bullmq-pro/groups/rate-limiting.md) - * [Concurrency](bullmq-pro/groups/concurrency.md) - * [Local group concurrency](bullmq-pro/groups/local-group-concurrency.md) - * [Max group size](bullmq-pro/groups/max-group-size.md) - * [Pausing groups](bullmq-pro/groups/pausing-groups.md) - * [Prioritized intra-groups](bullmq-pro/groups/prioritized.md) - * [Sandboxes for groups](bullmq-pro/groups/sandboxes-for-groups.md) -* [Telemetry](bullmq-pro/telemetry.md) -* [Batches](bullmq-pro/batches.md) -* [NestJs](bullmq-pro/nestjs/README.md) - * [Producers](bullmq-pro/nestjs/producers.md) - * [Queue Events Listeners](bullmq-pro/nestjs/queue-events-listeners.md) - * [API Reference](https://nestjs.bullmq.pro/) - * [Changelog](bullmq-pro/nestjs/changelog.md) -* [API Reference](https://api.bullmq.pro) -* [Changelog](bullmq-pro/changelog.md) -* [Support](bullmq-pro/support.md) +- [Introduction](bullmq-pro/introduction.md) +- [Install](bullmq-pro/install.md) +- [Observables](bullmq-pro/observables/README.md) + - [Cancelation](bullmq-pro/observables/cancelation.md) +- [Groups](bullmq-pro/groups/README.md) + - [Getters](bullmq-pro/groups/getters.md) + - [Rate limiting](bullmq-pro/groups/rate-limiting.md) + - [Concurrency](bullmq-pro/groups/concurrency.md) + - [Local group concurrency](bullmq-pro/groups/local-group-concurrency.md) + - [Max group size](bullmq-pro/groups/max-group-size.md) + - [Pausing groups](bullmq-pro/groups/pausing-groups.md) + - [Prioritized intra-groups](bullmq-pro/groups/prioritized.md) + - [Sandboxes for groups](bullmq-pro/groups/sandboxes-for-groups.md) +- [Telemetry](bullmq-pro/telemetry.md) +- [Batches](bullmq-pro/batches.md) +- [NestJs](bullmq-pro/nestjs/README.md) + - [Producers](bullmq-pro/nestjs/producers.md) + - [Queue Events Listeners](bullmq-pro/nestjs/queue-events-listeners.md) + - [API Reference](https://nestjs.bullmq.pro/) + - [Changelog](bullmq-pro/nestjs/changelog.md) +- [API Reference](https://api.bullmq.pro) +- [Changelog](bullmq-pro/changelog.md) +- [Support](bullmq-pro/support.md) ## Bull -* [Introduction](bull/introduction.md) -* [Install](bull/install.md) -* [Quick Guide](bull/quick-guide.md) -* [Important Notes](bull/important-notes.md) -* [Reference](https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md) -* [Patterns](bull/patterns/README.md) - * [Persistent connections](bull/patterns/persistent-connections.md) - * [Message queue](bull/patterns/message-queue.md) - * [Returning Job Completions](bull/patterns/returning-job-completions.md) - * [Reusing Redis Connections](bull/patterns/reusing-redis-connections.md) - * [Redis cluster](bull/patterns/redis-cluster.md) - * [Custom backoff strategy](bull/patterns/custom-backoff-strategy.md) - * [Debugging](bull/patterns/debugging.md) - * [Manually fetching jobs](bull/patterns/manually-fetching-jobs.md) +- [Introduction](bull/introduction.md) +- [Install](bull/install.md) +- [Quick Guide](bull/quick-guide.md) +- [Important Notes](bull/important-notes.md) +- [Reference](https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md) +- [Patterns](bull/patterns/README.md) + - [Persistent connections](bull/patterns/persistent-connections.md) + - [Message queue](bull/patterns/message-queue.md) + - [Returning Job Completions](bull/patterns/returning-job-completions.md) + - [Reusing Redis Connections](bull/patterns/reusing-redis-connections.md) + - [Redis cluster](bull/patterns/redis-cluster.md) + - [Custom backoff strategy](bull/patterns/custom-backoff-strategy.md) + - [Debugging](bull/patterns/debugging.md) + - [Manually fetching jobs](bull/patterns/manually-fetching-jobs.md) ## Python -* [Introduction](python/introduction.md) -* [Changelog](python/changelog.md) +- [Introduction](python/introduction.md) +- [Changelog](python/changelog.md) diff --git a/docs/gitbook/bullmq-pro/telemetry.md b/docs/gitbook/bullmq-pro/telemetry.md index 3e40973f64..15c01adf32 100644 --- a/docs/gitbook/bullmq-pro/telemetry.md +++ b/docs/gitbook/bullmq-pro/telemetry.md @@ -3,46 +3,46 @@ In the same fashion we support telemetry in BullMQ open source edition, we also support telemetry for BullMQ Pro. It works basically the same, in fact you can just the same integrations available for BullMQ in the Pro version. So in order to enable it you would do something like this: ```typescript -import { QueuePro } from "@taskforcesh/bullmq-pro"; -import { BullMQOtel } from "bullmq-otel"; +import { QueuePro } from '@taskforcesh/bullmq-pro'; +import { BullMQOtel } from 'bullmq-otel'; // Initialize a Pro queue using BullMQ-Otel -const queue = new QueuePro("myProQueue", { +const queue = new QueuePro('myProQueue', { connection, - telemetry: new BullMQOtel("guide"), + telemetry: new BullMQOtel('guide'), }); await queue.add( - "myJob", - { data: "myData" }, + 'myJob', + { data: 'myData' }, { attempts: 2, backoff: 1000, group: { - id: "myGroupId", + id: 'myGroupId', }, - } + }, ); ``` For the Worker we will do it in a similar way: ```typescript -import { WorkerPro } from "@taskforcesh/bullmq-pro"; -import { BullMQOtel } from "bullmq-otel"; +import { WorkerPro } from '@taskforcesh/bullmq-pro'; +import { BullMQOtel } from 'bullmq-otel'; const worker = new WorkerPro( - "myProQueue", - async (job) => { - console.log("processing job", job.id); + 'myProQueue', + async job => { + console.log('processing job', job.id); }, { - name: "myWorker", + name: 'myWorker', connection, - telemetry: new BullMQOtel("guide"), + telemetry: new BullMQOtel('guide'), concurrency: 10, batch: { size: 10 }, - } + }, ); ``` diff --git a/src/classes/flow-producer.ts b/src/classes/flow-producer.ts index f732fd4e11..a513275f5b 100644 --- a/src/classes/flow-producer.ts +++ b/src/classes/flow-producer.ts @@ -332,11 +332,27 @@ export class FlowProducer extends EventEmitter { node.name, 'addNode', node.queueName, - async (span, dstPropagationMetadata) => { + async (span, srcPropagationMedatada) => { span?.setAttributes({ [TelemetryAttributes.JobName]: node.name, [TelemetryAttributes.JobId]: jobId, }); + const opts = node.opts; + let telemetry = opts?.telemetry; + + if (srcPropagationMedatada && opts) { + const omitContext = opts.telemetry?.omitContext; + const telemetryMetadata = + opts.telemetry?.metadata || + (!omitContext && srcPropagationMedatada); + + if (telemetryMetadata || omitContext) { + telemetry = { + metadata: telemetryMetadata, + omitContext, + }; + } + } const job = new this.Job( queue, @@ -344,9 +360,9 @@ export class FlowProducer extends EventEmitter { node.data, { ...jobsOpts, - ...node.opts, + ...opts, parent: parent?.parentOpts, - telemetryMetadata: dstPropagationMetadata, + telemetry, }, jobId, ); diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index c728f50834..2e41591e45 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -15,7 +15,7 @@ import { Job } from './job'; import { QueueBase } from './queue-base'; import { RedisConnection } from './redis-connection'; import { SpanKind, TelemetryAttributes } from '../enums'; -import { array2obj, optsAsJSON, optsFromJSON } from '../utils'; +import { array2obj } from '../utils'; export class JobScheduler extends QueueBase { private repeatStrategy: RepeatStrategy; @@ -104,7 +104,7 @@ export class JobScheduler extends QueueBase { jobSchedulerId, nextMillis, JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), - optsAsJSON(opts), + Job.optsAsJSON(opts), { name: jobName, endDate: endDate ? new Date(endDate).getTime() : undefined, @@ -126,6 +126,22 @@ export class JobScheduler extends QueueBase { 'add', `${this.name}.${jobName}`, async (span, srcPropagationMedatada) => { + let telemetry = opts.telemetry; + + if (srcPropagationMedatada) { + const omitContext = opts.telemetry?.omitContext; + const telemetryMetadata = + opts.telemetry?.metadata || + (!omitContext && srcPropagationMedatada); + + if (telemetryMetadata || omitContext) { + telemetry = { + metadata: telemetryMetadata, + omitContext, + }; + } + } + const job = this.createNextJob( (multi) as RedisClient, jobName, @@ -134,7 +150,7 @@ export class JobScheduler extends QueueBase { { ...opts, repeat: filteredRepeatOpts, - telemetryMetadata: srcPropagationMedatada, + telemetry, }, jobData, iterationCount, @@ -275,7 +291,7 @@ export class JobScheduler extends QueueBase { template.data = JSON.parse(rawData); } if (rawOpts) { - template.opts = optsFromJSON(rawOpts); + template.opts = Job.optsFromJSON(rawOpts); } return template; } diff --git a/src/classes/job.ts b/src/classes/job.ts index 9e71bb45e5..780005fc0a 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -19,6 +19,8 @@ import { JobState, JobJsonSandbox, MinimalQueue, + RedisJobOptions, + CompressableJobOptions, } from '../types'; import { errorObject, @@ -28,8 +30,7 @@ import { parseObjectValues, tryCatch, removeUndefinedFields, - optsAsJSON, - optsFromJSON, + invertObject, } from '../utils'; import { Backoffs } from './backoffs'; import { Scripts, raw2NextJobData } from './scripts'; @@ -39,6 +40,20 @@ import { SpanKind } from '../enums'; const logger = debuglog('bull'); +// Simple options decode map. +const optsDecodeMap = { + de: 'deduplication', + fpof: 'failParentOnFailure', + idof: 'ignoreDependencyOnFailure', + kl: 'keepLogs', + rdof: 'removeDependencyOnFailure', +} as const; + +const optsEncodeMap = { + ...invertObject(optsDecodeMap), + /*/ Legacy for backwards compatibility */ debounce: 'de', +} as const; + export const PRIORITY_LIMIT = 2 ** 21; /** @@ -312,7 +327,7 @@ export class Job< jobId?: string, ): Job { const data = JSON.parse(json.data || '{}'); - const opts = optsFromJSON(json.opts); + const opts = Job.optsFromJSON(json.opts); const job = new this( queue, @@ -376,6 +391,33 @@ export class Job< this.scripts = new Scripts(this.queue); } + static optsFromJSON(rawOpts?: string): JobsOptions { + const opts = JSON.parse(rawOpts || '{}'); + + const optionEntries = Object.entries(opts) as Array< + [keyof RedisJobOptions, any] + >; + + const options: Partial> = {}; + for (const item of optionEntries) { + const [attributeName, value] = item; + if ((optsDecodeMap as Record)[attributeName]) { + options[(optsDecodeMap as Record)[attributeName]] = + value; + } else { + if (attributeName === 'tm') { + options.telemetry = { ...options.telemetry, metadata: value }; + } else if (attributeName === 'omc') { + options.telemetry = { ...options.telemetry, omitContext: value }; + } else { + options[attributeName] = value; + } + } + } + + return options as JobsOptions; + } + /** * Fetches a Job from the queue given the passed job id. * @@ -436,7 +478,7 @@ export class Job< id: this.id, name: this.name, data: JSON.stringify(typeof this.data === 'undefined' ? {} : this.data), - opts: optsAsJSON(this.opts), + opts: Job.optsAsJSON(this.opts), parent: this.parent ? { ...this.parent } : undefined, parentKey: this.parentKey, progress: this.progress, @@ -454,6 +496,37 @@ export class Job< }); } + static optsAsJSON(opts: JobsOptions = {}): RedisJobOptions { + const optionEntries = Object.entries(opts) as Array< + [keyof JobsOptions, any] + >; + const options: Record = {}; + + for (const [attributeName, value] of optionEntries) { + if (typeof value === 'undefined') { + continue; + } + if (attributeName in optsEncodeMap) { + const compressableAttribute = attributeName as keyof Omit< + CompressableJobOptions, + 'debounce' | 'telemetry' + >; + + const key = optsEncodeMap[compressableAttribute]; + options[key] = value; + } else { + // Handle complex compressable fields separately + if (attributeName === 'telemetry') { + options.tm = value.metadata; + options.omc = value.omitContext; + } else { + options[attributeName] = value; + } + } + } + return options as RedisJobOptions; + } + /** * Prepares a job to be passed to Sandbox. * @returns @@ -650,6 +723,10 @@ export class Job< this.getSpanOperation('moveToFailed'), this.queue.name, async (span, dstPropagationMedatadata) => { + let tm; + if (!this.opts?.telemetry?.omitContext && dstPropagationMedatadata) { + tm = dstPropagationMedatadata; + } let result; this.updateStacktrace(err); @@ -657,7 +734,7 @@ export class Job< const fieldsToUpdate = { failedReason: this.failedReason, stacktrace: JSON.stringify(this.stacktrace), - tm: dstPropagationMedatadata, + tm, }; // diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index b355f90014..bdd9da7911 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -1,6 +1,7 @@ import { EventEmitter } from 'events'; import { QueueBaseOptions, RedisClient, Span } from '../interfaces'; import { MinimalQueue } from '../types'; + import { delay, DELAY_TIME_5, diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 34cb1a4bd8..092ebd2218 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -313,8 +313,11 @@ export class Queue< 'add', `${this.name}.${name}`, async (span, srcPropagationMedatada) => { - if (srcPropagationMedatada) { - opts = { ...opts, telemetryMetadata: srcPropagationMedatada }; + if (srcPropagationMedatada && !opts?.telemetry?.omitContext) { + const telemetry = { + metadata: srcPropagationMedatada, + }; + opts = { ...opts, telemetry }; } const job = await this.addJob(name, data, opts); @@ -403,16 +406,33 @@ export class Queue< return await this.Job.createBulk( this as MinimalQueue, - jobs.map(job => ({ - name: job.name, - data: job.data, - opts: { - ...this.jobsOpts, - ...job.opts, - jobId: job.opts?.jobId, - tm: span && srcPropagationMedatada, - }, - })), + jobs.map(job => { + let telemetry = job.opts?.telemetry; + if (srcPropagationMedatada) { + const omitContext = job.opts?.telemetry?.omitContext; + const telemetryMetadata = + job.opts?.telemetry?.metadata || + (!omitContext && srcPropagationMedatada); + + if (telemetryMetadata || omitContext) { + telemetry = { + metadata: telemetryMetadata, + omitContext, + }; + } + } + + return { + name: job.name, + data: job.data, + opts: { + ...this.jobsOpts, + ...job.opts, + jobId: job.opts?.jobId, + telemetry, + }, + }; + }), ); }, ); diff --git a/src/classes/worker.ts b/src/classes/worker.ts index 0c2ae6d284..f68811c589 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -569,7 +569,7 @@ export class Worker< return nextJob; }, - nextJob?.opts.telemetryMetadata, + nextJob?.opts?.telemetry?.metadata, ); } @@ -821,7 +821,7 @@ will never work with more accuracy than 1ms. */ return; } - const { telemetryMetadata: srcPropagationMedatada } = job.opts; + const srcPropagationMedatada = job.opts?.telemetry?.metadata; return this.trace>( SpanKind.CONSUMER, diff --git a/src/interfaces/base-job-options.ts b/src/interfaces/base-job-options.ts index 269309263b..7ad62439cc 100644 --- a/src/interfaces/base-job-options.ts +++ b/src/interfaces/base-job-options.ts @@ -114,9 +114,4 @@ export interface BaseJobOptions extends DefaultJobOptions { * Internal property used by repeatable jobs. */ prevMillis?: number; - - /** - * TelemetryMetadata, provide for context propagation. - */ - telemetryMetadata?: string; } diff --git a/src/types/job-options.ts b/src/types/job-options.ts index fb8b74d264..3c8c094d9c 100644 --- a/src/types/job-options.ts +++ b/src/types/job-options.ts @@ -1,6 +1,10 @@ import { BaseJobOptions, DebounceOptions } from '../interfaces'; -export type JobsOptions = BaseJobOptions & { +/** + * These options will be stored in Redis with smaller + * keys for compactness. + */ +export type CompressableJobOptions = { /** * Debounce options. * @deprecated use deduplication option @@ -26,8 +30,26 @@ export type JobsOptions = BaseJobOptions & { * If true, removes the job from its parent dependencies when it fails after all attempts. */ removeDependencyOnFailure?: boolean; + + /** + * Telemetry options + */ + telemetry?: { + /** + * Metadata, used for context propagation. + */ + metadata?: string; + + /** + * If `true` telemetry will omit the context propagation + * @default false + */ + omitContext?: boolean; + }; }; +export type JobsOptions = BaseJobOptions & CompressableJobOptions; + /** * These fields are the ones stored in Redis with smaller keys for compactness. */ @@ -61,4 +83,15 @@ export type RedisJobOptions = BaseJobOptions & { * TelemetryMetadata, provide for context propagation. */ tm?: string; + + /** + * Omit Context Propagation + */ + omc?: boolean; + + /** + * Deduplication identifier. + * @deprecated use deid + */ + de?: string; }; diff --git a/src/utils.ts b/src/utils.ts index 71c2d5e1da..38999e56b5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -17,7 +17,6 @@ import { EventEmitter } from 'events'; import * as semver from 'semver'; import { SpanKind, TelemetryAttributes } from './enums'; -import { JobsOptions, RedisJobOptions } from './types'; export const errorObject: { [index: string]: any } = { value: null }; @@ -98,16 +97,21 @@ export function increaseMaxListeners( emitter.setMaxListeners(maxListeners + count); } -export const invertObject = (obj: Record) => { - return Object.entries(obj).reduce>( - (encodeMap, [key, value]) => { - encodeMap[value] = key; - return encodeMap; - }, - {}, - ); +type Invert> = { + [V in T[keyof T]]: { + [K in keyof T]: T[K] extends V ? K : never; + }[keyof T]; }; +export function invertObject>( + obj: T, +): Invert { + return Object.entries(obj).reduce((result, [key, value]) => { + (result as Record)[value] = key; + return result; + }, {} as Invert); +} + export function isRedisInstance(obj: any): obj is Redis | Cluster { if (!obj) { return false; @@ -285,57 +289,6 @@ export const toString = (value: any): string => { export const QUEUE_EVENT_SUFFIX = ':qe'; -const optsDecodeMap = { - de: 'deduplication', - fpof: 'failParentOnFailure', - idof: 'ignoreDependencyOnFailure', - kl: 'keepLogs', - rdof: 'removeDependencyOnFailure', - tm: 'telemetryMetadata', -}; - -const optsEncodeMap = invertObject(optsDecodeMap); -optsEncodeMap.debounce = 'de'; - -export function optsAsJSON(opts: JobsOptions = {}): RedisJobOptions { - const optionEntries = Object.entries(opts) as Array<[keyof JobsOptions, any]>; - const options: Partial> = {}; - for (const item of optionEntries) { - const [attributeName, value] = item; - if (value !== undefined) { - if ((optsEncodeMap as Record)[attributeName]) { - options[(optsEncodeMap as Record)[attributeName]] = - value; - } else { - options[attributeName] = value; - } - } - } - - return options as RedisJobOptions; -} - -export function optsFromJSON(rawOpts?: string): JobsOptions { - const opts = JSON.parse(rawOpts || '{}'); - - const optionEntries = Object.entries(opts) as Array< - [keyof RedisJobOptions, any] - >; - - const options: Partial> = {}; - for (const item of optionEntries) { - const [attributeName, value] = item; - if ((optsDecodeMap as Record)[attributeName]) { - options[(optsDecodeMap as Record)[attributeName]] = - value; - } else { - options[attributeName] = value; - } - } - - return options as JobsOptions; -} - export function removeUndefinedFields>( obj: Record, ) { diff --git a/tests/test_telemetry_interface.ts b/tests/test_telemetry_interface.ts index b598475d86..fa6ff4290a 100644 --- a/tests/test_telemetry_interface.ts +++ b/tests/test_telemetry_interface.ts @@ -2,7 +2,7 @@ import { expect, assert } from 'chai'; import { default as IORedis } from 'ioredis'; import { after, beforeEach, describe, it, before } from 'mocha'; import { v4 } from 'uuid'; -import { FlowProducer, JobScheduler, Queue, Worker } from '../src/classes'; +import { FlowProducer, Job, JobScheduler, Queue, Worker } from '../src/classes'; import { removeAllQueueData } from '../src/utils'; import { Telemetry, @@ -93,7 +93,7 @@ describe('Telemetry', () => { this.options = options; } - setSpanOnContext(ctx: any): any { + setSpanOnContext(ctx: any, omitContext?: boolean): any { context['getSpan'] = () => this; return { ...context, getMetadata_span: this['name'] }; } @@ -260,6 +260,7 @@ describe('Telemetry', () => { }); it('should correctly handle errors and record them in telemetry for upsertJobScheduler', async () => { + const originalCreateNextJob = JobScheduler.prototype.createNextJob; const recordExceptionSpy = sinon.spy( MockSpan.prototype, 'recordException', @@ -283,6 +284,7 @@ describe('Telemetry', () => { const recordedError = recordExceptionSpy.firstCall.args[0]; assert.equal(recordedError.message, errMessage); } finally { + JobScheduler.prototype.createNextJob = originalCreateNextJob; recordExceptionSpy.restore(); } }); @@ -296,6 +298,7 @@ describe('Telemetry', () => { connection, telemetry: telemetryClient, name: 'testWorker', + prefix, }); await worker.waitUntilReady(); @@ -328,6 +331,7 @@ describe('Telemetry', () => { const worker = new Worker(queueName, async () => 'some result', { connection, telemetry: telemetryClient, + prefix, }); await worker.waitUntilReady(); @@ -349,17 +353,20 @@ describe('Telemetry', () => { const flowProducer = new FlowProducer({ connection, telemetry: telemetryClient, + prefix, }); const traceSpy = sinon.spy(telemetryClient.tracer, 'startSpan'); const testFlow = { name: 'parentJob', queueName, + prefix, data: { foo: 'bar' }, children: [ { name: 'childJob', queueName, + prefix, data: { baz: 'qux' }, }, ], @@ -385,6 +392,7 @@ describe('Telemetry', () => { const flowProducer = new FlowProducer({ connection, telemetry: telemetryClient, + prefix, }); const traceSpy = sinon.spy(telemetryClient.tracer, 'startSpan'); @@ -420,6 +428,7 @@ describe('Telemetry', () => { const flowProducer = new FlowProducer({ connection, telemetry: telemetryClient, + prefix, }); const traceSpy = sinon.spy(telemetryClient.tracer, 'startSpan'); @@ -472,6 +481,7 @@ describe('Telemetry', () => { const flowProducer = new FlowProducer({ connection, telemetry: telemetryClient, + prefix, }); const traceSpy = sinon.spy(telemetryClient.tracer, 'startSpan'); @@ -511,4 +521,137 @@ describe('Telemetry', () => { } }); }); + + describe('Omit Propagation', () => { + let fromMetadataSpy; + + beforeEach(() => { + fromMetadataSpy = sinon.spy( + telemetryClient.contextManager, + 'fromMetadata', + ); + }); + + afterEach(() => fromMetadataSpy.restore()); + + it('should omit propagation on queue add', async () => { + let worker; + const processing = new Promise(resolve => { + worker = new Worker(queueName, async () => resolve(), { + connection, + telemetry: telemetryClient, + prefix, + }); + }); + + const job = await queue.add( + 'testJob', + { foo: 'bar' }, + { telemetry: { omitContext: true } }, + ); + + await processing; + + expect(fromMetadataSpy.callCount).to.equal(0); + await worker.close(); + }); + + it('should omit propagation on queue addBulk', async () => { + let worker; + const processing = new Promise(resolve => { + worker = new Worker(queueName, async () => resolve(), { + connection, + telemetry: telemetryClient, + prefix, + }); + }); + + const jobs = [ + { + name: 'job1', + data: { foo: 'bar' }, + opts: { telemetry: { omitContext: true } }, + }, + ]; + const addedJos = await queue.addBulk(jobs); + expect(addedJos).to.have.length(1); + + await processing; + + expect(fromMetadataSpy.callCount).to.equal(0); + await worker.close(); + }); + + it('should omit propagation on job scheduler', async () => { + let worker; + const processing = new Promise(resolve => { + worker = new Worker(queueName, async () => resolve(), { + connection, + telemetry: telemetryClient, + prefix, + }); + }); + + const jobSchedulerId = 'testJobScheduler'; + const data = { foo: 'bar' }; + + const job = await queue.upsertJobScheduler( + jobSchedulerId, + { every: 1000, endDate: Date.now() + 1000, limit: 1 }, + { + name: 'repeatable-job', + data, + opts: { telemetry: { omitContext: true } }, + }, + ); + + await processing; + + expect(fromMetadataSpy.callCount).to.equal(0); + await worker.close(); + }); + + it('should omit propagation on flow producer', async () => { + let worker; + const processing = new Promise(resolve => { + worker = new Worker(queueName, async () => resolve(), { + connection, + telemetry: telemetryClient, + prefix, + }); + }); + + const flowProducer = new FlowProducer({ + connection, + telemetry: telemetryClient, + prefix, + }); + + const testFlow = { + name: 'parentJob', + queueName, + data: { foo: 'bar' }, + children: [ + { + name: 'childJob', + queueName, + data: { baz: 'qux' }, + opts: { telemetry: { omitContext: true } }, + }, + ], + opts: { telemetry: { omitContext: true } }, + }; + + const jobNode = await flowProducer.add(testFlow); + const jobs = jobNode.children + ? [jobNode.job, ...jobNode.children.map(c => c.job)] + : [jobNode.job]; + + await processing; + + expect(fromMetadataSpy.callCount).to.equal(0); + await flowProducer.close(); + await worker.close(); + }); + }); }); From fc8fac9d68e7f4de2322a82a38740a79b05e57f4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 10 Dec 2024 14:32:43 +0000 Subject: [PATCH 42/52] chore(release): 5.34.0 [skip ci] # [5.34.0](https://github.com/taskforcesh/bullmq/compare/v5.33.1...v5.34.0) (2024-12-10) ### Features * **telemetry:** add option to omit context propagation on jobs ([#2946](https://github.com/taskforcesh/bullmq/issues/2946)) ([6514c33](https://github.com/taskforcesh/bullmq/commit/6514c335231cb6e727819cf5e0c56ed3f5132838)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 5eee314976..9509a01115 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +# [5.34.0](https://github.com/taskforcesh/bullmq/compare/v5.33.1...v5.34.0) (2024-12-10) + + +### Features + +* **telemetry:** add option to omit context propagation on jobs ([#2946](https://github.com/taskforcesh/bullmq/issues/2946)) ([6514c33](https://github.com/taskforcesh/bullmq/commit/6514c335231cb6e727819cf5e0c56ed3f5132838)) + ## [5.33.1](https://github.com/taskforcesh/bullmq/compare/v5.33.0...v5.33.1) (2024-12-10) diff --git a/package.json b/package.json index 00e0d0a455..f027a38f37 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.33.1", + "version": "5.34.0", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index cb3e91677f..83d5b9f50d 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.33.1'; +export const version = '5.34.0'; From 3b218ff3a3af4286068b7aaee8f0d0909fd4f52e Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Tue, 10 Dec 2024 22:29:56 +0100 Subject: [PATCH 43/52] docs: typo in docstring for `moveToDelayed` (#2961) --- src/classes/job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classes/job.ts b/src/classes/job.ts index 780005fc0a..531839e62c 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -1147,7 +1147,7 @@ export class Job< /** * Moves the job to the delay set. * - * @param timestamp - timestamp where the job should be moved back to "wait" + * @param timestamp - timestamp when the job should be moved back to "wait" * @param token - token to check job is locked by current worker * @returns */ From 9917df166aff2e2f143c45297f41ac8520bfc8ae Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Thu, 12 Dec 2024 23:12:57 +0100 Subject: [PATCH 44/52] fix: guarantee every repeatable jobs are slotted --- src/classes/job-scheduler.ts | 28 +++- src/interfaces/repeat-options.ts | 15 +- tests/test_job_scheduler.ts | 258 +++++++++++++++++++++++++------ 3 files changed, 236 insertions(+), 65 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 2e41591e45..b913af1405 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -33,13 +33,13 @@ export class JobScheduler extends QueueBase { async upsertJobScheduler( jobSchedulerId: string, - repeatOpts: Omit, + repeatOpts: Omit, jobName: N, jobData: T, opts: JobSchedulerTemplateOptions, { override }: { override: boolean }, ): Promise | undefined> { - const { every, pattern } = repeatOpts; + const { every, pattern, offset } = repeatOpts; if (pattern && every) { throw new Error( @@ -59,6 +59,12 @@ export class JobScheduler extends QueueBase { ); } + if (repeatOpts.immediately && repeatOpts.every) { + console.warn( + "Using option immediately with every does not affect the job's schedule. Job will run immediately anyway.", + ); + } + // Check if we reached the limit of the repeatable job's iterations const iterationCount = repeatOpts.count ? repeatOpts.count + 1 : 1; if ( @@ -75,8 +81,6 @@ export class JobScheduler extends QueueBase { return; } - const prevMillis = opts.prevMillis || 0; - // Check if we have a start date for the repeatable job const { startDate, immediately, ...filteredRepeatOpts } = repeatOpts; if (startDate) { @@ -84,15 +88,25 @@ export class JobScheduler extends QueueBase { now = startMillis > now ? startMillis : now; } + const prevMillis = opts.prevMillis || 0; + now = prevMillis < now ? now : prevMillis; + let nextMillis: number; + let newOffset = offset; + if (every) { - nextMillis = prevMillis + every; + const nextSlot = Math.floor(now / every) * every + every; + if (prevMillis || offset) { + nextMillis = nextSlot + (offset || 0); + } else { + nextMillis = now; + newOffset = every - (nextSlot - now); + } if (nextMillis < now) { nextMillis = now; } } else if (pattern) { - now = prevMillis < now ? now : prevMillis; nextMillis = await this.repeatStrategy(now, repeatOpts, jobName); } @@ -149,7 +163,7 @@ export class JobScheduler extends QueueBase { jobSchedulerId, { ...opts, - repeat: filteredRepeatOpts, + repeat: { ...filteredRepeatOpts, offset: newOffset }, telemetry, }, jobData, diff --git a/src/interfaces/repeat-options.ts b/src/interfaces/repeat-options.ts index fe71d27d12..65c5da3d4d 100644 --- a/src/interfaces/repeat-options.ts +++ b/src/interfaces/repeat-options.ts @@ -32,7 +32,7 @@ export interface RepeatOptions extends Omit { /** * Repeated job should start right now - * ( work only with every settings) + * ( work only with cron settings) */ immediately?: boolean; @@ -42,16 +42,15 @@ export interface RepeatOptions extends Omit { count?: number; /** - * Internal property to store the previous time the job was executed. - */ - prevMillis?: number; + * Offset in milliseconds to affect the next iteration time + * + * */ + offset?: number; /** - * Internal property to store the offset to apply to the next iteration. - * - * @deprecated + * Internal property to store the previous time the job was executed. */ - offset?: number; + prevMillis?: number; /** * Internal property to store de job id diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 427cb99cee..a3288ddee8 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -191,6 +191,9 @@ describe('Job Scheduler', function () { describe('when job schedulers have same id and different every pattern', function () { it('should create only one job scheduler', async function () { + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + await Promise.all([ queue.upsertJobScheduler('test-scheduler1', { every: 1000 }), queue.upsertJobScheduler('test-scheduler1', { every: 2000 }), @@ -244,6 +247,9 @@ describe('Job Scheduler', function () { }); it('should create job schedulers with different cron patterns', async function () { + const date = new Date('2017-02-07T15:24:00.000Z'); + this.clock.setSystemTime(date); + const crons = [ '10 * * * * *', '2 10 * * * *', @@ -254,11 +260,11 @@ describe('Job Scheduler', function () { await Promise.all([ queue.upsertJobScheduler('first', { pattern: crons[0], - endDate: 12345, + endDate: Date.now() + 12345, }), queue.upsertJobScheduler('second', { pattern: crons[1], - endDate: 610000, + endDate: Date.now() + 6100000, }), queue.upsertJobScheduler('third', { pattern: crons[2], @@ -273,9 +279,13 @@ describe('Job Scheduler', function () { tz: 'Europa/Copenhaguen', }), ]); + const count = await repeat.getRepeatableCount(); expect(count).to.be.eql(5); + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.eql(5); + const jobs = await repeat.getRepeatableJobs(0, -1, true); expect(jobs) @@ -288,25 +298,25 @@ describe('Job Scheduler', function () { tz: 'Europa/Copenhaguen', pattern: null, every: '5000', - next: 5000, + next: 1486481040000, }) .and.to.deep.include({ key: 'first', name: 'first', - endDate: 12345, + endDate: Date.now() + 12345, tz: null, pattern: '10 * * * * *', every: null, - next: 10000, + next: 1486481050000, }) .and.to.deep.include({ key: 'second', name: 'second', - endDate: 610000, + endDate: Date.now() + 6100000, tz: null, pattern: '2 10 * * * *', every: null, - next: 602000, + next: 1486483802000, }) .and.to.deep.include({ key: 'fourth', @@ -315,7 +325,7 @@ describe('Job Scheduler', function () { tz: 'Africa/Accra', pattern: '2 * * 4 * *', every: null, - next: 259202000, + next: 1488585602000, }) .and.to.deep.include({ key: 'third', @@ -324,7 +334,7 @@ describe('Job Scheduler', function () { tz: 'Africa/Abidjan', pattern: '1 * * 5 * *', every: null, - next: 345601000, + next: 1488672001000, }); }); @@ -773,54 +783,177 @@ describe('Job Scheduler', function () { }); }); - it('should repeat every 2 seconds and start immediately', async function () { - const date = new Date('2017-02-07 9:24:00'); - this.clock.setSystemTime(date); - const nextTick = 2 * ONE_SECOND; + describe("when using 'every' option is on same millis as iteration time", function () { + it('should repeat every 2 seconds and start immediately', async function () { + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + const nextTick = 2 * ONE_SECOND; - const worker = new Worker( - queueName, - async () => { - this.clock.tick(nextTick); - }, - { connection, prefix }, - ); + const worker = new Worker( + queueName, + async () => { + this.clock.tick(nextTick); + }, + { connection, prefix }, + ); - let prev: Job; - let counter = 0; + let prev: Job; + let counter = 0; - const completing = new Promise((resolve, reject) => { - worker.on('completed', async job => { - try { - if (prev && counter === 1) { - expect(prev.timestamp).to.be.lte(job.timestamp); - expect(job.timestamp - prev.timestamp).to.be.lte(1); - } else if (prev) { - expect(prev.timestamp).to.be.lt(job.timestamp); - expect(job.timestamp - prev.timestamp).to.be.gte(2000); + const completing = new Promise((resolve, reject) => { + worker.on('completed', async job => { + try { + if (prev && counter === 1) { + expect(prev.timestamp).to.be.lte(job.timestamp); + expect(job.timestamp - prev.timestamp).to.be.lte(1); + } else if (prev) { + expect(prev.timestamp).to.be.lt(job.timestamp); + expect(job.timestamp - prev.timestamp).to.be.eq(2000); + } + prev = job; + counter++; + if (counter === 5) { + resolve(); + } + } catch (err) { + reject(err); } - prev = job; - counter++; - if (counter === 5) { - resolve(); + }); + }); + + await queue.upsertJobScheduler( + 'repeat', + { + every: 2000, + }, + { data: { foo: 'bar' } }, + ); + + const delayedCountBefore = await queue.getDelayedCount(); + expect(delayedCountBefore).to.be.eq(1); + + await completing; + + const waitingCount = await queue.getWaitingCount(); + expect(waitingCount).to.be.eq(0); + + const delayedCountAfter = await queue.getDelayedCount(); + expect(delayedCountAfter).to.be.eq(1); + + await worker.close(); + }); + }); + + describe("when using 'every' and time is one millisecond before iteration time", function () { + it('should repeat every 2 seconds and start immediately', async function () { + const startTimeMillis = new Date('2017-02-07 9:24:00').getTime(); + + const date = new Date(startTimeMillis - 1); + this.clock.setSystemTime(date); + const nextTick = 2 * ONE_SECOND; + + const worker = new Worker( + queueName, + async () => { + this.clock.tick(nextTick); + }, + { connection, prefix }, + ); + + let prev: Job; + let counter = 0; + + const completing = new Promise((resolve, reject) => { + worker.on('completed', async job => { + try { + if (prev && counter === 1) { + expect(prev.timestamp).to.be.lte(job.timestamp); + expect(job.timestamp - prev.timestamp).to.be.lte(1); + } else if (prev) { + expect(prev.timestamp).to.be.lt(job.timestamp); + expect(job.timestamp - prev.timestamp).to.be.eq(2000); + } + + prev = job; + counter++; + if (counter === 5) { + resolve(); + } + } catch (err) { + reject(err); } - } catch (err) { - reject(err); - } + }); }); + + await queue.upsertJobScheduler( + 'repeat', + { + every: 2000, + }, + { data: { foo: 'bar' } }, + ); + + await completing; + + await worker.close(); }); + }); - await queue.upsertJobScheduler( - 'repeat', - { - every: 2000, - }, - { data: { foo: 'bar' } }, - ); + describe("when using 'every' and time is one millisecond after iteration time", function () { + it('should repeat every 2 seconds and start immediately', async function () { + const startTimeMillis = new Date('2017-02-07 9:24:00').getTime() + 1; - await completing; + const date = new Date(startTimeMillis); + this.clock.setSystemTime(date); + const nextTick = 2 * ONE_SECOND; - await worker.close(); + const worker = new Worker( + queueName, + async () => { + this.clock.tick(nextTick); + }, + { connection, prefix }, + ); + + let prev: Job; + let counter = 0; + + const completing = new Promise((resolve, reject) => { + worker.on('completed', async job => { + try { + if (prev && counter === 1) { + expect(prev.timestamp).to.be.lte(job.timestamp); + expect(job.timestamp - prev.timestamp).to.be.lte(1); + } else if (prev) { + expect(prev.timestamp).to.be.lt(job.timestamp); + expect(job.timestamp - prev.timestamp).to.be.eq(2000); + } + + prev = job; + counter++; + if (counter === 5) { + resolve(); + } + } catch (err) { + reject(err); + } + }); + }); + + await queue.upsertJobScheduler( + 'repeat', + { + every: 2000, + }, + { data: { foo: 'bar' } }, + ); + + //this.clock.tick(1000); + + await completing; + + await worker.close(); + }); }); it('should start immediately even after removing the job scheduler and adding it again', async function () { @@ -850,7 +983,6 @@ describe('Job Scheduler', function () { 'repeat', { every: 2000, - immediately: true, }, { data: { foo: 'bar' } }, ); @@ -884,7 +1016,6 @@ describe('Job Scheduler', function () { 'repeat', { every: 2000, - immediately: true, }, { data: { foo: 'bar' } }, ); @@ -1196,12 +1327,15 @@ describe('Job Scheduler', function () { }); }); - await queue.upsertJobScheduler('repeat', { + const job = await queue.upsertJobScheduler('repeat', { pattern: '0 1 * * *', endDate: new Date('2017-05-10 13:13:00'), tz: 'Europe/Athens', utc: true, }); + + expect(job).to.be.ok; + this.clock.tick(nextTick + delay); worker.run(); @@ -1495,6 +1629,9 @@ describe('Job Scheduler', function () { }); it('should not create a new delayed job if the failed job is retried with retryJobs', async function () { + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + const repeatOpts = { every: 579, }; @@ -1543,6 +1680,9 @@ describe('Job Scheduler', function () { }); it('should not create a new delayed job if the failed job is retried with Job.retry()', async function () { + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + const repeatOpts = { every: 477, }; @@ -1884,6 +2024,9 @@ describe('Job Scheduler', function () { }).timeout(8000); it('should not allow to remove a delayed job if it belongs to a repeatable job', async function () { + const date = new Date('2019-07-13 1:58:23'); + this.clock.setSystemTime(date); + const repeat = { every: 1000, }; @@ -1902,6 +2045,9 @@ describe('Job Scheduler', function () { }); it('should not remove delayed jobs if they belong to a repeatable job when using drain', async function () { + const date = new Date('2014-09-03 5:32:12'); + this.clock.setSystemTime(date); + await queue.upsertJobScheduler('myTestJob', { every: 5000 }); await queue.add('test', { foo: 'bar' }, { delay: 1000 }); @@ -1919,6 +2065,9 @@ describe('Job Scheduler', function () { }); it('should not remove delayed jobs if they belong to a repeatable job when using clean', async function () { + const date = new Date('2012-08-05 2:32:12'); + this.clock.setSystemTime(date); + await queue.upsertJobScheduler('myTestJob', { every: 5000 }); await queue.add('test', { foo: 'bar' }, { delay: 1000 }); @@ -1936,6 +2085,9 @@ describe('Job Scheduler', function () { }); it("should keep one delayed job if updating a repeatable job's every option", async function () { + const date = new Date('2022-01-08 7:22:21'); + this.clock.setSystemTime(date); + await queue.upsertJobScheduler('myTestJob', { every: 5000 }); await queue.upsertJobScheduler('myTestJob', { every: 4000 }); await queue.upsertJobScheduler('myTestJob', { every: 5000 }); @@ -2175,6 +2327,9 @@ describe('Job Scheduler', function () { }); it("should return a valid job with the job's options and data passed as the job template", async function () { + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + const repeatOpts = { every: 1000, }; @@ -2224,6 +2379,9 @@ describe('Job Scheduler', function () { }); it('should have the right count value', async function () { + const date = new Date('2017-02-07 9:24:00'); + this.clock.setSystemTime(date); + await queue.upsertJobScheduler('test', { every: 1000 }); this.clock.tick(ONE_SECOND + 100); From 30409306c2faf41125c7e143f8abe4c033f9324a Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Thu, 12 Dec 2024 23:19:33 +0100 Subject: [PATCH 45/52] chore(job-scheduler): make sure offset is never negative --- src/classes/job-scheduler.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index b913af1405..2400073427 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -101,6 +101,9 @@ export class JobScheduler extends QueueBase { } else { nextMillis = now; newOffset = every - (nextSlot - now); + + // newOffset should always be positive, but as an extra safety check + newOffset = newOffset < 0 ? 0 : newOffset; } if (nextMillis < now) { From 7113f4a1a815576f094eb919f4e90ce32f5576f0 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Fri, 13 Dec 2024 09:02:44 +0100 Subject: [PATCH 46/52] test: remove comment --- tests/test_job_scheduler.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index a3288ddee8..3e94a5dcbf 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -948,8 +948,6 @@ describe('Job Scheduler', function () { { data: { foo: 'bar' } }, ); - //this.clock.tick(1000); - await completing; await worker.close(); From 791ba26ec8f0639b9597ed8b7f689f47f6c81d27 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Fri, 13 Dec 2024 09:30:47 +0100 Subject: [PATCH 47/52] test: fix flaky test --- tests/test_job_scheduler.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 3e94a5dcbf..1b3781ae04 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1710,11 +1710,12 @@ describe('Job Scheduler', function () { }); const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); - const delayedCount = await queue.getDelayedCount(); - expect(delayedCount).to.be.equal(1); await repeatableJob!.promote(); + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.equal(0); + this.clock.tick(177); await failing; From af75315f0c7923f5e0a667a9ed4606b28b89b719 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Fri, 13 Dec 2024 11:26:28 +0100 Subject: [PATCH 48/52] fix(job-scheduler): avoid duplicated delayed jobs when repeatable jobs are retried --- src/classes/job-scheduler.ts | 10 +++++- src/classes/job.ts | 11 ++++++ src/classes/worker.ts | 10 +++--- src/interfaces/job-json.ts | 2 ++ tests/test_job_scheduler.ts | 65 +++++++++++++++++++++++++++--------- 5 files changed, 77 insertions(+), 21 deletions(-) diff --git a/src/classes/job-scheduler.ts b/src/classes/job-scheduler.ts index 2400073427..743af75b72 100644 --- a/src/classes/job-scheduler.ts +++ b/src/classes/job-scheduler.ts @@ -37,7 +37,7 @@ export class JobScheduler extends QueueBase { jobName: N, jobData: T, opts: JobSchedulerTemplateOptions, - { override }: { override: boolean }, + { override, producerId }: { override: boolean; producerId?: string }, ): Promise | undefined> { const { every, pattern, offset } = repeatOpts; @@ -171,6 +171,7 @@ export class JobScheduler extends QueueBase { }, jobData, iterationCount, + producerId, ); const results = await multi.exec(); // multi.exec returns an array of results [ err, result ][] @@ -206,6 +207,8 @@ export class JobScheduler extends QueueBase { opts: JobsOptions, data: T, currentCount: number, + // The job id of the job that produced this next iteration + producerId?: string, ) { // // Generate unique job id for this iteration. @@ -232,6 +235,11 @@ export class JobScheduler extends QueueBase { const job = new this.Job(this, name, data, mergedOpts, jobId); job.addJob(client); + if (producerId) { + const producerJobKey = this.toKey(producerId); + client.hset(producerJobKey, 'nrjid', job.id); + } + return job; } diff --git a/src/classes/job.ts b/src/classes/job.ts index 531839e62c..19d96d3921 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -168,6 +168,12 @@ export class Job< */ repeatJobKey?: string; + /** + * Produced next repetable job Id. + * + */ + nextRepeatableJobId?: string; + /** * The token used for locking this job. */ @@ -384,6 +390,10 @@ export class Job< job.processedBy = json.pb; } + if (json.nrjid) { + job.nextRepeatableJobId = json.nrjid; + } + return job; } @@ -493,6 +503,7 @@ export class Job< deduplicationId: this.deduplicationId, repeatJobKey: this.repeatJobKey, returnvalue: JSON.stringify(this.returnvalue), + nrjid: this.nextRepeatableJobId, }); } diff --git a/src/classes/worker.ts b/src/classes/worker.ts index f68811c589..b79ddf4648 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -788,7 +788,7 @@ will never work with more accuracy than 1ms. */ job.token = token; // Add next scheduled job if necessary. - if (job.opts.repeat) { + if (job.opts.repeat && !job.nextRepeatableJobId) { // Use new job scheduler if possible if (job.repeatJobKey) { const jobScheduler = await this.jobScheduler; @@ -798,7 +798,7 @@ will never work with more accuracy than 1ms. */ job.name, job.data, job.opts, - { override: false }, + { override: false, producerId: job.id }, ); } else { const repeat = await this.repeat; @@ -835,6 +835,8 @@ will never work with more accuracy than 1ms. */ }); const handleCompleted = async (result: ResultType) => { + jobsInProgress.delete(inProgressItem); + if (!this.connection.closing) { const completed = await job.moveToCompleted( result, @@ -855,6 +857,8 @@ will never work with more accuracy than 1ms. */ }; const handleFailed = async (err: Error) => { + jobsInProgress.delete(inProgressItem); + if (!this.connection.closing) { try { // Check if the job was manually rate-limited @@ -911,8 +915,6 @@ will never work with more accuracy than 1ms. */ [TelemetryAttributes.JobFinishedTimestamp]: Date.now(), [TelemetryAttributes.JobProcessedTimestamp]: processedOn, }); - - jobsInProgress.delete(inProgressItem); } }, srcPropagationMedatada, diff --git a/src/interfaces/job-json.ts b/src/interfaces/job-json.ts index 25ad335145..e27c302432 100644 --- a/src/interfaces/job-json.ts +++ b/src/interfaces/job-json.ts @@ -18,6 +18,7 @@ export interface JobJson { parent?: ParentKeys; parentKey?: string; repeatJobKey?: string; + nextRepeatableJobKey?: string; debounceId?: string; deduplicationId?: string; processedBy?: string; @@ -41,6 +42,7 @@ export interface JobJsonRaw { parent?: string; deid?: string; rjk?: string; + nrjid?: string; atm?: string; ats?: string; pb?: string; // Worker name diff --git a/tests/test_job_scheduler.ts b/tests/test_job_scheduler.ts index 1b3781ae04..c2da779a83 100644 --- a/tests/test_job_scheduler.ts +++ b/tests/test_job_scheduler.ts @@ -1636,20 +1636,24 @@ describe('Job Scheduler', function () { let isFirstRun = true; - const worker = new Worker( - queueName, - async () => { - this.clock.tick(177); - if (isFirstRun) { - isFirstRun = false; - throw new Error('failed'); - } - }, - { - connection, - prefix, - }, - ); + let worker; + const processingAfterFailing = new Promise(resolve => { + worker = new Worker( + queueName, + async () => { + this.clock.tick(177); + if (isFirstRun) { + isFirstRun = false; + throw new Error('failed'); + } + resolve(); + }, + { + connection, + prefix, + }, + ); + }); const failing = new Promise(resolve => { worker.on('failed', async () => { @@ -1658,26 +1662,40 @@ describe('Job Scheduler', function () { }); const repeatableJob = await queue.upsertJobScheduler('test', repeatOpts); - const delayedCount = await queue.getDelayedCount(); - expect(delayedCount).to.be.equal(1); await repeatableJob!.promote(); + + const delayedCountBeforeFailing = await queue.getDelayedCount(); + expect(delayedCountBeforeFailing).to.be.equal(0); + await failing; const failedCount = await queue.getFailedCount(); expect(failedCount).to.be.equal(1); + const delayedCountAfterFailing = await queue.getDelayedCount(); + expect(delayedCountAfterFailing).to.be.equal(1); + // Retry the failed job this.clock.tick(1143); await queue.retryJobs({ state: 'failed' }); const failedCountAfterRetry = await queue.getFailedCount(); expect(failedCountAfterRetry).to.be.equal(0); + await processingAfterFailing; + + await worker.close(); + const delayedCount2 = await queue.getDelayedCount(); expect(delayedCount2).to.be.equal(1); + + const waitingCount = await queue.getWaitingCount(); + expect(waitingCount).to.be.equal(0); }); it('should not create a new delayed job if the failed job is retried with Job.retry()', async function () { + let expectError; + const date = new Date('2017-02-07 9:24:00'); this.clock.setSystemTime(date); @@ -1692,6 +1710,13 @@ describe('Job Scheduler', function () { async () => { this.clock.tick(177); + try { + const delayedCount = await queue.getDelayedCount(); + expect(delayedCount).to.be.equal(1); + } catch (error) { + expectError = error; + } + if (isFirstRun) { isFirstRun = false; throw new Error('failed'); @@ -1731,6 +1756,14 @@ describe('Job Scheduler', function () { const failedCountAfterRetry = await queue.getFailedCount(); expect(failedCountAfterRetry).to.be.equal(0); + this.clock.tick(177); + + await worker.close(); + + if (expectError) { + throw expectError; + } + const delayedCount2 = await queue.getDelayedCount(); expect(delayedCount2).to.be.equal(1); }); From af020612644be1ba549a1c4802f6ab83c33aeedb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Dec 2024 14:15:44 +0000 Subject: [PATCH 49/52] chore(release): 5.34.1 [skip ci] ## [5.34.1](https://github.com/taskforcesh/bullmq/compare/v5.34.0...v5.34.1) (2024-12-13) ### Bug Fixes * guarantee every repeatable jobs are slotted ([9917df1](https://github.com/taskforcesh/bullmq/commit/9917df166aff2e2f143c45297f41ac8520bfc8ae)) * **job-scheduler:** avoid duplicated delayed jobs when repeatable jobs are retried ([af75315](https://github.com/taskforcesh/bullmq/commit/af75315f0c7923f5e0a667a9ed4606b28b89b719)) --- docs/gitbook/changelog.md | 8 ++++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 9509a01115..a4c2c179bb 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,11 @@ +## [5.34.1](https://github.com/taskforcesh/bullmq/compare/v5.34.0...v5.34.1) (2024-12-13) + + +### Bug Fixes + +* guarantee every repeatable jobs are slotted ([9917df1](https://github.com/taskforcesh/bullmq/commit/9917df166aff2e2f143c45297f41ac8520bfc8ae)) +* **job-scheduler:** avoid duplicated delayed jobs when repeatable jobs are retried ([af75315](https://github.com/taskforcesh/bullmq/commit/af75315f0c7923f5e0a667a9ed4606b28b89b719)) + # [5.34.0](https://github.com/taskforcesh/bullmq/compare/v5.33.1...v5.34.0) (2024-12-10) diff --git a/package.json b/package.json index f027a38f37..64cb485ade 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.34.0", + "version": "5.34.1", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index 83d5b9f50d..b247264678 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.34.0'; +export const version = '5.34.1'; From 4360572745a929c7c4f6266ec03d4eba77a9715c Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Sat, 14 Dec 2024 19:35:27 +0100 Subject: [PATCH 50/52] fix(scripts): make sure jobs fields are not empty before unpack --- src/commands/includes/updateJobFields.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/includes/updateJobFields.lua b/src/commands/includes/updateJobFields.lua index 9d9057f21a..d2c5f2944d 100644 --- a/src/commands/includes/updateJobFields.lua +++ b/src/commands/includes/updateJobFields.lua @@ -2,7 +2,7 @@ Function to update a bunch of fields in a job. ]] local function updateJobFields(jobKey, msgpackedFields) - if msgpackedFields then + if msgpackedFields and #msgpackedFields > 0 then local fieldsToUpdate = cmsgpack.unpack(msgpackedFields) if fieldsToUpdate then redis.call("HMSET", jobKey, unpack(fieldsToUpdate)) From bfd0b8a808a33be0da92a006a7b4f8b17a5d79b3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 14 Dec 2024 18:51:58 +0000 Subject: [PATCH 51/52] chore(release): 5.34.2 [skip ci] ## [5.34.2](https://github.com/taskforcesh/bullmq/compare/v5.34.1...v5.34.2) (2024-12-14) ### Bug Fixes * **scripts:** make sure jobs fields are not empty before unpack ([4360572](https://github.com/taskforcesh/bullmq/commit/4360572745a929c7c4f6266ec03d4eba77a9715c)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- src/version.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index a4c2c179bb..5f9602872b 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [5.34.2](https://github.com/taskforcesh/bullmq/compare/v5.34.1...v5.34.2) (2024-12-14) + + +### Bug Fixes + +* **scripts:** make sure jobs fields are not empty before unpack ([4360572](https://github.com/taskforcesh/bullmq/commit/4360572745a929c7c4f6266ec03d4eba77a9715c)) + ## [5.34.1](https://github.com/taskforcesh/bullmq/compare/v5.34.0...v5.34.1) (2024-12-13) diff --git a/package.json b/package.json index 64cb485ade..309043f090 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.34.1", + "version": "5.34.2", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", diff --git a/src/version.ts b/src/version.ts index b247264678..d0feaa17ea 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const version = '5.34.1'; +export const version = '5.34.2'; From e758d23d8e2a0b25006335fb75534c3bcc19c34a Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Tue, 17 Dec 2024 12:09:07 +0000 Subject: [PATCH 52/52] GITBOOK-212: change request with no subject merged in GitBook --- docs/gitbook/SUMMARY.md | 239 ++++++++++++------------ docs/gitbook/bullmq-pro/new-releases.md | 5 + 2 files changed, 125 insertions(+), 119 deletions(-) create mode 100644 docs/gitbook/bullmq-pro/new-releases.md diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index dea69273f3..54d647f7ef 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -1,136 +1,137 @@ # Table of contents -- [What is BullMQ](README.md) -- [Quick Start]() -- [API Reference](https://api.docs.bullmq.io) -- [Changelogs](changelog.md) - - [v4](changelogs/changelog-v4.md) - - [v3](changelogs/changelog-v3.md) - - [v2](changelogs/changelog-v2.md) - - [v1](changelogs/changelog-v1.md) +* [What is BullMQ](README.md) +* [Quick Start]() +* [API Reference](https://api.docs.bullmq.io) +* [Changelogs](changelog.md) + * [v4](changelogs/changelog-v4.md) + * [v3](changelogs/changelog-v3.md) + * [v2](changelogs/changelog-v2.md) + * [v1](changelogs/changelog-v1.md) ## Guide -- [Introduction](guide/introduction.md) -- [Connections](guide/connections.md) -- [Queues](guide/queues/README.md) - - [Auto-removal of jobs](guide/queues/auto-removal-of-jobs.md) - - [Adding jobs in bulk](guide/queues/adding-bulks.md) - - [Global Concurrency](guide/queues/global-concurrency.md) - - [Removing Jobs](guide/queues/removing-jobs.md) -- [Workers](guide/workers/README.md) - - [Auto-removal of jobs](guide/workers/auto-removal-of-jobs.md) - - [Concurrency](guide/workers/concurrency.md) - - [Graceful shutdown](guide/workers/graceful-shutdown.md) - - [Stalled Jobs](guide/workers/stalled-jobs.md) - - [Sandboxed processors](guide/workers/sandboxed-processors.md) - - [Pausing queues](guide/workers/pausing-queues.md) -- [Jobs](guide/jobs/README.md) - - [FIFO](guide/jobs/fifo.md) - - [LIFO](guide/jobs/lifo.md) - - [Job Ids](guide/jobs/job-ids.md) - - [Job Data](guide/jobs/job-data.md) - - [Deduplication](guide/jobs/deduplication.md) - - [Delayed](guide/jobs/delayed.md) - - [Repeatable](guide/jobs/repeatable.md) - - [Prioritized](guide/jobs/prioritized.md) - - [Removing jobs](guide/jobs/removing-job.md) - - [Stalled](guide/jobs/stalled.md) - - [Getters](guide/jobs/getters.md) -- [Job Schedulers](guide/job-schedulers/README.md) - - [Repeat Strategies](guide/job-schedulers/repeat-strategies.md) - - [Repeat options](guide/job-schedulers/repeat-options.md) - - [Manage Job Schedulers](guide/job-schedulers/manage-job-schedulers.md) -- [Flows](guide/flows/README.md) - - [Adding flows in bulk](guide/flows/adding-bulks.md) - - [Get Flow Tree](guide/flows/get-flow-tree.md) - - [Fail Parent](guide/flows/fail-parent.md) - - [Remove Dependency](guide/flows/remove-dependency.md) - - [Ignore Dependency](guide/flows/ignore-dependency.md) - - [Remove Child Dependency](guide/flows/remove-child-dependency.md) -- [Metrics](guide/metrics/metrics.md) -- [Rate limiting](guide/rate-limiting.md) -- [Parallelism and Concurrency](guide/parallelism-and-concurrency.md) -- [Retrying failing jobs](guide/retrying-failing-jobs.md) -- [Returning job data](guide/returning-job-data.md) -- [Events](guide/events/README.md) - - [Create Custom Events](guide/events/create-custom-events.md) -- [Telemetry](guide/telemetry/README.md) - - [Getting started](guide/telemetry/getting-started.md) - - [Running Jaeger](guide/telemetry/running-jaeger.md) - - [Running a simple example](guide/telemetry/running-a-simple-example.md) -- [QueueScheduler](guide/queuescheduler.md) -- [Redis™ Compatibility](guide/redis-tm-compatibility/README.md) - - [Dragonfly](guide/redis-tm-compatibility/dragonfly.md) -- [Redis™ hosting](guide/redis-tm-hosting/README.md) - - [AWS MemoryDB](guide/redis-tm-hosting/aws-memorydb.md) - - [AWS Elasticache](guide/redis-tm-hosting/aws-elasticache.md) -- [Architecture](guide/architecture.md) -- [NestJs](guide/nestjs/README.md) - - [Producers](guide/nestjs/producers.md) - - [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) -- [Going to production](guide/going-to-production.md) -- [Migration to newer versions](guide/migration-to-newer-versions.md) -- [Troubleshooting](guide/troubleshooting.md) +* [Introduction](guide/introduction.md) +* [Connections](guide/connections.md) +* [Queues](guide/queues/README.md) + * [Auto-removal of jobs](guide/queues/auto-removal-of-jobs.md) + * [Adding jobs in bulk](guide/queues/adding-bulks.md) + * [Global Concurrency](guide/queues/global-concurrency.md) + * [Removing Jobs](guide/queues/removing-jobs.md) +* [Workers](guide/workers/README.md) + * [Auto-removal of jobs](guide/workers/auto-removal-of-jobs.md) + * [Concurrency](guide/workers/concurrency.md) + * [Graceful shutdown](guide/workers/graceful-shutdown.md) + * [Stalled Jobs](guide/workers/stalled-jobs.md) + * [Sandboxed processors](guide/workers/sandboxed-processors.md) + * [Pausing queues](guide/workers/pausing-queues.md) +* [Jobs](guide/jobs/README.md) + * [FIFO](guide/jobs/fifo.md) + * [LIFO](guide/jobs/lifo.md) + * [Job Ids](guide/jobs/job-ids.md) + * [Job Data](guide/jobs/job-data.md) + * [Deduplication](guide/jobs/deduplication.md) + * [Delayed](guide/jobs/delayed.md) + * [Repeatable](guide/jobs/repeatable.md) + * [Prioritized](guide/jobs/prioritized.md) + * [Removing jobs](guide/jobs/removing-job.md) + * [Stalled](guide/jobs/stalled.md) + * [Getters](guide/jobs/getters.md) +* [Job Schedulers](guide/job-schedulers/README.md) + * [Repeat Strategies](guide/job-schedulers/repeat-strategies.md) + * [Repeat options](guide/job-schedulers/repeat-options.md) + * [Manage Job Schedulers](guide/job-schedulers/manage-job-schedulers.md) +* [Flows](guide/flows/README.md) + * [Adding flows in bulk](guide/flows/adding-bulks.md) + * [Get Flow Tree](guide/flows/get-flow-tree.md) + * [Fail Parent](guide/flows/fail-parent.md) + * [Remove Dependency](guide/flows/remove-dependency.md) + * [Ignore Dependency](guide/flows/ignore-dependency.md) + * [Remove Child Dependency](guide/flows/remove-child-dependency.md) +* [Metrics](guide/metrics/metrics.md) +* [Rate limiting](guide/rate-limiting.md) +* [Parallelism and Concurrency](guide/parallelism-and-concurrency.md) +* [Retrying failing jobs](guide/retrying-failing-jobs.md) +* [Returning job data](guide/returning-job-data.md) +* [Events](guide/events/README.md) + * [Create Custom Events](guide/events/create-custom-events.md) +* [Telemetry](guide/telemetry/README.md) + * [Getting started](guide/telemetry/getting-started.md) + * [Running Jaeger](guide/telemetry/running-jaeger.md) + * [Running a simple example](guide/telemetry/running-a-simple-example.md) +* [QueueScheduler](guide/queuescheduler.md) +* [Redis™ Compatibility](guide/redis-tm-compatibility/README.md) + * [Dragonfly](guide/redis-tm-compatibility/dragonfly.md) +* [Redis™ hosting](guide/redis-tm-hosting/README.md) + * [AWS MemoryDB](guide/redis-tm-hosting/aws-memorydb.md) + * [AWS Elasticache](guide/redis-tm-hosting/aws-elasticache.md) +* [Architecture](guide/architecture.md) +* [NestJs](guide/nestjs/README.md) + * [Producers](guide/nestjs/producers.md) + * [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) +* [Going to production](guide/going-to-production.md) +* [Migration to newer versions](guide/migration-to-newer-versions.md) +* [Troubleshooting](guide/troubleshooting.md) ## Patterns -- [Adding jobs in bulk across different queues](patterns/adding-bulks.md) -- [Manually processing jobs](patterns/manually-fetching-jobs.md) -- [Named Processor](patterns/named-processor.md) -- [Flows](patterns/flows.md) -- [Idempotent jobs](patterns/idempotent-jobs.md) -- [Throttle jobs](patterns/throttle-jobs.md) -- [Process Step Jobs](patterns/process-step-jobs.md) -- [Failing fast when Redis is down](patterns/failing-fast-when-redis-is-down.md) -- [Stop retrying jobs](patterns/stop-retrying-jobs.md) -- [Timeout jobs](patterns/timeout-jobs.md) -- [Redis Cluster](patterns/redis-cluster.md) +* [Adding jobs in bulk across different queues](patterns/adding-bulks.md) +* [Manually processing jobs](patterns/manually-fetching-jobs.md) +* [Named Processor](patterns/named-processor.md) +* [Flows](patterns/flows.md) +* [Idempotent jobs](patterns/idempotent-jobs.md) +* [Throttle jobs](patterns/throttle-jobs.md) +* [Process Step Jobs](patterns/process-step-jobs.md) +* [Failing fast when Redis is down](patterns/failing-fast-when-redis-is-down.md) +* [Stop retrying jobs](patterns/stop-retrying-jobs.md) +* [Timeout jobs](patterns/timeout-jobs.md) +* [Redis Cluster](patterns/redis-cluster.md) ## BullMQ Pro -- [Introduction](bullmq-pro/introduction.md) -- [Install](bullmq-pro/install.md) -- [Observables](bullmq-pro/observables/README.md) - - [Cancelation](bullmq-pro/observables/cancelation.md) -- [Groups](bullmq-pro/groups/README.md) - - [Getters](bullmq-pro/groups/getters.md) - - [Rate limiting](bullmq-pro/groups/rate-limiting.md) - - [Concurrency](bullmq-pro/groups/concurrency.md) - - [Local group concurrency](bullmq-pro/groups/local-group-concurrency.md) - - [Max group size](bullmq-pro/groups/max-group-size.md) - - [Pausing groups](bullmq-pro/groups/pausing-groups.md) - - [Prioritized intra-groups](bullmq-pro/groups/prioritized.md) - - [Sandboxes for groups](bullmq-pro/groups/sandboxes-for-groups.md) -- [Telemetry](bullmq-pro/telemetry.md) -- [Batches](bullmq-pro/batches.md) -- [NestJs](bullmq-pro/nestjs/README.md) - - [Producers](bullmq-pro/nestjs/producers.md) - - [Queue Events Listeners](bullmq-pro/nestjs/queue-events-listeners.md) - - [API Reference](https://nestjs.bullmq.pro/) - - [Changelog](bullmq-pro/nestjs/changelog.md) -- [API Reference](https://api.bullmq.pro) -- [Changelog](bullmq-pro/changelog.md) -- [Support](bullmq-pro/support.md) +* [Introduction](bullmq-pro/introduction.md) +* [Install](bullmq-pro/install.md) +* [Observables](bullmq-pro/observables/README.md) + * [Cancelation](bullmq-pro/observables/cancelation.md) +* [Groups](bullmq-pro/groups/README.md) + * [Getters](bullmq-pro/groups/getters.md) + * [Rate limiting](bullmq-pro/groups/rate-limiting.md) + * [Concurrency](bullmq-pro/groups/concurrency.md) + * [Local group concurrency](bullmq-pro/groups/local-group-concurrency.md) + * [Max group size](bullmq-pro/groups/max-group-size.md) + * [Pausing groups](bullmq-pro/groups/pausing-groups.md) + * [Prioritized intra-groups](bullmq-pro/groups/prioritized.md) + * [Sandboxes for groups](bullmq-pro/groups/sandboxes-for-groups.md) +* [Telemetry](bullmq-pro/telemetry.md) +* [Batches](bullmq-pro/batches.md) +* [NestJs](bullmq-pro/nestjs/README.md) + * [Producers](bullmq-pro/nestjs/producers.md) + * [Queue Events Listeners](bullmq-pro/nestjs/queue-events-listeners.md) + * [API Reference](https://nestjs.bullmq.pro/) + * [Changelog](bullmq-pro/nestjs/changelog.md) +* [API Reference](https://api.bullmq.pro) +* [Changelog](bullmq-pro/changelog.md) +* [New Releases](bullmq-pro/new-releases.md) +* [Support](bullmq-pro/support.md) ## Bull -- [Introduction](bull/introduction.md) -- [Install](bull/install.md) -- [Quick Guide](bull/quick-guide.md) -- [Important Notes](bull/important-notes.md) -- [Reference](https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md) -- [Patterns](bull/patterns/README.md) - - [Persistent connections](bull/patterns/persistent-connections.md) - - [Message queue](bull/patterns/message-queue.md) - - [Returning Job Completions](bull/patterns/returning-job-completions.md) - - [Reusing Redis Connections](bull/patterns/reusing-redis-connections.md) - - [Redis cluster](bull/patterns/redis-cluster.md) - - [Custom backoff strategy](bull/patterns/custom-backoff-strategy.md) - - [Debugging](bull/patterns/debugging.md) - - [Manually fetching jobs](bull/patterns/manually-fetching-jobs.md) +* [Introduction](bull/introduction.md) +* [Install](bull/install.md) +* [Quick Guide](bull/quick-guide.md) +* [Important Notes](bull/important-notes.md) +* [Reference](https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md) +* [Patterns](bull/patterns/README.md) + * [Persistent connections](bull/patterns/persistent-connections.md) + * [Message queue](bull/patterns/message-queue.md) + * [Returning Job Completions](bull/patterns/returning-job-completions.md) + * [Reusing Redis Connections](bull/patterns/reusing-redis-connections.md) + * [Redis cluster](bull/patterns/redis-cluster.md) + * [Custom backoff strategy](bull/patterns/custom-backoff-strategy.md) + * [Debugging](bull/patterns/debugging.md) + * [Manually fetching jobs](bull/patterns/manually-fetching-jobs.md) ## Python -- [Introduction](python/introduction.md) -- [Changelog](python/changelog.md) +* [Introduction](python/introduction.md) +* [Changelog](python/changelog.md) diff --git a/docs/gitbook/bullmq-pro/new-releases.md b/docs/gitbook/bullmq-pro/new-releases.md new file mode 100644 index 0000000000..c11fdbc7c7 --- /dev/null +++ b/docs/gitbook/bullmq-pro/new-releases.md @@ -0,0 +1,5 @@ +# New Releases + +If you want to get notifications when we do a new release of BullMQ Pro, please enable notifications on this Github issue where we automatically create a new comment for every new release: + +[https://github.com/taskforcesh/bullmq-pro-support/issues/86](https://github.com/taskforcesh/bullmq-pro-support/issues/86)