From 481b6643b5e3f57f686e63d84e6e2001d54192fe Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 11 Sep 2024 21:05:03 -0500 Subject: [PATCH 01/28] refactor: use isQueuePausedOrMaxed instead of getTargetQueueList --- src/commands/addStandardJob-8.lua | 9 +++++---- src/commands/changePriority-7.lua | 14 +++++++------- src/commands/includes/moveParentToWaitIfNeeded.lua | 8 +++----- .../includes/removeParentDependencyKey.lua | 7 +++---- src/commands/moveJobFromActiveToWait-10.lua | 8 ++++---- src/commands/moveJobsToWait-8.lua | 8 ++++---- src/commands/moveStalledJobsToWait-9.lua | 10 ++++------ src/commands/moveToActive-11.lua | 8 ++++---- src/commands/moveToFinished-14.lua | 8 ++++---- src/commands/promote-9.lua | 8 ++++---- src/commands/reprocessJob-8.lua | 8 ++++---- src/commands/retryJob-11.lua | 9 ++++----- 12 files changed, 50 insertions(+), 55 deletions(-) diff --git a/src/commands/addStandardJob-8.lua b/src/commands/addStandardJob-8.lua index 54fb363d73..15dd82e93d 100644 --- a/src/commands/addStandardJob-8.lua +++ b/src/commands/addStandardJob-8.lua @@ -16,7 +16,7 @@ Input: KEYS[1] 'wait', - KEYS[2] 'paused' + KEYS[2] 'paused' // TODO: remove KEYS[3] 'meta' KEYS[4] 'id' KEYS[5] 'completed' @@ -64,8 +64,8 @@ local parentData --- @include "includes/addJobInTargetList" --- @include "includes/debounceJob" --- @include "includes/getOrSetMaxEvents" ---- @include "includes/getTargetQueueList" --- @include "includes/handleDuplicatedJob" +--- @include "includes/isQueuePausedOrMaxed" --- @include "includes/storeJob" if parentKey ~= nil then @@ -76,6 +76,7 @@ end local jobCounter = rcall("INCR", KEYS[4]) +local waitKey = KEYS[1] local metaKey = KEYS[3] local maxEvents = getOrSetMaxEvents(metaKey) @@ -104,11 +105,11 @@ end storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp, parentKey, parentData, repeatJobKey) -local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[1], KEYS[2]) +local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[6]) -- LIFO or FIFO local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH' -addJobInTargetList(target, KEYS[8], pushCmd, isPausedOrMaxed, jobId) +addJobInTargetList(waitKey, KEYS[8], pushCmd, isPausedOrMaxed, jobId) -- Emit waiting event rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", diff --git a/src/commands/changePriority-7.lua b/src/commands/changePriority-7.lua index d23461edb1..f6e5c52f53 100644 --- a/src/commands/changePriority-7.lua +++ b/src/commands/changePriority-7.lua @@ -2,7 +2,7 @@ Change job priority Input: KEYS[1] 'wait', - KEYS[2] 'paused' + KEYS[2] 'paused' // TODO remove KEYS[3] 'meta' KEYS[4] 'prioritized' KEYS[5] 'active' @@ -26,14 +26,14 @@ local rcall = redis.call -- Includes --- @include "includes/addJobInTargetList" --- @include "includes/addJobWithPriority" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" --- @include "includes/pushBackJobWithPriority" -local function reAddJobWithNewPriority( prioritizedKey, markerKey, targetKey, +local function reAddJobWithNewPriority( prioritizedKey, markerKey, waitKey, priorityCounter, lifo, priority, jobId, isPausedOrMaxed) if priority == 0 then local pushCmd = lifo and 'RPUSH' or 'LPUSH' - addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId) + addJobInTargetList(waitKey, markerKey, pushCmd, isPausedOrMaxed, jobId) else if lifo then pushBackJobWithPriority(prioritizedKey, priority, jobId) @@ -46,17 +46,17 @@ end if rcall("EXISTS", jobKey) == 1 then local metaKey = KEYS[3] - local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[5], KEYS[1], KEYS[2]) + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[5]) local prioritizedKey = KEYS[4] local priorityCounterKey = KEYS[6] local markerKey = KEYS[7] -- Re-add with the new priority if rcall("ZREM", KEYS[4], jobId) > 0 then - reAddJobWithNewPriority( prioritizedKey, markerKey, target, + reAddJobWithNewPriority( prioritizedKey, markerKey, KEYS[1], priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed) elseif rcall("LREM", target, -1, jobId) > 0 then - reAddJobWithNewPriority( prioritizedKey, markerKey, target, + reAddJobWithNewPriority( prioritizedKey, markerKey, KEYS[1], priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed) end diff --git a/src/commands/includes/moveParentToWaitIfNeeded.lua b/src/commands/includes/moveParentToWaitIfNeeded.lua index dff73397ea..ca4ee7f97b 100644 --- a/src/commands/includes/moveParentToWaitIfNeeded.lua +++ b/src/commands/includes/moveParentToWaitIfNeeded.lua @@ -7,7 +7,6 @@ --- @include "addJobInTargetList" --- @include "addJobWithPriority" --- @include "isQueuePausedOrMaxed" ---- @include "getTargetQueueList" local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp) @@ -35,10 +34,9 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey) else if priority == 0 then - local parentTarget, isParentPausedOrMaxed = - getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey, - parentPausedKey) - addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, + local isParentPausedOrMaxed = + isQueuePausedOrMaxed(parentMetaKey, parentActiveKey) + addJobInTargetList(parentWaitKey, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId) else local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey) diff --git a/src/commands/includes/removeParentDependencyKey.lua b/src/commands/includes/removeParentDependencyKey.lua index 87262850c9..7e5b34ce90 100644 --- a/src/commands/includes/removeParentDependencyKey.lua +++ b/src/commands/includes/removeParentDependencyKey.lua @@ -7,13 +7,12 @@ -- Includes --- @include "addJobInTargetList" --- @include "destructureJobKey" ---- @include "getTargetQueueList" +--- @include "isQueuePausedOrMaxed" --- @include "removeJobKeys" local function moveParentToWait(parentPrefix, parentId, emitEvent) - local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active", - parentPrefix .. "wait", parentPrefix .. "paused") - addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId) + local isPausedOrMaxed = isQueuePausedOrMaxed(parentPrefix .. "meta", parentPrefix .. "active") + addJobInTargetList(parentPrefix .. "wait", parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId) if emitEvent then local parentEventStream = parentPrefix .. "events" diff --git a/src/commands/moveJobFromActiveToWait-10.lua b/src/commands/moveJobFromActiveToWait-10.lua index e90d6d2d10..52b2fcf228 100644 --- a/src/commands/moveJobFromActiveToWait-10.lua +++ b/src/commands/moveJobFromActiveToWait-10.lua @@ -6,7 +6,7 @@ KEYS[3] stalled key KEYS[4] job lock key - KEYS[5] paused key + KEYS[5] paused key // TODO remove KEYS[6] meta key KEYS[7] limiter key KEYS[8] prioritized key @@ -23,7 +23,7 @@ local rcall = redis.call --- @include "includes/addJobInTargetList" --- @include "includes/pushBackJobWithPriority" --- @include "includes/getOrSetMaxEvents" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" local jobId = ARGV[1] local token = ARGV[2] @@ -35,7 +35,7 @@ if lockToken == token then local metaKey = KEYS[6] local removed = rcall("LREM", KEYS[1], 1, jobId) if removed > 0 then - local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[1], KEYS[2], KEYS[5]) + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[1]) rcall("SREM", KEYS[3], jobId) @@ -44,7 +44,7 @@ if lockToken == token then if priority > 0 then pushBackJobWithPriority(KEYS[8], priority, jobId) else - addJobInTargetList(target, KEYS[9], "RPUSH", isPausedOrMaxed, jobId) + addJobInTargetList(KEYS[2], KEYS[9], "RPUSH", isPausedOrMaxed, jobId) end rcall("DEL", lockKey) diff --git a/src/commands/moveJobsToWait-8.lua b/src/commands/moveJobsToWait-8.lua index 15e99c6295..9a69fc93e9 100644 --- a/src/commands/moveJobsToWait-8.lua +++ b/src/commands/moveJobsToWait-8.lua @@ -8,7 +8,7 @@ KEYS[2] events stream KEYS[3] state key (failed, completed, delayed) KEYS[4] 'wait' - KEYS[5] 'paused' + KEYS[5] 'paused' // TODO remove KEYS[6] 'meta' KEYS[7] 'active' KEYS[8] 'marker' @@ -30,10 +30,10 @@ local rcall = redis.call; --- @include "includes/addBaseMarkerIfNeeded" --- @include "includes/batches" --- @include "includes/getOrSetMaxEvents" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" local metaKey = KEYS[6] -local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5]) +local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[7]) local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount) if (#jobs > 0) then @@ -60,7 +60,7 @@ if (#jobs > 0) then for from, to in batches(#jobs, 7000) do rcall("ZREM", KEYS[3], unpack(jobs, from, to)) - rcall("LPUSH", target, unpack(jobs, from, to)) + rcall("LPUSH", KEYS[4], unpack(jobs, from, to)) end addBaseMarkerIfNeeded(KEYS[8], isPausedOrMaxed) diff --git a/src/commands/moveStalledJobsToWait-9.lua b/src/commands/moveStalledJobsToWait-9.lua index c40d85fa0e..da06c41f8b 100644 --- a/src/commands/moveStalledJobsToWait-9.lua +++ b/src/commands/moveStalledJobsToWait-9.lua @@ -8,7 +8,7 @@ KEYS[4] 'failed', (ZSET) KEYS[5] 'stalled-check', (KEY) KEYS[6] 'meta', (KEY) - KEYS[7] 'paused', (LIST) + KEYS[7] 'paused', (LIST) // TODO remove KEYS[8] 'marker' KEYS[9] 'event stream' (STREAM) @@ -26,7 +26,7 @@ local rcall = redis.call -- Includes --- @include "includes/addJobInTargetList" --- @include "includes/batches" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" --- @include "includes/moveParentIfNeeded" --- @include "includes/removeDebounceKeyIfNeeded" --- @include "includes/removeJob" @@ -40,7 +40,6 @@ local activeKey = KEYS[3] local failedKey = KEYS[4] local stalledCheckKey = KEYS[5] local metaKey = KEYS[6] -local pausedKey = KEYS[7] local markerKey = KEYS[8] local eventStreamKey = KEYS[9] local maxStalledJobCount = ARGV[1] @@ -126,11 +125,10 @@ if (#stalling > 0) then table.insert(failed, jobId) else - local target, isPausedOrMaxed= - getTargetQueueList(metaKey, activeKey, waitKey, pausedKey) + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, activeKey) -- Move the job back to the wait queue, to immediately be picked up by a waiting worker. - addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId) + addJobInTargetList(waitKey, markerKey, "RPUSH", isPausedOrMaxed, jobId) rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active') diff --git a/src/commands/moveToActive-11.lua b/src/commands/moveToActive-11.lua index 35c63db7b2..2f8535345f 100644 --- a/src/commands/moveToActive-11.lua +++ b/src/commands/moveToActive-11.lua @@ -18,7 +18,7 @@ KEYS[7] delayed key -- Delayed jobs - KEYS[8] paused key + KEYS[8] paused key // TODO remove KEYS[9] meta key KEYS[10] pc priority counter @@ -45,16 +45,16 @@ local opts = cmsgpack.unpack(ARGV[3]) -- Includes --- @include "includes/getNextDelayedTimestamp" --- @include "includes/getRateLimitTTL" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" --- @include "includes/moveJobFromPriorityToActive" --- @include "includes/prepareJobForProcessing" --- @include "includes/promoteDelayedJobs" -local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8]) +local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[9], activeKey) -- Check if there are delayed jobs that we can move to wait. local markerKey = KEYS[11] -promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1], +promoteDelayedJobs(delayedKey, markerKey, KEYS[1], KEYS[3], eventStreamKey, ARGV[1], ARGV[2], KEYS[10], isPausedOrMaxed) local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max']) diff --git a/src/commands/moveToFinished-14.lua b/src/commands/moveToFinished-14.lua index 29014bbf0b..50e857b885 100644 --- a/src/commands/moveToFinished-14.lua +++ b/src/commands/moveToFinished-14.lua @@ -15,7 +15,7 @@ KEYS[6] rate limiter key KEYS[7] delayed key - KEYS[8] paused key + KEYS[8] paused key // TODO remove KEYS[9] meta key KEYS[10] pc priority counter @@ -56,7 +56,7 @@ local rcall = redis.call --- @include "includes/collectMetrics" --- @include "includes/getNextDelayedTimestamp" --- @include "includes/getRateLimitTTL" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" --- @include "includes/moveJobFromPriorityToActive" --- @include "includes/moveParentIfNeeded" --- @include "includes/prepareJobForProcessing" @@ -180,10 +180,10 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists -- and not rate limited. if (ARGV[6] == "1") then - local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8]) + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[2]) -- Check if there are delayed jobs that can be promoted - promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, prefix, + promoteDelayedJobs(KEYS[7], KEYS[14], KEYS[1], KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10], isPausedOrMaxed) local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max']) diff --git a/src/commands/promote-9.lua b/src/commands/promote-9.lua index f0a273f0d7..a933a83f4d 100644 --- a/src/commands/promote-9.lua +++ b/src/commands/promote-9.lua @@ -4,7 +4,7 @@ Input: KEYS[1] 'delayed' KEYS[2] 'wait' - KEYS[3] 'paused' + KEYS[3] 'paused' // TODO remove KEYS[4] 'meta' KEYS[5] 'prioritized' KEYS[6] 'active' @@ -28,7 +28,7 @@ local jobId = ARGV[2] -- Includes --- @include "includes/addJobInTargetList" --- @include "includes/addJobWithPriority" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" if rcall("ZREM", KEYS[1], jobId) == 1 then local jobKey = ARGV[1] .. jobId @@ -36,11 +36,11 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then local metaKey = KEYS[4] local markerKey = KEYS[9] - local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[2], KEYS[3]) + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[6]) if priority == 0 then -- LIFO or FIFO - addJobInTargetList(target, markerKey, "LPUSH", isPausedOrMaxed, jobId) + addJobInTargetList(KEYS[2], markerKey, "LPUSH", isPausedOrMaxed, jobId) else addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed) end diff --git a/src/commands/reprocessJob-8.lua b/src/commands/reprocessJob-8.lua index 300ab6a1e8..ccb48eefeb 100644 --- a/src/commands/reprocessJob-8.lua +++ b/src/commands/reprocessJob-8.lua @@ -7,7 +7,7 @@ KEYS[3] job state KEYS[4] wait key KEYS[5] meta - KEYS[6] paused key + KEYS[6] paused key // TODO remove KEYS[7] active key KEYS[8] marker key @@ -26,15 +26,15 @@ local rcall = redis.call; -- Includes --- @include "includes/addJobInTargetList" --- @include "includes/getOrSetMaxEvents" ---- @include "includes/getTargetQueueList" +--- @include "includes/isQueuePausedOrMaxed" if rcall("EXISTS", KEYS[1]) == 1 then local jobId = ARGV[1] if (rcall("ZREM", KEYS[3], jobId) == 1) then rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3]) - local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[7], KEYS[4], KEYS[6]) - addJobInTargetList(target, KEYS[8], ARGV[2], isPausedOrMaxed, jobId) + local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[7]) + addJobInTargetList(KEYS[4], KEYS[8], ARGV[2], isPausedOrMaxed, jobId) local maxEvents = getOrSetMaxEvents(KEYS[5]) -- Emit waiting event diff --git a/src/commands/retryJob-11.lua b/src/commands/retryJob-11.lua index 33d1f7a85a..9acf0b5266 100644 --- a/src/commands/retryJob-11.lua +++ b/src/commands/retryJob-11.lua @@ -4,7 +4,7 @@ Input: KEYS[1] 'active', KEYS[2] 'wait' - KEYS[3] 'paused' + KEYS[3] 'paused' // TODO remove KEYS[4] job key KEYS[5] 'meta' KEYS[6] events stream @@ -35,17 +35,16 @@ local rcall = redis.call --- @include "includes/addJobInTargetList" --- @include "includes/addJobWithPriority" --- @include "includes/getOrSetMaxEvents" ---- @include "includes/getTargetQueueList" --- @include "includes/promoteDelayedJobs" --- @include "includes/removeLock" --- @include "includes/isQueuePausedOrMaxed" -local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[1], KEYS[2], KEYS[3]) +local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1]) local markerKey = KEYS[10] -- Check if there are delayed jobs that we can move to wait. -- test example: when there are delayed jobs between retries -promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed) +promoteDelayedJobs(KEYS[7], markerKey, KEYS[2], KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed) if rcall("EXISTS", KEYS[4]) == 1 then local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4]) @@ -63,7 +62,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then -- Standard or priority add if priority == 0 then - addJobInTargetList(target, markerKey, ARGV[3], isPausedOrMaxed, ARGV[4]) + addJobInTargetList(KEYS[2], markerKey, ARGV[3], isPausedOrMaxed, ARGV[4]) else addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed) end From 2654bb89b25e2968f0ecd7ac6f9fd2fb1f30eb62 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 00:05:45 -0500 Subject: [PATCH 02/28] chore: consider moving jobs from paused to wait if there are --- src/commands/changePriority-7.lua | 2 +- src/commands/getCountsPerPriority-4.lua | 6 +++--- .../includes/getWaitPlusPrioritizedCount.lua | 11 +++++++++++ src/commands/pause-7.lua | 16 ++++++++++++---- 4 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 src/commands/includes/getWaitPlusPrioritizedCount.lua diff --git a/src/commands/changePriority-7.lua b/src/commands/changePriority-7.lua index f6e5c52f53..d6e1919598 100644 --- a/src/commands/changePriority-7.lua +++ b/src/commands/changePriority-7.lua @@ -55,7 +55,7 @@ if rcall("EXISTS", jobKey) == 1 then if rcall("ZREM", KEYS[4], jobId) > 0 then reAddJobWithNewPriority( prioritizedKey, markerKey, KEYS[1], priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed) - elseif rcall("LREM", target, -1, jobId) > 0 then + elseif rcall("LREM", KEYS[1], -1, jobId) > 0 then reAddJobWithNewPriority( prioritizedKey, markerKey, KEYS[1], priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed) end diff --git a/src/commands/getCountsPerPriority-4.lua b/src/commands/getCountsPerPriority-4.lua index fa24ba2328..60f6a90d4c 100644 --- a/src/commands/getCountsPerPriority-4.lua +++ b/src/commands/getCountsPerPriority-4.lua @@ -21,10 +21,10 @@ local prioritizedKey = KEYS[4] for i = 1, #ARGV do local priority = tonumber(ARGV[i]) if priority == 0 then - if isQueuePaused(KEYS[3]) then - results[#results+1] = rcall("LLEN", pausedKey) + results[#results+1] = rcall("LLEN", waitKey) + if isQueuePaused(KEYS[3]) then -- TODO: remove in next breaking change + results[#results] = results[#results] + rcall("LLEN", pausedKey) else - results[#results+1] = rcall("LLEN", waitKey) end else results[#results+1] = rcall("ZCOUNT", prioritizedKey, diff --git a/src/commands/includes/getWaitPlusPrioritizedCount.lua b/src/commands/includes/getWaitPlusPrioritizedCount.lua new file mode 100644 index 0000000000..019fcbed87 --- /dev/null +++ b/src/commands/includes/getWaitPlusPrioritizedCount.lua @@ -0,0 +1,11 @@ +--[[ + Get count jobs in wait or prioritized. +]] + +local function getWaitPlusPrioritizedCount(waitKey, prioritizedKey) + local waitCount = rcall("LLEN", waitKey) + local prioritizedCount = rcall("ZCARD", prioritizedKey) + + return waitCount + prioritizedCount +end + \ No newline at end of file diff --git a/src/commands/pause-7.lua b/src/commands/pause-7.lua index 90d0f945f7..880494ef67 100644 --- a/src/commands/pause-7.lua +++ b/src/commands/pause-7.lua @@ -19,19 +19,27 @@ local rcall = redis.call -- Includes --- @include "includes/addDelayMarkerIfNeeded" +--- @include "includes/getWaitPlusPrioritizedCount" local markerKey = KEYS[7] -local hasJobs = rcall("EXISTS", KEYS[1]) == 1 ---TODO: check this logic to be reused when changing a delay -if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end if ARGV[1] == "paused" then rcall("HSET", KEYS[3], "paused", 1) rcall("DEL", markerKey) else rcall("HDEL", KEYS[3], "paused") + --jobs in paused key + local hasJobs = rcall("EXISTS", KEYS[1]) == 1 + + if hasJobs then + --move a maximum of 7000 per resumed call in order to not block + --if users have more jobs in paused state, call resumed multiple times + local jobs = rcall('LRANGE', KEYS[1], 0, 6999) + rcall("LPUSH", KEYS[2], unpack(jobs)) + rcall("LTRIM", KEYS[1], #jobs, -1) + end - if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then + if getWaitPlusPrioritizedCount(KEYS[2], KEYS[4]) > 0 then -- Add marker if there are waiting or priority jobs rcall("ZADD", markerKey, 0, "0") else From 9286d688c98eb3195567c5b8d71bb7879ccc3fe2 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 19:41:45 -0500 Subject: [PATCH 03/28] refactor: obliterate to consider removing from wait --- src/commands/obliterate-2.lua | 2 +- tests/test_job.ts | 4 ++-- tests/test_pause.ts | 7 +++---- tests/test_queue.ts | 12 ++++++------ tests/test_rate_limiter.ts | 11 +++++------ tests/test_worker.ts | 4 ++-- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/commands/obliterate-2.lua b/src/commands/obliterate-2.lua index 1a7be36393..7814f12dbe 100644 --- a/src/commands/obliterate-2.lua +++ b/src/commands/obliterate-2.lua @@ -81,7 +81,7 @@ if(maxCount <= 0) then return 1 end -local waitKey = baseKey .. 'paused' +local waitKey = baseKey .. 'wait' maxCount = removeListJobs(waitKey, true, baseKey, maxCount) if(maxCount <= 0) then return 1 diff --git a/tests/test_job.ts b/tests/test_job.ts index 01c52232fc..0dffe8319f 100644 --- a/tests/test_job.ts +++ b/tests/test_job.ts @@ -1366,7 +1366,7 @@ describe('Job', function () { await queue.pause(); await delayedJob.promote(); - const pausedJobsCount = await queue.getJobCountByTypes('paused'); + const pausedJobsCount = await queue.getWaitingCount(); expect(pausedJobsCount).to.be.equal(2); await queue.resume(); @@ -1388,7 +1388,7 @@ describe('Job', function () { await queue.pause(); await delayedJob.promote(); - const pausedJobsCount = await queue.getJobCountByTypes('paused'); + const pausedJobsCount = await queue.getWaitingCount(); expect(pausedJobsCount).to.be.equal(1); await queue.resume(); diff --git a/tests/test_pause.ts b/tests/test_pause.ts index 9ad1f6e5c5..38e809da18 100644 --- a/tests/test_pause.ts +++ b/tests/test_pause.ts @@ -58,9 +58,8 @@ describe('Pause', function () { if (processed) { throw new Error('should not process delayed jobs in paused queue.'); } - const counts2 = await queue.getJobCounts('waiting', 'paused', 'delayed'); - expect(counts2).to.have.property('waiting', 0); - expect(counts2).to.have.property('paused', 1); + const counts2 = await queue.getJobCounts('waiting', 'delayed'); + expect(counts2).to.have.property('waiting', 1); expect(counts2).to.have.property('delayed', 0); await worker.close(); @@ -380,7 +379,7 @@ describe('Pause', function () { const waitingEvent = new Promise(resolve => { queueEvents.on('waiting', async ({ prev }) => { if (prev === 'failed') { - const count = await queue.getJobCountByTypes('paused'); + const count = await queue.getJobCountByTypes('wait'); expect(count).to.be.equal(1); await queue.resume(); resolve(); diff --git a/tests/test_queue.ts b/tests/test_queue.ts index 91e05328f7..0a2b29d5e2 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -389,7 +389,7 @@ describe('queues', function () { describe('when queue is paused', () => { it('clean queue including paused jobs', async () => { const maxJobs = 50; - const added = []; + const added:Promise[] = []; await queue.pause(); for (let i = 1; i <= maxJobs; i++) { @@ -399,8 +399,8 @@ describe('queues', function () { await Promise.all(added); const count = await queue.count(); expect(count).to.be.eql(maxJobs); - const count2 = await queue.getJobCounts('paused'); - expect(count2.paused).to.be.eql(maxJobs); + const count2 = await queue.getJobCounts('wait'); + expect(count2.wait).to.be.eql(maxJobs); await queue.drain(); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); @@ -618,7 +618,7 @@ describe('queues', function () { }); describe('when queue is paused', () => { - it('moves retried jobs to paused', async () => { + it('moves retried jobs to wait', async () => { await queue.waitUntilReady(); const jobCount = 8; @@ -661,8 +661,8 @@ describe('queues', function () { await queue.pause(); await queue.retryJobs({ count: 2 }); - const pausedCount = await queue.getJobCounts('paused'); - expect(pausedCount.paused).to.be.equal(jobCount); + const pausedCount = await queue.getJobCounts('wait'); + expect(pausedCount.wait).to.be.equal(jobCount); await worker.close(); }); diff --git a/tests/test_rate_limiter.ts b/tests/test_rate_limiter.ts index bb25a9d53c..aa7fdf606f 100644 --- a/tests/test_rate_limiter.ts +++ b/tests/test_rate_limiter.ts @@ -137,7 +137,7 @@ describe('Rate Limiter', function () { }); describe('when queue is paused between rate limit', () => { - it('should add active jobs to paused', async function () { + it('should add active jobs to wait', async function () { this.timeout(20000); const numJobs = 4; @@ -184,10 +184,9 @@ describe('Rate Limiter', function () { await delay(500); - const counts = await queue.getJobCounts('paused', 'completed', 'wait'); - expect(counts).to.have.property('paused', numJobs - 1); + const counts = await queue.getJobCounts('completed', 'wait'); expect(counts).to.have.property('completed', 1); - expect(counts).to.have.property('wait', 0); + expect(counts).to.have.property('wait', numJobs - 1); await worker1.close(); await worker2.close(); @@ -797,7 +796,7 @@ describe('Rate Limiter', function () { }); describe('when queue is paused', () => { - it('moves job to paused', async function () { + it('moves job to wait', async function () { const dynamicLimit = 250; const duration = 100; @@ -839,7 +838,7 @@ describe('Rate Limiter', function () { await result; - const pausedCount = await queue.getJobCountByTypes('paused'); + const pausedCount = await queue.getJobCountByTypes('wait'); expect(pausedCount).to.equal(1); await worker.close(); diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 0509cf9a28..4d482f893b 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -1859,7 +1859,7 @@ describe('workers', function () { }); describe('when queue is paused and retry a job', () => { - it('moves job to paused', async () => { + it('moves job to wait', async () => { const worker = new Worker( queueName, async () => { @@ -1889,7 +1889,7 @@ describe('workers', function () { await queue.pause(); await job.retry('completed'); - const pausedJobsCount = await queue.getJobCountByTypes('paused'); + const pausedJobsCount = await queue.getJobCountByTypes('wait'); expect(pausedJobsCount).to.be.equal(1); await worker.close(); From 1f2f88953d55fcc39e234fd350df1bf9114d2382 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 19:46:12 -0500 Subject: [PATCH 04/28] refactor: remove paused key from addStandardJob --- python/bullmq/scripts.py | 4 +-- src/classes/scripts.ts | 1 - ...StandardJob-8.lua => addStandardJob-7.lua} | 25 +++++++++---------- 3 files changed, 14 insertions(+), 16 deletions(-) rename src/commands/{addStandardJob-8.lua => addStandardJob-7.lua} (86%) diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index 68879740f3..85d4a5992d 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -38,7 +38,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection self.redisConnection = redisConnection self.redisClient = redisConnection.conn self.commands = { - "addStandardJob": self.redisClient.register_script(self.getScript("addStandardJob-8.lua")), + "addStandardJob": self.redisClient.register_script(self.getScript("addStandardJob-7.lua")), "addDelayedJob": self.redisClient.register_script(self.getScript("addDelayedJob-6.lua")), "addParentJob": self.redisClient.register_script(self.getScript("addParentJob-4.lua")), "addPrioritizedJob": self.redisClient.register_script(self.getScript("addPrioritizedJob-8.lua")), @@ -131,7 +131,7 @@ def addStandardJob(self, job: Job, timestamp: int, pipe = None): """ Add a standard job to the queue """ - keys = self.getKeys(['wait', 'paused', 'meta', 'id', + keys = self.getKeys(['wait', 'meta', 'id', 'completed', 'active', 'events', 'marker']) args = self.addJobArgs(job, None) args.append(timestamp) diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index de4fd6b091..fe9f11fdbd 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -153,7 +153,6 @@ export class Scripts { const queueKeys = this.queue.keys; const keys: (string | Buffer)[] = [ queueKeys.wait, - queueKeys.paused, queueKeys.meta, queueKeys.id, queueKeys.completed, diff --git a/src/commands/addStandardJob-8.lua b/src/commands/addStandardJob-7.lua similarity index 86% rename from src/commands/addStandardJob-8.lua rename to src/commands/addStandardJob-7.lua index 15dd82e93d..17f29e94f6 100644 --- a/src/commands/addStandardJob-8.lua +++ b/src/commands/addStandardJob-7.lua @@ -16,13 +16,12 @@ Input: KEYS[1] 'wait', - KEYS[2] 'paused' // TODO: remove - KEYS[3] 'meta' - KEYS[4] 'id' - KEYS[5] 'completed' - KEYS[6] 'active' - KEYS[7] events stream key - KEYS[8] marker key + KEYS[2] 'meta' + KEYS[3] 'id' + KEYS[4] 'completed' + KEYS[5] 'active' + KEYS[6] events stream key + KEYS[7] marker key ARGV[1] msgpacked arguments array [1] key prefix, @@ -43,7 +42,7 @@ jobId - OK -5 - Missing parent key ]] -local eventsKey = KEYS[7] +local eventsKey = KEYS[6] local jobId local jobIdKey @@ -74,10 +73,10 @@ if parentKey ~= nil then parentData = cjson.encode(parent) end -local jobCounter = rcall("INCR", KEYS[4]) +local jobCounter = rcall("INCR", KEYS[3]) local waitKey = KEYS[1] -local metaKey = KEYS[3] +local metaKey = KEYS[2] local maxEvents = getOrSetMaxEvents(metaKey) local parentDependenciesKey = args[7] @@ -90,7 +89,7 @@ else jobIdKey = args[1] .. jobId if rcall("EXISTS", jobIdKey) == 1 then return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent, - parentData, parentDependenciesKey, KEYS[5], eventsKey, + parentData, parentDependenciesKey, KEYS[4], eventsKey, maxEvents, timestamp) end end @@ -105,11 +104,11 @@ end storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp, parentKey, parentData, repeatJobKey) -local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[6]) +local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[5]) -- LIFO or FIFO local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH' -addJobInTargetList(waitKey, KEYS[8], pushCmd, isPausedOrMaxed, jobId) +addJobInTargetList(waitKey, KEYS[7], pushCmd, isPausedOrMaxed, jobId) -- Emit waiting event rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", From a66ddba8a08377cf3379c601f0b084a733439518 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 19:56:06 -0500 Subject: [PATCH 05/28] refactor: remove paused key scripts --- python/bullmq/scripts.py | 6 ++--- src/classes/scripts.ts | 3 --- ...ngePriority-7.lua => changePriority-6.lua} | 23 +++++++++---------- ...t-10.lua => moveJobFromActiveToWait-9.lua} | 21 ++++++++--------- ...{reprocessJob-8.lua => reprocessJob-7.lua} | 9 ++++---- 5 files changed, 27 insertions(+), 35 deletions(-) rename src/commands/{changePriority-7.lua => changePriority-6.lua} (79%) rename src/commands/{moveJobFromActiveToWait-10.lua => moveJobFromActiveToWait-9.lua} (71%) rename src/commands/{reprocessJob-8.lua => reprocessJob-7.lua} (83%) diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index 85d4a5992d..f8e7115ec2 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -42,7 +42,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "addDelayedJob": self.redisClient.register_script(self.getScript("addDelayedJob-6.lua")), "addParentJob": self.redisClient.register_script(self.getScript("addParentJob-4.lua")), "addPrioritizedJob": self.redisClient.register_script(self.getScript("addPrioritizedJob-8.lua")), - "changePriority": self.redisClient.register_script(self.getScript("changePriority-7.lua")), + "changePriority": self.redisClient.register_script(self.getScript("changePriority-6.lua")), "cleanJobsInSet": self.redisClient.register_script(self.getScript("cleanJobsInSet-2.lua")), "extendLock": self.redisClient.register_script(self.getScript("extendLock-2.lua")), "getCounts": self.redisClient.register_script(self.getScript("getCounts-1.lua")), @@ -60,7 +60,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "pause": self.redisClient.register_script(self.getScript("pause-7.lua")), "promote": self.redisClient.register_script(self.getScript("promote-9.lua")), "removeJob": self.redisClient.register_script(self.getScript("removeJob-2.lua")), - "reprocessJob": self.redisClient.register_script(self.getScript("reprocessJob-8.lua")), + "reprocessJob": self.redisClient.register_script(self.getScript("reprocessJob-7.lua")), "retryJob": self.redisClient.register_script(self.getScript("retryJob-11.lua")), "moveJobsToWait": self.redisClient.register_script(self.getScript("moveJobsToWait-8.lua")), "saveStacktrace": self.redisClient.register_script(self.getScript("saveStacktrace-1.lua")), @@ -373,7 +373,6 @@ async def isJobInList(self, list_key: str, job_id: str): async def changePriority(self, job_id: str, priority:int = 0, lifo:bool = False): keys = [self.keys['wait'], - self.keys['paused'], self.keys['meta'], self.keys['prioritized'], self.keys['active'], @@ -407,7 +406,6 @@ async def reprocessJob(self, job: Job, state: str): keys.append(self.keys[state]) keys.append(self.keys['wait']) keys.append(self.keys['meta']) - keys.append(self.keys['paused']) keys.append(self.keys['active']) keys.append(self.keys['marker']) diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index fe9f11fdbd..b1bc385e25 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -879,7 +879,6 @@ export class Scripts { ): (string | number)[] { const keys: (string | number)[] = [ this.queue.keys.wait, - this.queue.keys.paused, this.queue.keys.meta, this.queue.keys.prioritized, this.queue.keys.active, @@ -1163,7 +1162,6 @@ export class Scripts { this.queue.toKey(state), this.queue.keys.wait, this.queue.keys.meta, - this.queue.keys.paused, this.queue.keys.active, this.queue.keys.marker, ]; @@ -1312,7 +1310,6 @@ export class Scripts { this.queue.keys.wait, this.queue.keys.stalled, lockKey, - this.queue.keys.paused, this.queue.keys.meta, this.queue.keys.limiter, this.queue.keys.prioritized, diff --git a/src/commands/changePriority-7.lua b/src/commands/changePriority-6.lua similarity index 79% rename from src/commands/changePriority-7.lua rename to src/commands/changePriority-6.lua index d6e1919598..94048a41b4 100644 --- a/src/commands/changePriority-7.lua +++ b/src/commands/changePriority-6.lua @@ -2,12 +2,11 @@ Change job priority Input: KEYS[1] 'wait', - KEYS[2] 'paused' // TODO remove - KEYS[3] 'meta' - KEYS[4] 'prioritized' - KEYS[5] 'active' - KEYS[6] 'pc' priority counter - KEYS[7] 'marker' + KEYS[2] 'meta' + KEYS[3] 'prioritized' + KEYS[4] 'active' + KEYS[5] 'pc' priority counter + KEYS[6] 'marker' ARGV[1] priority value ARGV[2] job key @@ -45,14 +44,14 @@ local function reAddJobWithNewPriority( prioritizedKey, markerKey, waitKey, end if rcall("EXISTS", jobKey) == 1 then - local metaKey = KEYS[3] - local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[5]) - local prioritizedKey = KEYS[4] - local priorityCounterKey = KEYS[6] - local markerKey = KEYS[7] + local metaKey = KEYS[2] + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[4]) + local prioritizedKey = KEYS[3] + local priorityCounterKey = KEYS[5] + local markerKey = KEYS[6] -- Re-add with the new priority - if rcall("ZREM", KEYS[4], jobId) > 0 then + if rcall("ZREM", KEYS[3], jobId) > 0 then reAddJobWithNewPriority( prioritizedKey, markerKey, KEYS[1], priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed) elseif rcall("LREM", KEYS[1], -1, jobId) > 0 then diff --git a/src/commands/moveJobFromActiveToWait-10.lua b/src/commands/moveJobFromActiveToWait-9.lua similarity index 71% rename from src/commands/moveJobFromActiveToWait-10.lua rename to src/commands/moveJobFromActiveToWait-9.lua index 52b2fcf228..f340021ce7 100644 --- a/src/commands/moveJobFromActiveToWait-10.lua +++ b/src/commands/moveJobFromActiveToWait-9.lua @@ -6,12 +6,11 @@ KEYS[3] stalled key KEYS[4] job lock key - KEYS[5] paused key // TODO remove - KEYS[6] meta key - KEYS[7] limiter key - KEYS[8] prioritized key - KEYS[9] marker key - KEYS[10] event key + KEYS[5] meta key + KEYS[6] limiter key + KEYS[7] prioritized key + KEYS[8] marker key + KEYS[9] event key ARGV[1] job id ARGV[2] lock token @@ -30,9 +29,9 @@ local token = ARGV[2] local lockKey = KEYS[4] local lockToken = rcall("GET", lockKey) -local pttl = rcall("PTTL", KEYS[7]) +local pttl = rcall("PTTL", KEYS[6]) if lockToken == token then - local metaKey = KEYS[6] + local metaKey = KEYS[5] local removed = rcall("LREM", KEYS[1], 1, jobId) if removed > 0 then local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[1]) @@ -42,9 +41,9 @@ if lockToken == token then local priority = tonumber(rcall("HGET", ARGV[3], "priority")) or 0 if priority > 0 then - pushBackJobWithPriority(KEYS[8], priority, jobId) + pushBackJobWithPriority(KEYS[7], priority, jobId) else - addJobInTargetList(KEYS[2], KEYS[9], "RPUSH", isPausedOrMaxed, jobId) + addJobInTargetList(KEYS[2], KEYS[8], "RPUSH", isPausedOrMaxed, jobId) end rcall("DEL", lockKey) @@ -52,7 +51,7 @@ if lockToken == token then local maxEvents = getOrSetMaxEvents(metaKey) -- Emit waiting event - rcall("XADD", KEYS[10], "MAXLEN", "~", maxEvents, "*", "event", "waiting", + rcall("XADD", KEYS[9], "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", jobId) end end diff --git a/src/commands/reprocessJob-8.lua b/src/commands/reprocessJob-7.lua similarity index 83% rename from src/commands/reprocessJob-8.lua rename to src/commands/reprocessJob-7.lua index ccb48eefeb..f393ede5da 100644 --- a/src/commands/reprocessJob-8.lua +++ b/src/commands/reprocessJob-7.lua @@ -7,9 +7,8 @@ KEYS[3] job state KEYS[4] wait key KEYS[5] meta - KEYS[6] paused key // TODO remove - KEYS[7] active key - KEYS[8] marker key + KEYS[6] active key + KEYS[7] marker key ARGV[1] job.id ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH' @@ -33,8 +32,8 @@ if rcall("EXISTS", KEYS[1]) == 1 then if (rcall("ZREM", KEYS[3], jobId) == 1) then rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3]) - local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[7]) - addJobInTargetList(KEYS[4], KEYS[8], ARGV[2], isPausedOrMaxed, jobId) + local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[6]) + addJobInTargetList(KEYS[4], KEYS[7], ARGV[2], isPausedOrMaxed, jobId) local maxEvents = getOrSetMaxEvents(KEYS[5]) -- Emit waiting event From 87447a88fd395143754e30a3ba069930f1d5128a Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 20:08:51 -0500 Subject: [PATCH 06/28] refactor: remove paused key from other scripts --- python/bullmq/scripts.py | 9 ++--- src/classes/scripts.ts | 4 +- src/commands/{promote-9.lua => promote-8.lua} | 23 ++++++------ .../{retryJob-11.lua => retryJob-10.lua} | 37 ++++++++++--------- 4 files changed, 35 insertions(+), 38 deletions(-) rename src/commands/{promote-9.lua => promote-8.lua} (66%) rename src/commands/{retryJob-11.lua => retryJob-10.lua} (60%) diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index f8e7115ec2..21084f8f16 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -58,10 +58,10 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "moveToWaitingChildren": self.redisClient.register_script(self.getScript("moveToWaitingChildren-5.lua")), "obliterate": self.redisClient.register_script(self.getScript("obliterate-2.lua")), "pause": self.redisClient.register_script(self.getScript("pause-7.lua")), - "promote": self.redisClient.register_script(self.getScript("promote-9.lua")), + "promote": self.redisClient.register_script(self.getScript("promote-8.lua")), "removeJob": self.redisClient.register_script(self.getScript("removeJob-2.lua")), "reprocessJob": self.redisClient.register_script(self.getScript("reprocessJob-7.lua")), - "retryJob": self.redisClient.register_script(self.getScript("retryJob-11.lua")), + "retryJob": self.redisClient.register_script(self.getScript("retryJob-10.lua")), "moveJobsToWait": self.redisClient.register_script(self.getScript("moveJobsToWait-8.lua")), "saveStacktrace": self.redisClient.register_script(self.getScript("saveStacktrace-1.lua")), "updateData": self.redisClient.register_script(self.getScript("updateData-1.lua")), @@ -258,15 +258,15 @@ def saveStacktraceArgs(self, job_id: str, stacktrace: str, failedReason: str): return (keys, args) def retryJobArgs(self, job_id: str, lifo: bool, token: str, opts: dict = {}): - keys = self.getKeys(['active', 'wait', 'paused']) + keys = self.getKeys(['active', 'wait']) keys.append(self.toKey(job_id)) keys.append(self.keys['meta']) keys.append(self.keys['events']) keys.append(self.keys['delayed']) keys.append(self.keys['prioritized']) keys.append(self.keys['pc']) - keys.append(self.keys['marker']) keys.append(self.keys['stalled']) + keys.append(self.keys['marker']) push_cmd = "RPUSH" if lifo else "LPUSH" @@ -301,7 +301,6 @@ def promoteArgs(self, job_id: str): keys = self.getKeys(['delayed', 'wait', 'paused', 'meta', 'prioritized', 'active', 'pc', 'events', 'marker']) keys.append(self.toKey(job_id)) keys.append(self.keys['events']) - keys.append(self.keys['paused']) keys.append(self.keys['meta']) args = [self.keys[''], job_id] diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index b1bc385e25..96435db531 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -1074,15 +1074,14 @@ export class Scripts { const keys: (string | number)[] = [ this.queue.keys.active, this.queue.keys.wait, - this.queue.keys.paused, this.queue.toKey(jobId), this.queue.keys.meta, this.queue.keys.events, this.queue.keys.delayed, this.queue.keys.prioritized, this.queue.keys.pc, - this.queue.keys.marker, this.queue.keys.stalled, + this.queue.keys.marker, ]; const pushCmd = (lifo ? 'R' : 'L') + 'PUSH'; @@ -1230,7 +1229,6 @@ export class Scripts { const keys = [ this.queue.keys.delayed, this.queue.keys.wait, - this.queue.keys.paused, this.queue.keys.meta, this.queue.keys.prioritized, this.queue.keys.active, diff --git a/src/commands/promote-9.lua b/src/commands/promote-8.lua similarity index 66% rename from src/commands/promote-9.lua rename to src/commands/promote-8.lua index a933a83f4d..f691f13e26 100644 --- a/src/commands/promote-9.lua +++ b/src/commands/promote-8.lua @@ -4,13 +4,12 @@ Input: KEYS[1] 'delayed' KEYS[2] 'wait' - KEYS[3] 'paused' // TODO remove - KEYS[4] 'meta' - KEYS[5] 'prioritized' - KEYS[6] 'active' - KEYS[7] 'pc' priority counter - KEYS[8] 'event stream' - KEYS[9] 'marker' + KEYS[3] 'meta' + KEYS[4] 'prioritized' + KEYS[5] 'active' + KEYS[6] 'pc' priority counter + KEYS[7] 'event stream' + KEYS[8] 'marker' ARGV[1] queue.toKey('') ARGV[2] jobId @@ -33,20 +32,20 @@ local jobId = ARGV[2] if rcall("ZREM", KEYS[1], jobId) == 1 then local jobKey = ARGV[1] .. jobId local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0 - local metaKey = KEYS[4] - local markerKey = KEYS[9] + local metaKey = KEYS[3] + local markerKey = KEYS[8] - local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[6]) + local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[5]) if priority == 0 then -- LIFO or FIFO addJobInTargetList(KEYS[2], markerKey, "LPUSH", isPausedOrMaxed, jobId) else - addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed) + addJobWithPriority(markerKey, KEYS[4], priority, jobId, KEYS[6], isPausedOrMaxed) end -- Emit waiting event (wait..ing@token) - rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev", + rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev", "delayed"); rcall("HSET", jobKey, "delay", 0) diff --git a/src/commands/retryJob-11.lua b/src/commands/retryJob-10.lua similarity index 60% rename from src/commands/retryJob-11.lua rename to src/commands/retryJob-10.lua index 9acf0b5266..d5977b41d2 100644 --- a/src/commands/retryJob-11.lua +++ b/src/commands/retryJob-10.lua @@ -4,15 +4,14 @@ Input: KEYS[1] 'active', KEYS[2] 'wait' - KEYS[3] 'paused' // TODO remove - KEYS[4] job key - KEYS[5] 'meta' - KEYS[6] events stream - KEYS[7] delayed key - KEYS[8] prioritized key - KEYS[9] 'pc' priority counter + KEYS[3] job key + KEYS[4] 'meta' + KEYS[5] events stream + KEYS[6] delayed key + KEYS[7] prioritized key + KEYS[8] 'pc' priority counter + KEYS[9] 'stalled' KEYS[10] 'marker' - KEYS[11] 'stalled' ARGV[1] key prefix ARGV[2] timestamp @@ -39,15 +38,17 @@ local rcall = redis.call --- @include "includes/removeLock" --- @include "includes/isQueuePausedOrMaxed" -local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1]) +local jobKey = KEYS[3] +local metaKey = KEYS[4] +local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[1]) local markerKey = KEYS[10] -- Check if there are delayed jobs that we can move to wait. -- test example: when there are delayed jobs between retries -promoteDelayedJobs(KEYS[7], markerKey, KEYS[2], KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed) +promoteDelayedJobs(KEYS[6], markerKey, KEYS[2], KEYS[7], KEYS[5], ARGV[1], ARGV[2], KEYS[8], isPausedOrMaxed) -if rcall("EXISTS", KEYS[4]) == 1 then - local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4]) +if rcall("EXISTS", jobKey) == 1 then + local errorCode = removeLock(jobKey, KEYS[9], ARGV[5], ARGV[4]) if errorCode < 0 then return errorCode end @@ -55,24 +56,24 @@ if rcall("EXISTS", KEYS[4]) == 1 then local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[4]) if (numRemovedElements < 1) then return -3 end - local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0 + local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0 --need to re-evaluate after removing job from active - isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1]) + isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[1]) -- Standard or priority add if priority == 0 then addJobInTargetList(KEYS[2], markerKey, ARGV[3], isPausedOrMaxed, ARGV[4]) else - addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed) + addJobWithPriority(markerKey, KEYS[7], priority, ARGV[4], KEYS[8], isPausedOrMaxed) end - rcall("HINCRBY", KEYS[4], "atm", 1) + rcall("HINCRBY", jobKey, "atm", 1) - local maxEvents = getOrSetMaxEvents(KEYS[5]) + local maxEvents = getOrSetMaxEvents(metaKey) -- Emit waiting event - rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "waiting", + rcall("XADD", KEYS[5], "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", ARGV[4], "prev", "failed") return 0 From c3acbb9a8164e8e5e49155f05acd139354e6bf88 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 20:15:58 -0500 Subject: [PATCH 07/28] refactor: remove paused key on moveToActive --- python/bullmq/scripts.py | 12 ++++++------ src/classes/scripts.ts | 3 --- ...{moveJobsToWait-8.lua => moveJobsToWait-7.lua} | 13 ++++++------- ...bsToWait-9.lua => moveStalledJobsToWait-8.lua} | 9 ++++----- .../{moveToActive-11.lua => moveToActive-10.lua} | 15 +++++++-------- 5 files changed, 23 insertions(+), 29 deletions(-) rename src/commands/{moveJobsToWait-8.lua => moveJobsToWait-7.lua} (87%) rename src/commands/{moveStalledJobsToWait-9.lua => moveStalledJobsToWait-8.lua} (97%) rename src/commands/{moveToActive-11.lua => moveToActive-10.lua} (90%) diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index 21084f8f16..03f74fceb9 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -51,8 +51,8 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "getState": self.redisClient.register_script(self.getScript("getState-8.lua")), "getStateV2": self.redisClient.register_script(self.getScript("getStateV2-8.lua")), "isJobInList": self.redisClient.register_script(self.getScript("isJobInList-1.lua")), - "moveStalledJobsToWait": self.redisClient.register_script(self.getScript("moveStalledJobsToWait-9.lua")), - "moveToActive": self.redisClient.register_script(self.getScript("moveToActive-11.lua")), + "moveStalledJobsToWait": self.redisClient.register_script(self.getScript("moveStalledJobsToWait-8.lua")), + "moveToActive": self.redisClient.register_script(self.getScript("moveToActive-10.lua")), "moveToDelayed": self.redisClient.register_script(self.getScript("moveToDelayed-8.lua")), "moveToFinished": self.redisClient.register_script(self.getScript("moveToFinished-14.lua")), "moveToWaitingChildren": self.redisClient.register_script(self.getScript("moveToWaitingChildren-5.lua")), @@ -62,7 +62,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "removeJob": self.redisClient.register_script(self.getScript("removeJob-2.lua")), "reprocessJob": self.redisClient.register_script(self.getScript("reprocessJob-7.lua")), "retryJob": self.redisClient.register_script(self.getScript("retryJob-10.lua")), - "moveJobsToWait": self.redisClient.register_script(self.getScript("moveJobsToWait-8.lua")), + "moveJobsToWait": self.redisClient.register_script(self.getScript("moveJobsToWait-7.lua")), "saveStacktrace": self.redisClient.register_script(self.getScript("saveStacktrace-1.lua")), "updateData": self.redisClient.register_script(self.getScript("updateData-1.lua")), "updateProgress": self.redisClient.register_script(self.getScript("updateProgress-3.lua")), @@ -446,7 +446,7 @@ async def obliterate(self, count: int, force: bool = False): def moveJobsToWaitArgs(self, state: str, count: int, timestamp: int) -> int: keys = self.getKeys( - ['', 'events', state, 'wait', 'paused', 'meta', 'active', 'marker']) + ['', 'events', state, 'wait', 'meta', 'active', 'marker']) args = [count or 1000, timestamp or round(time.time()*1000), state] return (keys, args) @@ -479,7 +479,7 @@ async def moveToActive(self, token: str, opts: dict) -> list[Any]: limiter = opts.get("limiter", None) keys = self.getKeys(['wait', 'active', 'prioritized', 'events', - 'stalled', 'limiter', 'delayed', 'paused', 'meta', 'pc', 'marker']) + 'stalled', 'limiter', 'delayed', 'meta', 'pc', 'marker']) packedOpts = msgpack.packb( {"token": token, "lockDuration": lockDuration, "limiter": limiter}, use_bin_type=True) args = [self.keys[''], timestamp, packedOpts] @@ -576,7 +576,7 @@ def extendLock(self, jobId: str, token: str, duration: int, client: Redis = None def moveStalledJobsToWait(self, maxStalledCount: int, stalledInterval: int): keys = self.getKeys(['stalled', 'wait', 'active', 'failed', - 'stalled-check', 'meta', 'paused', 'marker', 'events']) + 'stalled-check', 'meta', 'marker', 'events']) args = [maxStalledCount, self.keys[''], round( time.time() * 1000), stalledInterval] return self.commands["moveStalledJobsToWait"](keys, args) diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index 96435db531..f642a9bbd1 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -1105,7 +1105,6 @@ export class Scripts { this.queue.keys.events, this.queue.toKey(state), this.queue.toKey('wait'), - this.queue.toKey('paused'), this.queue.keys.meta, this.queue.keys.active, this.queue.keys.marker, @@ -1199,7 +1198,6 @@ export class Scripts { queueKeys.stalled, queueKeys.limiter, queueKeys.delayed, - queueKeys.paused, queueKeys.meta, queueKeys.pc, queueKeys.marker, @@ -1259,7 +1257,6 @@ export class Scripts { this.queue.keys.failed, this.queue.keys['stalled-check'], this.queue.keys.meta, - this.queue.keys.paused, this.queue.keys.marker, this.queue.keys.events, ]; diff --git a/src/commands/moveJobsToWait-8.lua b/src/commands/moveJobsToWait-7.lua similarity index 87% rename from src/commands/moveJobsToWait-8.lua rename to src/commands/moveJobsToWait-7.lua index 9a69fc93e9..09cf80708e 100644 --- a/src/commands/moveJobsToWait-8.lua +++ b/src/commands/moveJobsToWait-7.lua @@ -8,10 +8,9 @@ KEYS[2] events stream KEYS[3] state key (failed, completed, delayed) KEYS[4] 'wait' - KEYS[5] 'paused' // TODO remove - KEYS[6] 'meta' - KEYS[7] 'active' - KEYS[8] 'marker' + KEYS[5] 'meta' + KEYS[6] 'active' + KEYS[7] 'marker' ARGV[1] count ARGV[2] timestamp @@ -32,8 +31,8 @@ local rcall = redis.call; --- @include "includes/getOrSetMaxEvents" --- @include "includes/isQueuePausedOrMaxed" -local metaKey = KEYS[6] -local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[7]) +local metaKey = KEYS[5] +local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[6]) local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount) if (#jobs > 0) then @@ -63,7 +62,7 @@ if (#jobs > 0) then rcall("LPUSH", KEYS[4], unpack(jobs, from, to)) end - addBaseMarkerIfNeeded(KEYS[8], isPausedOrMaxed) + addBaseMarkerIfNeeded(KEYS[7], isPausedOrMaxed) end maxCount = maxCount - #jobs diff --git a/src/commands/moveStalledJobsToWait-9.lua b/src/commands/moveStalledJobsToWait-8.lua similarity index 97% rename from src/commands/moveStalledJobsToWait-9.lua rename to src/commands/moveStalledJobsToWait-8.lua index da06c41f8b..95bef98bf3 100644 --- a/src/commands/moveStalledJobsToWait-9.lua +++ b/src/commands/moveStalledJobsToWait-8.lua @@ -8,9 +8,8 @@ KEYS[4] 'failed', (ZSET) KEYS[5] 'stalled-check', (KEY) KEYS[6] 'meta', (KEY) - KEYS[7] 'paused', (LIST) // TODO remove - KEYS[8] 'marker' - KEYS[9] 'event stream' (STREAM) + KEYS[7] 'marker' + KEYS[8] 'event stream' (STREAM) ARGV[1] Max stalled job count ARGV[2] queue.toKey('') @@ -40,8 +39,8 @@ local activeKey = KEYS[3] local failedKey = KEYS[4] local stalledCheckKey = KEYS[5] local metaKey = KEYS[6] -local markerKey = KEYS[8] -local eventStreamKey = KEYS[9] +local markerKey = KEYS[7] +local eventStreamKey = KEYS[8] local maxStalledJobCount = ARGV[1] local queueKeyPrefix = ARGV[2] local timestamp = ARGV[3] diff --git a/src/commands/moveToActive-11.lua b/src/commands/moveToActive-10.lua similarity index 90% rename from src/commands/moveToActive-11.lua rename to src/commands/moveToActive-10.lua index 2f8535345f..c0390f11d1 100644 --- a/src/commands/moveToActive-11.lua +++ b/src/commands/moveToActive-10.lua @@ -18,12 +18,11 @@ KEYS[7] delayed key -- Delayed jobs - KEYS[8] paused key // TODO remove - KEYS[9] meta key - KEYS[10] pc priority counter + KEYS[8] meta key + KEYS[9] pc priority counter -- Marker - KEYS[11] marker key + KEYS[10] marker key -- Arguments ARGV[1] key prefix @@ -50,12 +49,12 @@ local opts = cmsgpack.unpack(ARGV[3]) --- @include "includes/prepareJobForProcessing" --- @include "includes/promoteDelayedJobs" -local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[9], activeKey) +local isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[8], activeKey) -- Check if there are delayed jobs that we can move to wait. -local markerKey = KEYS[11] +local markerKey = KEYS[10] promoteDelayedJobs(delayedKey, markerKey, KEYS[1], KEYS[3], eventStreamKey, ARGV[1], - ARGV[2], KEYS[10], isPausedOrMaxed) + ARGV[2], KEYS[9], isPausedOrMaxed) local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max']) local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey) @@ -73,7 +72,7 @@ if jobId then return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2], maxJobs, opts) else - jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10]) + jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[9]) if jobId then return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2], maxJobs, opts) From e973ec55adcf55b18271af3d2013b4033ddac5a0 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 20:21:48 -0500 Subject: [PATCH 08/28] refactor: remove paused key on moveToFinished --- python/bullmq/scripts.py | 4 +-- src/classes/scripts.ts | 1 - ...oFinished-14.lua => moveToFinished-13.lua} | 27 +++++++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) rename src/commands/{moveToFinished-14.lua => moveToFinished-13.lua} (93%) diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index 03f74fceb9..122612447e 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -54,7 +54,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "moveStalledJobsToWait": self.redisClient.register_script(self.getScript("moveStalledJobsToWait-8.lua")), "moveToActive": self.redisClient.register_script(self.getScript("moveToActive-10.lua")), "moveToDelayed": self.redisClient.register_script(self.getScript("moveToDelayed-8.lua")), - "moveToFinished": self.redisClient.register_script(self.getScript("moveToFinished-14.lua")), + "moveToFinished": self.redisClient.register_script(self.getScript("moveToFinished-13.lua")), "moveToWaitingChildren": self.redisClient.register_script(self.getScript("moveToWaitingChildren-5.lua")), "obliterate": self.redisClient.register_script(self.getScript("obliterate-2.lua")), "pause": self.redisClient.register_script(self.getScript("pause-7.lua")), @@ -512,7 +512,7 @@ def moveToFinishedArgs(self, job: Job, val: Any, propVal: str, shouldRemove, tar metricsKey = self.toKey('metrics:' + target) keys = self.getKeys(['wait', 'active', 'prioritized', 'events', - 'stalled', 'limiter', 'delayed', 'paused', 'meta', 'pc', target]) + 'stalled', 'limiter', 'delayed', 'meta', 'pc', target]) keys.append(self.toKey(job.id)) keys.append(metricsKey) keys.append(self.keys['marker']) diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index f642a9bbd1..2e14f0bfe2 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -60,7 +60,6 @@ export class Scripts { queueKeys.stalled, queueKeys.limiter, queueKeys.delayed, - queueKeys.paused, queueKeys.meta, queueKeys.pc, undefined, diff --git a/src/commands/moveToFinished-14.lua b/src/commands/moveToFinished-13.lua similarity index 93% rename from src/commands/moveToFinished-14.lua rename to src/commands/moveToFinished-13.lua index 50e857b885..9cdcae1650 100644 --- a/src/commands/moveToFinished-14.lua +++ b/src/commands/moveToFinished-13.lua @@ -15,14 +15,13 @@ KEYS[6] rate limiter key KEYS[7] delayed key - KEYS[8] paused key // TODO remove - KEYS[9] meta key - KEYS[10] pc priority counter + KEYS[8] meta key + KEYS[9] pc priority counter - KEYS[11] completed/failed key - KEYS[12] jobId key - KEYS[13] metrics key - KEYS[14] marker key + KEYS[10] completed/failed key + KEYS[11] jobId key + KEYS[12] metrics key + KEYS[13] marker key ARGV[1] jobId ARGV[2] timestamp @@ -70,7 +69,7 @@ local rcall = redis.call --- @include "includes/trimEvents" --- @include "includes/updateParentDepsIfNeeded" -local jobIdKey = KEYS[12] +local jobIdKey = KEYS[11] if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists local opts = cmsgpack.unpack(ARGV[8]) @@ -101,7 +100,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists if (numRemovedElements < 1) then return -3 end local eventStreamKey = KEYS[4] - local metaKey = KEYS[9] + local metaKey = KEYS[8] -- Trim events before emiting them to avoid trimming events emitted in this script trimEvents(metaKey, eventStreamKey) @@ -137,7 +136,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists -- Remove job? if maxCount ~= 0 then - local targetSet = KEYS[11] + local targetSet = KEYS[10] -- Add to complete/failed set rcall("ZADD", targetSet, timestamp, jobId) rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp) @@ -173,7 +172,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists -- Collect metrics if maxMetricsSize ~= "" then - collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp) + collectMetrics(KEYS[12], KEYS[12] .. ':data', maxMetricsSize, timestamp) end -- Try to get next job to avoid an extra roundtrip if the queue is not closing, @@ -183,8 +182,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, KEYS[2]) -- Check if there are delayed jobs that can be promoted - promoteDelayedJobs(KEYS[7], KEYS[14], KEYS[1], KEYS[3], eventStreamKey, prefix, - timestamp, KEYS[10], isPausedOrMaxed) + promoteDelayedJobs(KEYS[7], KEYS[13], KEYS[1], KEYS[3], eventStreamKey, prefix, + timestamp, KEYS[9], isPausedOrMaxed) local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max']) -- Check if we are rate limited first. @@ -201,7 +200,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, opts) else - jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10]) + jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[9]) if jobId then return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, From 65c3beba2d02781557759196bb2a6160245f9d3a Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 12 Sep 2024 20:33:46 -0500 Subject: [PATCH 09/28] chore: fix moveToFinishedKeys ref --- src/classes/scripts.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index 2e14f0bfe2..0ad4bd9335 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -484,10 +484,10 @@ export class Scripts { const metricsKey = this.queue.toKey(`metrics:${target}`); const keys = this.moveToFinishedKeys; - keys[10] = queueKeys[target]; - keys[11] = this.queue.toKey(job.id ?? ''); - keys[12] = metricsKey; - keys[13] = this.queue.keys.marker; + keys[9] = queueKeys[target]; + keys[10] = this.queue.toKey(job.id ?? ''); + keys[11] = metricsKey; + keys[12] = this.queue.keys.marker; const keepJobs = this.getKeepJobs(shouldRemove, workerKeepJobs); From 7a93835556811c17098e9a65c650c2c39f2b2a3a Mon Sep 17 00:00:00 2001 From: roggervalf Date: Fri, 13 Sep 2024 20:24:46 -0500 Subject: [PATCH 10/28] chore: add repairPausedKey script --- python/bullmq/queue.py | 13 ++++++++- python/bullmq/scripts.py | 10 +++++++ src/classes/queue.ts | 14 ++++++++++ src/classes/scripts.ts | 12 +++++++++ src/commands/includes/getTargetQueueList.lua | 22 --------------- src/commands/pause-7.lua | 2 +- src/commands/repairPausedKey-2.lua | 28 ++++++++++++++++++++ tests/test_queue.ts | 16 +++++++++++ 8 files changed, 93 insertions(+), 24 deletions(-) delete mode 100644 src/commands/includes/getTargetQueueList.lua create mode 100644 src/commands/repairPausedKey-2.lua diff --git a/python/bullmq/queue.py b/python/bullmq/queue.py index 0bcffc1fae..dc9dd8dda1 100644 --- a/python/bullmq/queue.py +++ b/python/bullmq/queue.py @@ -141,7 +141,7 @@ async def getJobLogs(self, job_id:str, start = 0, end = -1, asc = True): "logs": result[0], "count": result[1] } - + async def obliterate(self, force: bool = False): """ Completely destroys the queue and all of its contents irreversibly. @@ -198,6 +198,17 @@ def removeDeprecatedPriorityKey(self): """ return self.client.delete(self.toKey("priority")) + async def repairDeprecatedPausedKey(self, maxCount: int = 1000): + """ + Repair deprecated paused key. + + @param maxCount: Max quantity of jobs to be moved to wait per iteration. + """ + while True: + cursor = await self.scripts.repairDeprecatedPausedKey(maxCount) + if cursor is None or cursor == 0 or cursor == "0": + break + async def getJobCountByTypes(self, *types): result = await self.getJobCounts(*types) sum = 0 diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index 122612447e..3add72d1f6 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -60,6 +60,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "pause": self.redisClient.register_script(self.getScript("pause-7.lua")), "promote": self.redisClient.register_script(self.getScript("promote-8.lua")), "removeJob": self.redisClient.register_script(self.getScript("removeJob-2.lua")), + "repairPausedKey": self.redisClient.register_script(self.getScript("repairPausedKey-2.lua")), "reprocessJob": self.redisClient.register_script(self.getScript("reprocessJob-7.lua")), "retryJob": self.redisClient.register_script(self.getScript("retryJob-10.lua")), "moveJobsToWait": self.redisClient.register_script(self.getScript("moveJobsToWait-7.lua")), @@ -461,6 +462,15 @@ async def retryJobs(self, state: str, count: int, timestamp: int): result = await self.commands["moveJobsToWait"](keys=keys, args=args) return result + async def repairDeprecatedPausedKey(self, maxCount: int): + keys = self.getKeys( + ['paused', 'wait']) + + args = [maxCount] + + result = await self.commands["repairPausedKey"](keys=keys, args=args) + return result + async def promoteJobs(self, count: int): """ Promote jobs in delayed state diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 4cd2ed8db5..403ee259f4 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -586,4 +586,18 @@ export class Queue< const client = await this.client; return client.del(this.toKey('priority')); } + + /** + * Repair deprecated paused key + * + * @param maxCount - Max quantity of jobs to be moved to wait per iteration. + */ + async repairDeprecatedPausedKey(maxCount: number = 1000): Promise { + let cursor = 0; + do { + cursor = await this.scripts.repairDeprecatedPausedKey( + maxCount + ); + } while (cursor); + } } diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index 0ad4bd9335..3d4a130c82 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -1420,6 +1420,18 @@ export class Scripts { }; } } + + async repairDeprecatedPausedKey(maxCount: number): Promise { + const client = await this.queue.client; + + const keys: (string | number)[] = [ + this.queue.keys.paused, + this.queue.keys.wait, + ]; + const args = [maxCount]; + + return (client).repairPausedKey(keys.concat(args)); + } } export function raw2NextJobData(raw: any[]) { diff --git a/src/commands/includes/getTargetQueueList.lua b/src/commands/includes/getTargetQueueList.lua deleted file mode 100644 index 2a7b03571a..0000000000 --- a/src/commands/includes/getTargetQueueList.lua +++ /dev/null @@ -1,22 +0,0 @@ ---[[ - Function to check for the meta.paused key to decide if we are paused or not - (since an empty list and !EXISTS are not really the same). -]] - -local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey) - local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency") - - if queueAttributes[1] then - return pausedKey, true - else - if queueAttributes[2] then - local activeCount = rcall("LLEN", activeKey) - if activeCount >= tonumber(queueAttributes[2]) then - return waitKey, true - else - return waitKey, false - end - end - end - return waitKey, false -end diff --git a/src/commands/pause-7.lua b/src/commands/pause-7.lua index 880494ef67..471083b14a 100644 --- a/src/commands/pause-7.lua +++ b/src/commands/pause-7.lua @@ -35,7 +35,7 @@ else --move a maximum of 7000 per resumed call in order to not block --if users have more jobs in paused state, call resumed multiple times local jobs = rcall('LRANGE', KEYS[1], 0, 6999) - rcall("LPUSH", KEYS[2], unpack(jobs)) + rcall("RPUSH", KEYS[2], unpack(jobs)) rcall("LTRIM", KEYS[1], #jobs, -1) end diff --git a/src/commands/repairPausedKey-2.lua b/src/commands/repairPausedKey-2.lua new file mode 100644 index 0000000000..8d3cfac384 --- /dev/null +++ b/src/commands/repairPausedKey-2.lua @@ -0,0 +1,28 @@ +--[[ + Move paused job ids to wait state to repair these states + + Input: + KEYS[1] paused key + KEYS[2] wait key + + ARGV[1] count +]] + +local maxCount = tonumber(ARGV[1]) + +local rcall = redis.call + +local hasJobs = rcall("EXISTS", KEYS[1]) == 1 + +if hasJobs then + local jobs = rcall('LRANGE', KEYS[1], 0, maxCount - 1) + rcall("RPUSH", KEYS[2], unpack(jobs)) + rcall("LTRIM", KEYS[1], #jobs, -1) + + if (maxCount - #jobs) <= 0 then + return 1 + end +end + +return 0 + \ No newline at end of file diff --git a/tests/test_queue.ts b/tests/test_queue.ts index 0a2b29d5e2..c75f6df2ba 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -428,6 +428,22 @@ describe('queues', function () { }); }); + describe('.repairDeprecatedPausedKey', () => { + it('moves jobs from paused to wait', async () => { + const client = await queue.client; + await client.lpush(`${prefix}:${queue.name}:paused`, 'a', 'b', 'c'); + await client.lpush(`${prefix}:${queue.name}:wait`, 'd', 'e', 'f'); + + await queue.repairDeprecatedPausedKey(); + + const jobs = await client.lrange( + `${prefix}:${queue.name}:wait`, 0, -1 + ); + + expect(jobs).to.be.eql(['f', 'e', 'd', 'c', 'b', 'a']); + }); + }); + describe('.removeLegacyMarkers', () => { it('removes old markers', async () => { const client = await queue.client; From 537ce42171a979138059c2329df05df7740136a3 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Fri, 20 Sep 2024 23:29:35 -0500 Subject: [PATCH 11/28] docs(migrations): add new section --- docs/gitbook/SUMMARY.md | 3 ++- .../migration-to-newer-versions.md | 1 - docs/gitbook/guide/migrations/v6.md | 26 +++++++++++++++++++ python/bullmq/queue.py | 6 ++--- python/bullmq/scripts.py | 2 +- src/classes/queue.ts | 6 ++--- src/classes/scripts.ts | 2 +- tests/test_queue.ts | 4 +-- 8 files changed, 38 insertions(+), 12 deletions(-) rename docs/gitbook/guide/{ => migrations}/migration-to-newer-versions.md (99%) create mode 100644 docs/gitbook/guide/migrations/v6.md diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index c4c338c6e3..1c07173c2b 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -61,7 +61,8 @@ * [Producers](guide/nestjs/producers.md) * [Queue Events Listeners](guide/nestjs/queue-events-listeners.md) * [Going to production](guide/going-to-production.md) -* [Migration to newer versions](guide/migration-to-newer-versions.md) +* [Migration to newer versions](guide/migrations/migration-to-newer-versions.md) + * [Version 6](guide/migrations/v6.md) * [Troubleshooting](guide/troubleshooting.md) ## Patterns diff --git a/docs/gitbook/guide/migration-to-newer-versions.md b/docs/gitbook/guide/migrations/migration-to-newer-versions.md similarity index 99% rename from docs/gitbook/guide/migration-to-newer-versions.md rename to docs/gitbook/guide/migrations/migration-to-newer-versions.md index 0b8ca2a175..ad2c27916b 100644 --- a/docs/gitbook/guide/migration-to-newer-versions.md +++ b/docs/gitbook/guide/migrations/migration-to-newer-versions.md @@ -55,4 +55,3 @@ Since BullMQ supports global pause, one possible strategy, if suitable for your ### Use new queues altogether This drastic solution involves discontinuing use of older queues and creating new ones. You could rename older queues (e.g., "myQueueV2"), use a new Redis host, or maintain two versions of the service—one running an older BullMQ version with old queues, and a newer one with the latest BullMQ and a different set of queues. When the older version has no more jobs to process, it can be retired, leaving only the upgraded version. - diff --git a/docs/gitbook/guide/migrations/v6.md b/docs/gitbook/guide/migrations/v6.md new file mode 100644 index 0000000000..27fca69942 --- /dev/null +++ b/docs/gitbook/guide/migrations/v6.md @@ -0,0 +1,26 @@ +--- +description: Tips and hints on how to migrate to v5. +--- + +# Migration to v6 + +## Migration of deprecated paused key + +If you have paused queues after upgrading to this version. You must use **migrateDeprecatedPausedKey** from your queue instances in order to move your jobs from this state to wait state. + +Paused key is not longer needed as this state is already represented by queue meta key. It also improve the process of pausing or resuming a queue as we don't need to rename any key. + +## Remove legacy markers + +When migrating from versions before v5, you must use **removeLegacyMarkers** method from queue your queue instances. + +It's recommended to do this process: + +1. Pause your queues. +2. Upgrade to v6. +3. Execute **removeLegacyMarkers** in each queue. +4. Resume your queues. + +This way you will prevent that your workers pick a legacy marker. + +A second option would be to do incremental upgrades. diff --git a/python/bullmq/queue.py b/python/bullmq/queue.py index dc9dd8dda1..f66a0a0915 100644 --- a/python/bullmq/queue.py +++ b/python/bullmq/queue.py @@ -198,14 +198,14 @@ def removeDeprecatedPriorityKey(self): """ return self.client.delete(self.toKey("priority")) - async def repairDeprecatedPausedKey(self, maxCount: int = 1000): + async def migrateDeprecatedPausedKey(self, maxCount: int = 1000): """ - Repair deprecated paused key. + Migrate deprecated paused key. @param maxCount: Max quantity of jobs to be moved to wait per iteration. """ while True: - cursor = await self.scripts.repairDeprecatedPausedKey(maxCount) + cursor = await self.scripts.migrateDeprecatedPausedKey(maxCount) if cursor is None or cursor == 0 or cursor == "0": break diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index 3add72d1f6..c9ae1ab4fa 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -462,7 +462,7 @@ async def retryJobs(self, state: str, count: int, timestamp: int): result = await self.commands["moveJobsToWait"](keys=keys, args=args) return result - async def repairDeprecatedPausedKey(self, maxCount: int): + async def migrateDeprecatedPausedKey(self, maxCount: int): keys = self.getKeys( ['paused', 'wait']) diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 403ee259f4..6029d9d0ac 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -588,14 +588,14 @@ export class Queue< } /** - * Repair deprecated paused key + * Migrate deprecated paused key * * @param maxCount - Max quantity of jobs to be moved to wait per iteration. */ - async repairDeprecatedPausedKey(maxCount: number = 1000): Promise { + async migrateDeprecatedPausedKey(maxCount: number = 1000): Promise { let cursor = 0; do { - cursor = await this.scripts.repairDeprecatedPausedKey( + cursor = await this.scripts.migrateDeprecatedPausedKey( maxCount ); } while (cursor); diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index cfba67f407..f866054444 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -1421,7 +1421,7 @@ export class Scripts { } } - async repairDeprecatedPausedKey(maxCount: number): Promise { + async migrateDeprecatedPausedKey(maxCount: number): Promise { const client = await this.queue.client; const keys: (string | number)[] = [ diff --git a/tests/test_queue.ts b/tests/test_queue.ts index c75f6df2ba..e9278743cf 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -428,13 +428,13 @@ describe('queues', function () { }); }); - describe('.repairDeprecatedPausedKey', () => { + describe('.migrateDeprecatedPausedKey', () => { it('moves jobs from paused to wait', async () => { const client = await queue.client; await client.lpush(`${prefix}:${queue.name}:paused`, 'a', 'b', 'c'); await client.lpush(`${prefix}:${queue.name}:wait`, 'd', 'e', 'f'); - await queue.repairDeprecatedPausedKey(); + await queue.migrateDeprecatedPausedKey(); const jobs = await client.lrange( `${prefix}:${queue.name}:wait`, 0, -1 From e593d8a993d9f1ff54c23ff3b7eb75fc24da2a6a Mon Sep 17 00:00:00 2001 From: roggervalf Date: Fri, 20 Sep 2024 23:30:57 -0500 Subject: [PATCH 12/28] docs: fix typo --- docs/gitbook/guide/migrations/v6.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gitbook/guide/migrations/v6.md b/docs/gitbook/guide/migrations/v6.md index 27fca69942..558e510121 100644 --- a/docs/gitbook/guide/migrations/v6.md +++ b/docs/gitbook/guide/migrations/v6.md @@ -1,5 +1,5 @@ --- -description: Tips and hints on how to migrate to v5. +description: Tips and hints on how to migrate to v6. --- # Migration to v6 From efe1a56f982fb4dd7d1f9a0bce5345dbd5a7f14e Mon Sep 17 00:00:00 2001 From: roggervalf Date: Mon, 23 Sep 2024 22:56:49 -0500 Subject: [PATCH 13/28] chore: restore few decoded options --- src/classes/job.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/classes/job.ts b/src/classes/job.ts index 3d5cee310f..bf1d03401c 100644 --- a/src/classes/job.ts +++ b/src/classes/job.ts @@ -39,8 +39,11 @@ const logger = debuglog('bull'); const optsDecodeMap = { de: 'debounce', - ocf: 'onChildFailure', + fpof: 'failParentOnFailure', + idof: 'ignoreDependencyOnFailure', kl: 'keepLogs', + ocf: 'onChildFailure', + rdof: 'removeDependencyOnFailure', }; const optsEncodeMap = invertObject(optsDecodeMap); From 42ebfcb36d9be80ddb0a44f79f6ecff6c722d595 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 25 Sep 2024 23:58:34 -0500 Subject: [PATCH 14/28] refactor: add executeMigrations script --- python/bullmq/scripts.py | 4 +- src/classes/queue-keys.ts | 1 + src/classes/scripts.ts | 21 +++++- src/classes/worker.ts | 30 +++++++++ src/commands/executeMigrations-3.lua | 64 +++++++++++++++++++ ...2.lua => migrateDeprecatedPausedKey-2.lua} | 0 tests/test_queue.ts | 32 ---------- tests/test_worker.ts | 56 ++++++++++++++++ 8 files changed, 173 insertions(+), 35 deletions(-) create mode 100644 src/commands/executeMigrations-3.lua rename src/commands/{repairPausedKey-2.lua => migrateDeprecatedPausedKey-2.lua} (100%) diff --git a/python/bullmq/scripts.py b/python/bullmq/scripts.py index c9ae1ab4fa..0b5e2f2967 100644 --- a/python/bullmq/scripts.py +++ b/python/bullmq/scripts.py @@ -60,7 +60,7 @@ def __init__(self, prefix: str, queueName: str, redisConnection: RedisConnection "pause": self.redisClient.register_script(self.getScript("pause-7.lua")), "promote": self.redisClient.register_script(self.getScript("promote-8.lua")), "removeJob": self.redisClient.register_script(self.getScript("removeJob-2.lua")), - "repairPausedKey": self.redisClient.register_script(self.getScript("repairPausedKey-2.lua")), + "migrateDeprecatedPausedKey": self.redisClient.register_script(self.getScript("migrateDeprecatedPausedKey-2.lua")), "reprocessJob": self.redisClient.register_script(self.getScript("reprocessJob-7.lua")), "retryJob": self.redisClient.register_script(self.getScript("retryJob-10.lua")), "moveJobsToWait": self.redisClient.register_script(self.getScript("moveJobsToWait-7.lua")), @@ -468,7 +468,7 @@ async def migrateDeprecatedPausedKey(self, maxCount: int): args = [maxCount] - result = await self.commands["repairPausedKey"](keys=keys, args=args) + result = await self.commands["migrateDeprecatedPausedKey"](keys=keys, args=args) return result async def promoteJobs(self, count: int): diff --git a/src/classes/queue-keys.ts b/src/classes/queue-keys.ts index 9109ae8dba..14b5216131 100644 --- a/src/classes/queue-keys.ts +++ b/src/classes/queue-keys.ts @@ -21,6 +21,7 @@ export class QueueKeys { 'repeat', 'limiter', 'meta', + 'migrations', 'events', 'pc', // priority counter key 'marker', // marker key diff --git a/src/classes/scripts.ts b/src/classes/scripts.ts index f866054444..c692503255 100644 --- a/src/classes/scripts.ts +++ b/src/classes/scripts.ts @@ -1430,7 +1430,26 @@ export class Scripts { ]; const args = [maxCount]; - return (client).repairPausedKey(keys.concat(args)); + return (client).migrateDeprecatedPausedKey(keys.concat(args)); + } + + protected executeMigrationsArgs(currentMigrationExecution = 1): (string | number)[] { + const keys: (string | number)[] = [ + this.queue.keys.meta, + this.queue.keys.migrations, + this.queue.toKey(''), + ]; + const args = [6, Date.now(), currentMigrationExecution]; + + return keys.concat(args); + } + + async executeMigrations(currentMigrationExecution: number): Promise { + const client = await this.queue.client; + + const args = this.executeMigrationsArgs(currentMigrationExecution); + + return (client).executeMigrations(args); } } diff --git a/src/classes/worker.ts b/src/classes/worker.ts index 881a25ef9a..23ca319654 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -417,6 +417,7 @@ export class Worker< return; } + await this.executeMigrations(); await this.startStalledCheckTimer(); const jobsInProgress = new Set<{ job: Job; ts: number }>(); @@ -1086,4 +1087,33 @@ will never work with more accuracy than 1ms. */ ) { return this.scripts.moveJobFromActiveToWait(job.id, token); } + + /** + * Execute migration scripts while migrating to breaking changes. + */ + async executeMigrations( + ): Promise { + let currentMigrationExecution=1; + + while (currentMigrationExecution > 0) { + currentMigrationExecution = await this.scripts.executeMigrations( + currentMigrationExecution + ); + switch (currentMigrationExecution) { + case 2:{ + await this.scripts.removeLegacyMarkers(); + currentMigrationExecution++; + break; + } + case 3:{ + let cursor = 0; + do { + cursor = await this.scripts.migrateDeprecatedPausedKey(5000); + } while (cursor); + currentMigrationExecution++; + break; + } + } + } + } } diff --git a/src/commands/executeMigrations-3.lua b/src/commands/executeMigrations-3.lua new file mode 100644 index 0000000000..22fbb363e1 --- /dev/null +++ b/src/commands/executeMigrations-3.lua @@ -0,0 +1,64 @@ +--[[ + Remove jobs from the specific set. + + Input: + KEYS[1] meta key + KEYS[2] migrations key + KEYS[3] prefix key + + ARGV[1] current major version + ARGV[2] timestamp + ARGV[3] current migration execution +]] +local rcall = redis.call + +local currentMajorVersion = rcall("HGET", KEYS[1], "mv") + +local function getCurrentMigrationNumber(migrationKey) + local lastExecutedMigration = rcall("LRANGE", migrationKey, -1, -1) + + if #lastExecutedMigration > 0 then + return tonumber(string.match(lastExecutedMigration[1], "(.*)-.*-.*")) + 1 + else + return 1 + end +end + +local function saveMigration(migrationKey, migrationNumber, timestamp, migrationName) + rcall("RPUSH", migrationKey, migrationNumber .. "-" .. timestamp .. "-" .. migrationName) + return migrationNumber + 1 +end + +if currentMajorVersion then + if currentMajorVersion == ARGV[1] then + return 0 + end +else + local currentMigrationNumber = getCurrentMigrationNumber(KEYS[2]) + if currentMigrationNumber == 1 then + -- delete deprecated priority + rcall("DEL", KEYS[3] .. "priority") + currentMigrationNumber = saveMigration(KEYS[2], currentMigrationNumber, ARGV[2], "removeDeprecatedPriorityKey") + end + + local currentMigrationExecutionNumber = tonumber(ARGV[3]) + if currentMigrationNumber == 2 then + -- remove legacy markers + if currentMigrationNumber >= currentMigrationExecutionNumber then + return 2 + else + currentMigrationNumber = saveMigration(KEYS[2], currentMigrationNumber, ARGV[2], "removeLegacyMarkers") + end + end + + if currentMigrationNumber == 3 then + -- migrate deprecated paused key + if currentMigrationNumber >= currentMigrationExecutionNumber then + return 3 + else + currentMigrationNumber = saveMigration(KEYS[2], currentMigrationNumber, ARGV[2], "migrateDeprecatedPausedKey") + end + end + + return 0 +end diff --git a/src/commands/repairPausedKey-2.lua b/src/commands/migrateDeprecatedPausedKey-2.lua similarity index 100% rename from src/commands/repairPausedKey-2.lua rename to src/commands/migrateDeprecatedPausedKey-2.lua diff --git a/tests/test_queue.ts b/tests/test_queue.ts index e9278743cf..ac8a2eb433 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -428,38 +428,6 @@ describe('queues', function () { }); }); - describe('.migrateDeprecatedPausedKey', () => { - it('moves jobs from paused to wait', async () => { - const client = await queue.client; - await client.lpush(`${prefix}:${queue.name}:paused`, 'a', 'b', 'c'); - await client.lpush(`${prefix}:${queue.name}:wait`, 'd', 'e', 'f'); - - await queue.migrateDeprecatedPausedKey(); - - const jobs = await client.lrange( - `${prefix}:${queue.name}:wait`, 0, -1 - ); - - expect(jobs).to.be.eql(['f', 'e', 'd', 'c', 'b', 'a']); - }); - }); - - describe('.removeLegacyMarkers', () => { - it('removes old markers', async () => { - const client = await queue.client; - await client.zadd(`${prefix}:${queue.name}:completed`, 1, '0:2'); - await client.zadd(`${prefix}:${queue.name}:failed`, 2, '0:1'); - await client.rpush(`${prefix}:${queue.name}:wait`, '0:0'); - - await queue.removeLegacyMarkers(); - - const keys = await client.keys(`${prefix}:${queue.name}:*`); - - // meta key - expect(keys.length).to.be.eql(1); - }); - }); - describe('.retryJobs', () => { it('retries all failed jobs by default', async () => { await queue.waitUntilReady(); diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 94b464eb2a..f1ee400136 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -4537,4 +4537,60 @@ describe('workers', function () { await worker.close(); }); + + describe('.executeMigrations', () => { + describe('.removeLegacyMarkers', () => { + it('removes old markers', async () => { + const client = await queue.client; + await client.zadd(`${prefix}:${queue.name}:completed`, 1, '0:2'); + await client.zadd(`${prefix}:${queue.name}:completed`, 1, '0:2'); + await client.zadd(`${prefix}:${queue.name}:failed`, 2, '0:1'); + await client.rpush(`${prefix}:${queue.name}:wait`, '0:0'); + await client.rpush(`${prefix}:${queue.name}:migrations`, '5:'); + + const worker = new Worker( + queueName, + async () => { + }, + { connection, prefix }, + ); + await worker.waitUntilReady(); + + await delay(500); + + const keys = await client.keys(`${prefix}:${queue.name}:*`); + + // meta key, stalled-check, migrations + expect(keys.length).to.be.eql(3); + await worker.close(); + }); + }); + + describe('.migrateDeprecatedPausedKey', () => { + it('moves jobs from paused to wait', async () => { + const client = await queue.client; + await client.lpush(`${prefix}:${queue.name}:paused`, 'a', 'b', 'c'); + await client.lpush(`${prefix}:${queue.name}:wait`, 'd', 'e', 'f'); + + const worker = new Worker( + queueName, + async () => { + }, + { connection, prefix, autorun:false }, + ); + await worker.waitUntilReady(); + await worker.pause(); + worker.run(); + + await delay(500); + + const jobs = await client.lrange( + `${prefix}:${queue.name}:wait`, 0, -1 + ); + + expect(jobs).to.be.eql(['f', 'e', 'd', 'c', 'b', 'a']); + await worker.close(); + }); + }); + }); }); From a8103c02e660c01d8b834d65ba7fad256bf6685a Mon Sep 17 00:00:00 2001 From: Arman Poghosyan Date: Fri, 27 Sep 2024 04:48:11 +0400 Subject: [PATCH 15/28] docs(flows): add missing word in flows guide (#2788) --- docs/gitbook/guide/flows/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gitbook/guide/flows/README.md b/docs/gitbook/guide/flows/README.md index b101f51f43..703d103caa 100644 --- a/docs/gitbook/guide/flows/README.md +++ b/docs/gitbook/guide/flows/README.md @@ -78,7 +78,7 @@ Note that the parent queue does not need to be the same queue as the one used fo {% endhint %} {% hint style="warning" %} -If a jobId option is provided, make that it does not contain a colon **:** as this is considered a separator. +If a jobId option is provided, make sure that it does not contain a colon **:** as this is considered a separator. {% endhint %} When the parent job is processed it is possible to access the results generated by its child jobs. For example, lets assume the following worker for the child jobs: From f5f7426c4f607ffbed45175d9e0a7a0971b1dc04 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 26 Sep 2024 21:37:58 -0500 Subject: [PATCH 16/28] test: update test cases --- src/commands/executeMigrations-3.lua | 2 +- tests/test_obliterate.ts | 14 +++++++++----- tests/test_worker.ts | 15 +++++---------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/commands/executeMigrations-3.lua b/src/commands/executeMigrations-3.lua index 22fbb363e1..1ef403629c 100644 --- a/src/commands/executeMigrations-3.lua +++ b/src/commands/executeMigrations-3.lua @@ -1,5 +1,5 @@ --[[ - Remove jobs from the specific set. + Execute migrations. Input: KEYS[1] meta key diff --git a/tests/test_obliterate.ts b/tests/test_obliterate.ts index 48967bd5df..2daac5a6c4 100644 --- a/tests/test_obliterate.ts +++ b/tests/test_obliterate.ts @@ -6,7 +6,7 @@ import { v4 } from 'uuid'; import { Queue, QueueEvents, FlowProducer, Worker, Job } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -describe('Obliterate', function () { +describe.only('Obliterate', function () { const redisHost = process.env.REDIS_HOST || 'localhost'; const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; @@ -74,7 +74,8 @@ describe('Obliterate', function () { await queue.obliterate(); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + // only migration key should be kept + expect(keys.length).to.be.eql(1); await worker.close(); }); @@ -164,7 +165,8 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + // only migration key should be kept + expect(keys.length).to.be.eql(1); await worker.close(); await flow.close(); @@ -375,7 +377,8 @@ describe('Obliterate', function () { await queue.obliterate({ force: true }); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + // only migration key should be kept + expect(keys.length).to.be.eql(1); await worker.close(); }); @@ -475,6 +478,7 @@ describe('Obliterate', function () { await queue.obliterate(); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}*`); - expect(keys.length).to.be.eql(0); + // only migration key should be kept + expect(keys.length).to.be.eql(1); }); }); diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 2e4003f352..83aaae74bc 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -47,7 +47,7 @@ describe('workers', function () { sandbox.restore(); await queue.close(); await queueEvents.close(); - await removeAllQueueData(new IORedis(redisHost), queueName); + //await removeAllQueueData(new IORedis(redisHost), queueName); }); afterAll(async function () { @@ -133,7 +133,7 @@ describe('workers', function () { const completedCount = await queue.getCompletedCount(); - expect(completedCount).to.be.equal(2); + expect(completedCount).to.be.equal(1); await worker.close(); }); @@ -532,8 +532,6 @@ describe('workers', function () { expect(job.data.foo).to.be.eql('bar'); } - expect(bclientSpy.callCount).to.be.equal(1); - await new Promise((resolve, reject) => { worker.on('completed', (_job: Job, _result: any) => { completedJobs++; @@ -545,7 +543,7 @@ describe('workers', function () { // Check moveToActive was called only concurrency times expect(spy.callCount).to.be.equal(concurrency + 1); - expect(bclientSpy.callCount).to.be.equal(3); + expect(bclientSpy.callCount).to.be.equal(2); await worker.close(); }); @@ -575,9 +573,7 @@ describe('workers', function () { expect(job.id).to.be.ok; expect(job.data.foo).to.be.eql('bar'); } - - expect(bclientSpy.callCount).to.be.equal(1); - + await new Promise((resolve, reject) => { worker.on('completed', (job: Job, result: any) => { completedJobs++; @@ -589,7 +585,7 @@ describe('workers', function () { // Check moveToActive was called numJobs + 2 times expect(spy.callCount).to.be.equal(numJobs + 2); - expect(bclientSpy.callCount).to.be.equal(3); + expect(bclientSpy.callCount).to.be.equal(2); await worker.close(); }); @@ -4541,7 +4537,6 @@ describe('workers', function () { await client.zadd(`${prefix}:${queue.name}:completed`, 1, '0:2'); await client.zadd(`${prefix}:${queue.name}:failed`, 2, '0:1'); await client.rpush(`${prefix}:${queue.name}:wait`, '0:0'); - await client.rpush(`${prefix}:${queue.name}:migrations`, '5:'); const worker = new Worker( queueName, From 9a3067b10fc611281879cb968f5debadaccaa385 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Fri, 27 Sep 2024 05:54:10 -0600 Subject: [PATCH 17/28] chore: remove only statement --- tests/test_obliterate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_obliterate.ts b/tests/test_obliterate.ts index 2daac5a6c4..c5d12f4e01 100644 --- a/tests/test_obliterate.ts +++ b/tests/test_obliterate.ts @@ -6,7 +6,7 @@ import { v4 } from 'uuid'; import { Queue, QueueEvents, FlowProducer, Worker, Job } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -describe.only('Obliterate', function () { +describe('Obliterate', function () { const redisHost = process.env.REDIS_HOST || 'localhost'; const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; From 77b81ba8bed2dc05d07aff2197faba00de35827b Mon Sep 17 00:00:00 2001 From: roggervalf Date: Fri, 27 Sep 2024 21:06:30 -0500 Subject: [PATCH 18/28] test: fix test cases --- tests/test_clean.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_clean.ts b/tests/test_clean.ts index 1e19b2a79f..f2104e0744 100644 --- a/tests/test_clean.ts +++ b/tests/test_clean.ts @@ -320,11 +320,11 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - // Expected keys: meta, id, stalled-check and events - expect(keys.length).to.be.eql(4); + // Expected keys: meta, id, stalled-check, migrations and events + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['meta', 'id', 'stalled-check', 'events']).to.include( + expect(['meta', 'id', 'stalled-check', 'migrations', 'events']).to.include( type, ); } @@ -446,8 +446,8 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - // Expected keys: meta, id, stalled-check, events, failed and 2 jobs - expect(keys.length).to.be.eql(7); + // Expected keys: meta, id, stalled-check, migrations, events, failed and 2 jobs + expect(keys.length).to.be.eql(8); const jobs = await queue.getJobCountByTypes('completed'); expect(jobs).to.be.equal(2); From 67e9306d3d4be97eb124ae5cfb9bc550cfa04edb Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Mon, 30 Sep 2024 13:28:15 +0200 Subject: [PATCH 19/28] chore: add POC for migrations --- src/classes/migrations.ts | 118 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/classes/migrations.ts diff --git a/src/classes/migrations.ts b/src/classes/migrations.ts new file mode 100644 index 0000000000..6214a886f5 --- /dev/null +++ b/src/classes/migrations.ts @@ -0,0 +1,118 @@ +import { RedisClient } from '../interfaces'; + +export interface MigrationOptions { + prefix: string; + queueName: string; +} + +export type MigrationFunction = ( + client: RedisClient, + opts: MigrationOptions, +) => Promise; + +export const hasPendingMigrations = async ( + client: RedisClient, + opts: MigrationOptions, +) => { + const migrationsKey = getRedisKeyFromOpts(opts, 'migrations'); + const existingMigrations = await client.zrange(migrationsKey, 0, -1); + return migrations.some( + migration => + !existingMigrations.includes(`${migration.version}-${migration.name}`), + ); +}; + +export const migrations: { + name: string; + version: string; + migrate: MigrationFunction; +}[] = [ + /* + * Example migration function + * + { + name: 'migrate-paused-jobs', + version: '6.0.0', + migrate: async (client: RedisClient, opts: MigrationOptions) => { + let cursor: number = 0; + do { + cursor = (await client.eval( + ` + local maxCount = tonumber(ARGV[1]) + local rcall = redis.call + + local hasJobs = rcall("EXISTS", KEYS[1]) == 1 + + if hasJobs then + local jobs = rcall('LRANGE', KEYS[1], 0, maxCount - 1) + rcall("RPUSH", KEYS[2], unpack(jobs)) + rcall("LTRIM", KEYS[1], #jobs, -1) + + if (maxCount - #jobs) <= 0 then + return 1 + end + end + + return 0`, + 2, + getRedisKeyFromOpts(opts, "paused"), + getRedisKeyFromOpts(opts, "wait"), + )) as number; + } while (cursor); + }, + }, + */ +]; + +/** + * Run Migrations. + * + * This method is used to run possibly existing migrations for the queue. + * + * Normally, if there are pending migrations, the Queue, Worker and QueueEvents instances + * will throw an error when they are instantiated. Use then this method to run the migrations + * before instantiating the instances. + * + * @param redisClient The Redis client instance + * @param opts The options for the migration + * + * @sa https://docs.bullmq.io/guide/migrations + */ +export const runMigrations = async ( + redisClient: RedisClient, + opts: { + prefix?: string; + queueName: string; + }, +) => { + const prefix = opts.prefix || 'bull'; + const migrationsKey = getRedisKeyFromOpts({ prefix, ...opts }, 'migrations'); + + // The migrations key is a ZSET with the migration timestamp as the score + for (const migration of migrations) { + const migrationId = `${migration.version}-${migration.name}`; + const pendingMigration = !!(await redisClient.zscore( + migrationsKey, + migrationId, + )); + if (pendingMigration) { + continue; + } + console.log(`[BULLMQ] Running migration ${migrationId}`); + try { + await migration.migrate(redisClient, { + prefix, + queueName: opts.queueName, + }); + await redisClient.zadd(migrationsKey, Date.now(), migrationId); + } catch (err) { + console.error(`[BULLMQ] Migration ${migrationId} failed: ${err}`); + break; + } + console.log(`[BULLMQ] Migration ${migrationId} completed`); + } +}; + +function getRedisKeyFromOpts(opts: MigrationOptions, key: string): string { + return `${opts.prefix}:${opts.queueName}:${key}`; +} From fcac0312bb3cdc78b1e521964b16a48488d2c0e3 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Mon, 30 Sep 2024 14:35:55 +0200 Subject: [PATCH 20/28] feat(queue-base): add migrations checker --- src/classes/migrations.ts | 2 +- src/classes/queue-base.ts | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/classes/migrations.ts b/src/classes/migrations.ts index 6214a886f5..6fdab34a68 100644 --- a/src/classes/migrations.ts +++ b/src/classes/migrations.ts @@ -10,7 +10,7 @@ export type MigrationFunction = ( opts: MigrationOptions, ) => Promise; -export const hasPendingMigrations = async ( +export const checkPendingMigrations = async ( client: RedisClient, opts: MigrationOptions, ) => { diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index 41097e966d..fcd315002e 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -11,6 +11,7 @@ import { RedisConnection } from './redis-connection'; import { Job } from './job'; import { KeysMap, QueueKeys } from './queue-keys'; import { Scripts } from './scripts'; +import { checkPendingMigrations } from './migrations'; /** * @class QueueBase @@ -21,6 +22,8 @@ import { Scripts } from './scripts'; * */ export class QueueBase extends EventEmitter implements MinimalQueue { + public readonly qualifiedName: string; + toKey: (type: string) => string; keys: KeysMap; closing: Promise | undefined; @@ -28,7 +31,8 @@ export class QueueBase extends EventEmitter implements MinimalQueue { protected closed: boolean = false; protected scripts: Scripts; protected connection: RedisConnection; - public readonly qualifiedName: string; + + private checkedPendingMigrations: boolean = false; /** * @@ -80,9 +84,26 @@ export class QueueBase extends EventEmitter implements MinimalQueue { /** * Returns a promise that resolves to a redis client. Normally used only by subclasses. + * This method will also check if there are pending migrations, if so it will throw an error. */ get client(): Promise { - return this.connection.client; + if (this.checkedPendingMigrations) { + return this.connection.client; + } else { + this.connection.client.then(client => { + return checkPendingMigrations(client, { + prefix: this.opts.prefix, + queueName: this.name, + }).then(hasPendingMigrations => { + if (hasPendingMigrations) { + throw new Error( + 'Queue has pending migrations. See https://docs.bullmq.io/guide/migrations', + ); + } + this.checkedPendingMigrations = true; + }); + }); + } } protected setScripts() { From 4cd34a1d2ee7342e67f430998cb95dd7f2a3a492 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Tue, 1 Oct 2024 12:14:00 -0500 Subject: [PATCH 21/28] refactor(script): consider using rename when no wait jobs --- src/commands/migrateDeprecatedPausedKey-2.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/commands/migrateDeprecatedPausedKey-2.lua b/src/commands/migrateDeprecatedPausedKey-2.lua index 8d3cfac384..ee72f3838f 100644 --- a/src/commands/migrateDeprecatedPausedKey-2.lua +++ b/src/commands/migrateDeprecatedPausedKey-2.lua @@ -12,9 +12,10 @@ local maxCount = tonumber(ARGV[1]) local rcall = redis.call -local hasJobs = rcall("EXISTS", KEYS[1]) == 1 +local hasJobsInPaused = rcall("EXISTS", KEYS[1]) == 1 +local hasJobsInWait = rcall("EXISTS", KEYS[2]) == 1 -if hasJobs then +if hasJobsInPaused and hasJobsInWait then local jobs = rcall('LRANGE', KEYS[1], 0, maxCount - 1) rcall("RPUSH", KEYS[2], unpack(jobs)) rcall("LTRIM", KEYS[1], #jobs, -1) @@ -22,6 +23,8 @@ if hasJobs then if (maxCount - #jobs) <= 0 then return 1 end +elseif hasJobsInPaused then + rcall("RENAME", KEYS[1], KEYS[2]) end return 0 From 50b0313055942fb3847cd5f2042af023f7a76ee1 Mon Sep 17 00:00:00 2001 From: Manuel Astudillo Date: Tue, 1 Oct 2024 22:23:16 +0200 Subject: [PATCH 22/28] fix: return client in get client in all paths --- src/classes/queue-base.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index fcd315002e..6ccadcda1d 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -90,7 +90,7 @@ export class QueueBase extends EventEmitter implements MinimalQueue { if (this.checkedPendingMigrations) { return this.connection.client; } else { - this.connection.client.then(client => { + return this.connection.client.then(client => { return checkPendingMigrations(client, { prefix: this.opts.prefix, queueName: this.name, @@ -101,6 +101,7 @@ export class QueueBase extends EventEmitter implements MinimalQueue { ); } this.checkedPendingMigrations = true; + return client; }); }); } From 92d91543046eca6aa91dd36e75284541d13ca696 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 16 Oct 2024 00:40:37 -0500 Subject: [PATCH 23/28] chore: add migrations in list --- src/classes/migrations.ts | 52 ++++++++++----------- src/classes/queue-base.ts | 12 +++-- src/classes/worker.ts | 30 ------------ tests/test_clean.ts | 26 +++++++---- tests/test_concurrency.ts | 2 + tests/test_migrations.ts | 96 ++++++++++++++++++++++++++++++++++++++ tests/test_obliterate.ts | 12 ++--- tests/test_queue.ts | 55 +++++++++++++++++----- tests/test_rate_limiter.ts | 3 ++ tests/test_worker.ts | 62 ++---------------------- 10 files changed, 204 insertions(+), 146 deletions(-) create mode 100644 tests/test_migrations.ts diff --git a/src/classes/migrations.ts b/src/classes/migrations.ts index 6fdab34a68..5e87b9a339 100644 --- a/src/classes/migrations.ts +++ b/src/classes/migrations.ts @@ -27,41 +27,39 @@ export const migrations: { version: string; migrate: MigrationFunction; }[] = [ - /* - * Example migration function - * + { + name: 'remove-legacy-markers', + version: '6.0.0', + migrate: async (client: RedisClient, opts: MigrationOptions) => { + const keys: (string | number)[] = [ + getRedisKeyFromOpts(opts, 'wait'), + getRedisKeyFromOpts(opts, 'paused'), + getRedisKeyFromOpts(opts, 'meta'), + getRedisKeyFromOpts(opts, 'completed'), + getRedisKeyFromOpts(opts, 'failed'), + ]; + const args = [getRedisKeyFromOpts(opts, '')]; + + await (client).removeLegacyMarkers(keys.concat(args)); + }, + }, { name: 'migrate-paused-jobs', version: '6.0.0', migrate: async (client: RedisClient, opts: MigrationOptions) => { - let cursor: number = 0; + let cursor = 0; do { - cursor = (await client.eval( - ` - local maxCount = tonumber(ARGV[1]) - local rcall = redis.call - - local hasJobs = rcall("EXISTS", KEYS[1]) == 1 - - if hasJobs then - local jobs = rcall('LRANGE', KEYS[1], 0, maxCount - 1) - rcall("RPUSH", KEYS[2], unpack(jobs)) - rcall("LTRIM", KEYS[1], #jobs, -1) - - if (maxCount - #jobs) <= 0 then - return 1 - end - end - - return 0`, - 2, - getRedisKeyFromOpts(opts, "paused"), - getRedisKeyFromOpts(opts, "wait"), - )) as number; + const keys: (string | number)[] = [ + getRedisKeyFromOpts(opts, 'paused'), + getRedisKeyFromOpts(opts, 'wait'), + ]; + const args = [1000]; + cursor = await (client).migrateDeprecatedPausedKey( + keys.concat(args), + ); } while (cursor); }, }, - */ ]; /** diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index 6ccadcda1d..4d19b690cf 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -11,7 +11,7 @@ import { RedisConnection } from './redis-connection'; import { Job } from './job'; import { KeysMap, QueueKeys } from './queue-keys'; import { Scripts } from './scripts'; -import { checkPendingMigrations } from './migrations'; +import { checkPendingMigrations, runMigrations } from './migrations'; /** * @class QueueBase @@ -96,9 +96,13 @@ export class QueueBase extends EventEmitter implements MinimalQueue { queueName: this.name, }).then(hasPendingMigrations => { if (hasPendingMigrations) { - throw new Error( - 'Queue has pending migrations. See https://docs.bullmq.io/guide/migrations', - ); + return runMigrations(client, { + prefix: this.opts.prefix, + queueName: this.name, + }).then(() => { + this.checkedPendingMigrations = true; + return client; + }); } this.checkedPendingMigrations = true; return client; diff --git a/src/classes/worker.ts b/src/classes/worker.ts index 3cf9a9570f..57156d9414 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -436,7 +436,6 @@ export class Worker< return; } - await this.executeMigrations(); await this.startStalledCheckTimer(); const jobsInProgress = new Set<{ job: Job; ts: number }>(); @@ -1121,33 +1120,4 @@ will never work with more accuracy than 1ms. */ ) { return this.scripts.moveJobFromActiveToWait(job.id, token); } - - /** - * Execute migration scripts while migrating to breaking changes. - */ - async executeMigrations( - ): Promise { - let currentMigrationExecution=1; - - while (currentMigrationExecution > 0) { - currentMigrationExecution = await this.scripts.executeMigrations( - currentMigrationExecution - ); - switch (currentMigrationExecution) { - case 2:{ - await this.scripts.removeLegacyMarkers(); - currentMigrationExecution++; - break; - } - case 3:{ - let cursor = 0; - do { - cursor = await this.scripts.migrateDeprecatedPausedKey(5000); - } while (cursor); - currentMigrationExecution++; - break; - } - } - } - } } diff --git a/tests/test_clean.ts b/tests/test_clean.ts index f2104e0744..1575ed60f8 100644 --- a/tests/test_clean.ts +++ b/tests/test_clean.ts @@ -269,10 +269,12 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['meta', 'events', 'marker', 'id']).to.include(type); + expect(['meta', 'events', 'marker', 'migrations', 'id']).to.include( + type, + ); } const countAfterEmpty = await queue.count(); @@ -324,9 +326,13 @@ describe('Cleaner', () => { expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['meta', 'id', 'stalled-check', 'migrations', 'events']).to.include( - type, - ); + expect([ + 'meta', + 'id', + 'stalled-check', + 'migrations', + 'events', + ]).to.include(type); } const jobs = await queue.getJobCountByTypes('completed'); @@ -492,7 +498,7 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(6); + expect(keys.length).to.be.eql(7); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(1); @@ -633,10 +639,12 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queueName}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['meta', 'events', 'marker', 'id']).to.include(type); + expect(['meta', 'migrations', 'events', 'marker', 'id']).to.include( + type, + ); } const eventsCount = await client.xlen( @@ -695,7 +703,7 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queueName}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(6); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); diff --git a/tests/test_concurrency.ts b/tests/test_concurrency.ts index 4834aa4bf5..961cd67ef9 100644 --- a/tests/test_concurrency.ts +++ b/tests/test_concurrency.ts @@ -150,6 +150,7 @@ describe('Concurrency', () => { }); const queueEvents = new QueueEvents(queueName, { connection, prefix }); await queueEvents.waitUntilReady(); + await queue.waitUntilReady(); await queue.setGlobalConcurrency(1); const worker = new Worker( @@ -233,6 +234,7 @@ describe('Concurrency', () => { prefix, }); const queueEvents = new QueueEvents(queueName, { connection, prefix }); + await queue.waitUntilReady(); await queueEvents.waitUntilReady(); await queue.setGlobalConcurrency(1); diff --git a/tests/test_migrations.ts b/tests/test_migrations.ts new file mode 100644 index 0000000000..b93adfa118 --- /dev/null +++ b/tests/test_migrations.ts @@ -0,0 +1,96 @@ +import { expect } from 'chai'; +import { default as IORedis } from 'ioredis'; +import { describe, beforeEach, it, before, after as afterAll } from 'mocha'; +import * as sinon from 'sinon'; +import { v4 } from 'uuid'; +import { Queue } from '../src/classes'; +import { removeAllQueueData } from '../src/utils'; + +describe('migrations', function () { + const redisHost = process.env.REDIS_HOST || 'localhost'; + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; + + const sandbox = sinon.createSandbox(); + + let queue: Queue; + let queueName: string; + + let connection; + before(async function () { + connection = new IORedis(redisHost, { maxRetriesPerRequest: null }); + }); + + beforeEach(async function () { + queueName = `test-${v4()}`; + queue = new Queue(queueName, { connection, prefix }); + await queue.waitUntilReady(); + }); + + afterEach(async function () { + sandbox.restore(); + await queue.close(); + await removeAllQueueData(new IORedis(redisHost), queueName); + }); + + afterAll(async function () { + await connection.quit(); + }); + + describe('execute migrations', () => { + describe('removeLegacyMarkers', () => { + it('removes old markers', async () => { + const client = await queue.client; + const queueName2 = `test-${v4()}`; + const completedKey = `${prefix}:${queueName2}:completed`; + const failedKey = `${prefix}:${queueName2}:failed`; + const waitingKey = `${prefix}:${queueName2}:wait`; + await client.zadd(completedKey, 1, '0:2'); + await client.zadd(completedKey, 1, '0:2'); + await client.zadd(failedKey, 2, '0:1'); + await client.rpush(waitingKey, '0:0'); + + const queue2 = new Queue(queueName2, { connection, prefix }); + + await queue2.waitUntilReady(); + + await queue2.waitUntilReady(); + + const keys = await client.keys(`${prefix}:${queueName2}:*`); + + // meta key, migrations + expect(keys.length).to.be.eql(2); + + const completedCount = await client.zcard(completedKey); + expect(completedCount).to.be.eql(0); + + const failedCount = await client.zcard(failedKey); + expect(failedCount).to.be.eql(0); + + const waitingCount = await client.llen(waitingKey); + expect(waitingCount).to.be.eql(0); + + await queue2.close(); + await removeAllQueueData(new IORedis(redisHost), queueName2); + }); + }); + + describe('migratePausedKey', () => { + it('moves jobs from paused to wait', async () => { + const client = await queue.client; + const queueName2 = `test-${v4()}`; + await client.lpush(`${prefix}:${queueName2}:paused`, 'a', 'b', 'c'); + await client.lpush(`${prefix}:${queueName2}:wait`, 'd', 'e', 'f'); + + const queue2 = new Queue(queueName2, { connection, prefix }); + + await queue2.waitUntilReady(); + + const jobs = await client.lrange(`${prefix}:${queueName2}:wait`, 0, -1); + + expect(jobs).to.be.eql(['f', 'e', 'd', 'c', 'b', 'a']); + await queue2.close(); + await removeAllQueueData(new IORedis(redisHost), queueName2); + }); + }); + }); +}); diff --git a/tests/test_obliterate.ts b/tests/test_obliterate.ts index c5d12f4e01..7fd2562054 100644 --- a/tests/test_obliterate.ts +++ b/tests/test_obliterate.ts @@ -44,7 +44,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + expect(keys.length).to.be.eql(1); }); it('should obliterate a queue with jobs in different statuses', async () => { @@ -107,7 +107,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + expect(keys.length).to.be.eql(1); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); @@ -206,7 +206,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(3); + expect(keys.length).to.be.eql(4); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(1); @@ -248,7 +248,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queueName}:*`); - expect(keys.length).to.be.eql(0); + expect(keys.length).to.be.eql(1); const eventsCount = await client.xlen( `${prefix}:${parentQueueName}:events`, @@ -297,7 +297,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + expect(keys.length).to.be.eql(1); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); @@ -402,7 +402,7 @@ describe('Obliterate', function () { await queue.obliterate(); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(0); + expect(keys.length).to.be.eql(1); }); it('should remove job logs', async () => { diff --git a/tests/test_queue.ts b/tests/test_queue.ts index 1dd69b7b84..2c15b74781 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -108,11 +108,18 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(6); for (const key of keys) { const type = key.split(':')[2]; - expect(['marker', 'events', 'meta', 'pc', 'id']).to.include(type); + expect([ + 'marker', + 'migrations', + 'events', + 'meta', + 'pc', + 'id', + ]).to.include(type); } }).timeout(10000); @@ -143,10 +150,16 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['events', 'meta', 'id', 'marker']).to.include(type); + expect([ + 'events', + 'meta', + 'migrations', + 'id', + 'marker', + ]).to.include(type); } const countAfterEmpty = await queue.count(); @@ -177,10 +190,16 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['id', 'meta', 'marker', 'events']).to.include(type); + expect([ + 'id', + 'meta', + 'marker', + 'migrations', + 'events', + ]).to.include(type); } const countAfterEmpty = await queue.count(); @@ -223,7 +242,7 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(6); + expect(keys.length).to.be.eql(7); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(1); @@ -265,10 +284,16 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['id', 'meta', 'events', 'marker']).to.include(type); + expect([ + 'id', + 'meta', + 'migrations', + 'events', + 'marker', + ]).to.include(type); } const countAfterEmpty = await queue.count(); @@ -316,10 +341,16 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; - expect(['id', 'meta', 'events', 'marker']).to.include(type); + expect([ + 'id', + 'meta', + 'migrations', + 'events', + 'marker', + ]).to.include(type); } const countAfterEmpty = await queue.count(); @@ -397,7 +428,7 @@ describe('queues', function () { describe('when queue is paused', () => { it('clean queue including paused jobs', async () => { const maxJobs = 50; - const added:Promise[] = []; + const added: Promise[] = []; await queue.pause(); for (let i = 1; i <= maxJobs; i++) { diff --git a/tests/test_rate_limiter.ts b/tests/test_rate_limiter.ts index aa7fdf606f..c23ac8fefc 100644 --- a/tests/test_rate_limiter.ts +++ b/tests/test_rate_limiter.ts @@ -28,6 +28,7 @@ describe('Rate Limiter', function () { queueName = `test-${v4()}`; queue = new Queue(queueName, { connection, prefix }); queueEvents = new QueueEvents(queueName, { connection, prefix }); + await queue.waitUntilReady(); await queueEvents.waitUntilReady(); }); @@ -691,6 +692,8 @@ describe('Rate Limiter', function () { }, ); + await worker.waitUntilReady(); + const result = new Promise((resolve, reject) => { queueEvents.on( 'completed', diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 83aaae74bc..4e9c527d6f 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -40,6 +40,7 @@ describe('workers', function () { queueName = `test-${v4()}`; queue = new Queue(queueName, { connection, prefix }); queueEvents = new QueueEvents(queueName, { connection, prefix }); + await queue.waitUntilReady(); await queueEvents.waitUntilReady(); }); @@ -47,7 +48,7 @@ describe('workers', function () { sandbox.restore(); await queue.close(); await queueEvents.close(); - //await removeAllQueueData(new IORedis(redisHost), queueName); + await removeAllQueueData(new IORedis(redisHost), queueName); }); afterAll(async function () { @@ -543,7 +544,7 @@ describe('workers', function () { // Check moveToActive was called only concurrency times expect(spy.callCount).to.be.equal(concurrency + 1); - expect(bclientSpy.callCount).to.be.equal(2); + expect(bclientSpy.callCount).to.be.equal(3); await worker.close(); }); @@ -573,7 +574,7 @@ describe('workers', function () { expect(job.id).to.be.ok; expect(job.data.foo).to.be.eql('bar'); } - + await new Promise((resolve, reject) => { worker.on('completed', (job: Job, result: any) => { completedJobs++; @@ -4528,59 +4529,4 @@ describe('workers', function () { await worker.close(); }); - - describe('.executeMigrations', () => { - describe('.removeLegacyMarkers', () => { - it('removes old markers', async () => { - const client = await queue.client; - await client.zadd(`${prefix}:${queue.name}:completed`, 1, '0:2'); - await client.zadd(`${prefix}:${queue.name}:completed`, 1, '0:2'); - await client.zadd(`${prefix}:${queue.name}:failed`, 2, '0:1'); - await client.rpush(`${prefix}:${queue.name}:wait`, '0:0'); - - const worker = new Worker( - queueName, - async () => { - }, - { connection, prefix }, - ); - await worker.waitUntilReady(); - - await delay(500); - - const keys = await client.keys(`${prefix}:${queue.name}:*`); - - // meta key, stalled-check, migrations - expect(keys.length).to.be.eql(3); - await worker.close(); - }); - }); - - describe('.migrateDeprecatedPausedKey', () => { - it('moves jobs from paused to wait', async () => { - const client = await queue.client; - await client.lpush(`${prefix}:${queue.name}:paused`, 'a', 'b', 'c'); - await client.lpush(`${prefix}:${queue.name}:wait`, 'd', 'e', 'f'); - - const worker = new Worker( - queueName, - async () => { - }, - { connection, prefix, autorun:false }, - ); - await worker.waitUntilReady(); - await worker.pause(); - worker.run(); - - await delay(500); - - const jobs = await client.lrange( - `${prefix}:${queue.name}:wait`, 0, -1 - ); - - expect(jobs).to.be.eql(['f', 'e', 'd', 'c', 'b', 'a']); - await worker.close(); - }); - }); - }); }); From 268b9d46f7666077f446cd6e5eaa8d01c51cd5f9 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 16 Oct 2024 09:13:01 -0500 Subject: [PATCH 24/28] test: fix test cases --- tests/test_worker.ts | 42 +----------------------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 4e9c527d6f..7cacde17da 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -100,46 +100,6 @@ describe('workers', function () { await worker.close(); }); - describe('when legacy marker is present', () => { - it('does not get stuck', async () => { - const client = await queue.client; - await client.rpush(`${prefix}:${queue.name}:wait`, '0:0'); - - const worker = new Worker( - queueName, - async () => { - await delay(200); - }, - { autorun: false, connection, prefix }, - ); - await worker.waitUntilReady(); - - const secondJob = await queue.add('test', { foo: 'bar' }); - - const completing = new Promise((resolve, reject) => { - worker.on('completed', async job => { - try { - if (job.id === secondJob.id) { - resolve(); - } - } catch (err) { - reject(err); - } - }); - }); - - worker.run(); - - await completing; - - const completedCount = await queue.getCompletedCount(); - - expect(completedCount).to.be.equal(1); - - await worker.close(); - }); - }); - it('process several jobs serially', async () => { let counter = 1; const maxJobs = 35; @@ -586,7 +546,7 @@ describe('workers', function () { // Check moveToActive was called numJobs + 2 times expect(spy.callCount).to.be.equal(numJobs + 2); - expect(bclientSpy.callCount).to.be.equal(2); + expect(bclientSpy.callCount).to.be.equal(3); await worker.close(); }); From 6e20049e22a4d010b2581eaf2c12ee2127b8bb7d Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 16 Oct 2024 09:23:01 -0500 Subject: [PATCH 25/28] docs(guide): update migrations for v6 --- docs/gitbook/guide/migrations/v6.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/gitbook/guide/migrations/v6.md b/docs/gitbook/guide/migrations/v6.md index 558e510121..63fbc9b2dc 100644 --- a/docs/gitbook/guide/migrations/v6.md +++ b/docs/gitbook/guide/migrations/v6.md @@ -6,19 +6,18 @@ description: Tips and hints on how to migrate to v6. ## Migration of deprecated paused key -If you have paused queues after upgrading to this version. You must use **migrateDeprecatedPausedKey** from your queue instances in order to move your jobs from this state to wait state. +If you have paused queues after upgrading to this version. These jobs will be moved to wait state when initializing any of our instances (Worker, Queue, QueueEvents or FlowProducer). Paused key is not longer needed as this state is already represented by queue meta key. It also improve the process of pausing or resuming a queue as we don't need to rename any key. ## Remove legacy markers -When migrating from versions before v5, you must use **removeLegacyMarkers** method from queue your queue instances. - +When migrating from versions before v5. It's recommended to do this process: 1. Pause your queues. 2. Upgrade to v6. -3. Execute **removeLegacyMarkers** in each queue. +3. Instantiate any instance where migrations will be executed. 4. Resume your queues. This way you will prevent that your workers pick a legacy marker. From d0c4015ff486614d61ceb7064c40999aa01da1a7 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 16 Oct 2024 09:33:53 -0500 Subject: [PATCH 26/28] test: waitUntilReady in test cases --- src/classes/queue.ts | 6 ++---- tests/test_clean.ts | 2 +- tests/test_sandboxed_process.ts | 2 ++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/classes/queue.ts b/src/classes/queue.ts index 64b34cf22d..84dc32537e 100644 --- a/src/classes/queue.ts +++ b/src/classes/queue.ts @@ -715,12 +715,10 @@ export class Queue< * * @param maxCount - Max quantity of jobs to be moved to wait per iteration. */ - async migrateDeprecatedPausedKey(maxCount: number = 1000): Promise { + async migrateDeprecatedPausedKey(maxCount = 1000): Promise { let cursor = 0; do { - cursor = await this.scripts.migrateDeprecatedPausedKey( - maxCount - ); + cursor = await this.scripts.migrateDeprecatedPausedKey(maxCount); } while (cursor); } } diff --git a/tests/test_clean.ts b/tests/test_clean.ts index 1575ed60f8..13f50ad65f 100644 --- a/tests/test_clean.ts +++ b/tests/test_clean.ts @@ -28,8 +28,8 @@ describe('Cleaner', () => { queueName = `test-${v4()}`; queue = new Queue(queueName, { connection, prefix }); queueEvents = new QueueEvents(queueName, { connection, prefix }); - await queueEvents.waitUntilReady(); await queue.waitUntilReady(); + await queueEvents.waitUntilReady(); }); afterEach(async function () { diff --git a/tests/test_sandboxed_process.ts b/tests/test_sandboxed_process.ts index c34bd7d6ee..f3efbaf40a 100644 --- a/tests/test_sandboxed_process.ts +++ b/tests/test_sandboxed_process.ts @@ -28,6 +28,7 @@ describe('Sandboxed process using child processes', () => { queueName = `test-${v4()}`; queue = new Queue(queueName, { connection, prefix }); queueEvents = new QueueEvents(queueName, { connection, prefix }); + await queue.waitUntilReady(); await queueEvents.waitUntilReady(); }); @@ -696,6 +697,7 @@ function sandboxProcessTests( drainDelay: 1, useWorkerThreads, }); + await worker.waitUntilReady(); const completing = new Promise((resolve, reject) => { worker.on('completed', async (job: Job, value: any) => { From fd3ba1764484d36127ff6927e8a02e18fc7ca026 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 16 Oct 2024 20:11:37 -0500 Subject: [PATCH 27/28] feat: add skipMigrationsExecution option --- src/classes/queue-base.ts | 23 +++++++++++------ src/interfaces/queue-options.ts | 6 +++++ tests/test_clean.ts | 34 ++++++++++--------------- tests/test_migrations.ts | 12 +++++++-- tests/test_obliterate.ts | 24 +++++++++--------- tests/test_queue.ts | 44 ++++++++------------------------- tests/test_sandboxed_process.ts | 1 + tests/test_worker.ts | 2 +- 8 files changed, 68 insertions(+), 78 deletions(-) diff --git a/src/classes/queue-base.ts b/src/classes/queue-base.ts index 4d19b690cf..cc96c2e743 100644 --- a/src/classes/queue-base.ts +++ b/src/classes/queue-base.ts @@ -96,13 +96,22 @@ export class QueueBase extends EventEmitter implements MinimalQueue { queueName: this.name, }).then(hasPendingMigrations => { if (hasPendingMigrations) { - return runMigrations(client, { - prefix: this.opts.prefix, - queueName: this.name, - }).then(() => { - this.checkedPendingMigrations = true; - return client; - }); + if ( + this.opts.skipMigrationsExecution || + typeof this.opts.skipMigrationsExecution === 'undefined' + ) { + console.error( + 'Queue has pending migrations. See https://docs.bullmq.io/guide/migrations', + ); + } else { + return runMigrations(client, { + prefix: this.opts.prefix, + queueName: this.name, + }).then(() => { + this.checkedPendingMigrations = true; + return client; + }); + } } this.checkedPendingMigrations = true; return client; diff --git a/src/interfaces/queue-options.ts b/src/interfaces/queue-options.ts index c8f00bd78c..844992c5d0 100644 --- a/src/interfaces/queue-options.ts +++ b/src/interfaces/queue-options.ts @@ -31,6 +31,12 @@ export interface QueueBaseOptions { * @defaultValue false */ skipVersionCheck?: boolean; + + /** + * Avoid migrations execution. + * @defaultValue true + */ + skipMigrationsExecution?: boolean; } /** diff --git a/tests/test_clean.ts b/tests/test_clean.ts index 13f50ad65f..f330e09962 100644 --- a/tests/test_clean.ts +++ b/tests/test_clean.ts @@ -269,12 +269,10 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect(['meta', 'events', 'marker', 'migrations', 'id']).to.include( - type, - ); + expect(['meta', 'events', 'marker', 'id']).to.include(type); } const countAfterEmpty = await queue.count(); @@ -322,17 +320,13 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - // Expected keys: meta, id, stalled-check, migrations and events - expect(keys.length).to.be.eql(5); + // Expected keys: meta, id, stalled-check and events + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect([ - 'meta', - 'id', - 'stalled-check', - 'migrations', - 'events', - ]).to.include(type); + expect(['meta', 'id', 'stalled-check', 'events']).to.include( + type, + ); } const jobs = await queue.getJobCountByTypes('completed'); @@ -452,8 +446,8 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - // Expected keys: meta, id, stalled-check, migrations, events, failed and 2 jobs - expect(keys.length).to.be.eql(8); + // Expected keys: meta, id, stalled-check, events, failed and 2 jobs + expect(keys.length).to.be.eql(7); const jobs = await queue.getJobCountByTypes('completed'); expect(jobs).to.be.equal(2); @@ -498,7 +492,7 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(7); + expect(keys.length).to.be.eql(6); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(1); @@ -639,12 +633,10 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queueName}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect(['meta', 'migrations', 'events', 'marker', 'id']).to.include( - type, - ); + expect(['meta', 'events', 'marker', 'id']).to.include(type); } const eventsCount = await client.xlen( @@ -703,7 +695,7 @@ describe('Cleaner', () => { const client = await queue.client; const keys = await client.keys(`${prefix}:${queueName}:*`); - expect(keys.length).to.be.eql(6); + expect(keys.length).to.be.eql(5); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); diff --git a/tests/test_migrations.ts b/tests/test_migrations.ts index b93adfa118..9d961ed074 100644 --- a/tests/test_migrations.ts +++ b/tests/test_migrations.ts @@ -49,7 +49,11 @@ describe('migrations', function () { await client.zadd(failedKey, 2, '0:1'); await client.rpush(waitingKey, '0:0'); - const queue2 = new Queue(queueName2, { connection, prefix }); + const queue2 = new Queue(queueName2, { + connection, + prefix, + skipMigrationsExecution: false, + }); await queue2.waitUntilReady(); @@ -81,7 +85,11 @@ describe('migrations', function () { await client.lpush(`${prefix}:${queueName2}:paused`, 'a', 'b', 'c'); await client.lpush(`${prefix}:${queueName2}:wait`, 'd', 'e', 'f'); - const queue2 = new Queue(queueName2, { connection, prefix }); + const queue2 = new Queue(queueName2, { + connection, + prefix, + skipMigrationsExecution: false, + }); await queue2.waitUntilReady(); diff --git a/tests/test_obliterate.ts b/tests/test_obliterate.ts index 7fd2562054..89b8223b5a 100644 --- a/tests/test_obliterate.ts +++ b/tests/test_obliterate.ts @@ -44,7 +44,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); }); it('should obliterate a queue with jobs in different statuses', async () => { @@ -74,8 +74,8 @@ describe('Obliterate', function () { await queue.obliterate(); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - // only migration key should be kept - expect(keys.length).to.be.eql(1); + + expect(keys.length).to.be.eql(0); await worker.close(); }); @@ -107,7 +107,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); @@ -165,8 +165,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - // only migration key should be kept - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); await worker.close(); await flow.close(); @@ -206,7 +205,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(4); + expect(keys.length).to.be.eql(3); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(1); @@ -248,7 +247,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queueName}:*`); - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); const eventsCount = await client.xlen( `${prefix}:${parentQueueName}:events`, @@ -297,7 +296,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(0); @@ -378,7 +377,7 @@ describe('Obliterate', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); // only migration key should be kept - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); await worker.close(); }); @@ -402,7 +401,7 @@ describe('Obliterate', function () { await queue.obliterate(); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); }); it('should remove job logs', async () => { @@ -478,7 +477,6 @@ describe('Obliterate', function () { await queue.obliterate(); const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}*`); - // only migration key should be kept - expect(keys.length).to.be.eql(1); + expect(keys.length).to.be.eql(0); }); }); diff --git a/tests/test_queue.ts b/tests/test_queue.ts index 2c15b74781..f05443f566 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -108,7 +108,7 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(6); + expect(keys.length).to.be.eql(5); for (const key of keys) { const type = key.split(':')[2]; @@ -150,16 +150,10 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect([ - 'events', - 'meta', - 'migrations', - 'id', - 'marker', - ]).to.include(type); + expect(['events', 'meta', 'id', 'marker']).to.include(type); } const countAfterEmpty = await queue.count(); @@ -190,16 +184,10 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect([ - 'id', - 'meta', - 'marker', - 'migrations', - 'events', - ]).to.include(type); + expect(['id', 'meta', 'marker', 'events']).to.include(type); } const countAfterEmpty = await queue.count(); @@ -242,7 +230,7 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(7); + expect(keys.length).to.be.eql(6); const countAfterEmpty = await queue.count(); expect(countAfterEmpty).to.be.eql(1); @@ -284,16 +272,10 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect([ - 'id', - 'meta', - 'migrations', - 'events', - 'marker', - ]).to.include(type); + expect(['id', 'meta', 'events', 'marker']).to.include(type); } const countAfterEmpty = await queue.count(); @@ -341,16 +323,10 @@ describe('queues', function () { const client = await queue.client; const keys = await client.keys(`${prefix}:${queue.name}:*`); - expect(keys.length).to.be.eql(5); + expect(keys.length).to.be.eql(4); for (const key of keys) { const type = key.split(':')[2]; - expect([ - 'id', - 'meta', - 'migrations', - 'events', - 'marker', - ]).to.include(type); + expect(['id', 'meta', 'events', 'marker']).to.include(type); } const countAfterEmpty = await queue.count(); diff --git a/tests/test_sandboxed_process.ts b/tests/test_sandboxed_process.ts index f3efbaf40a..0c900a4ce1 100644 --- a/tests/test_sandboxed_process.ts +++ b/tests/test_sandboxed_process.ts @@ -463,6 +463,7 @@ function sandboxProcessTests( resolve(); }); }); + await worker.waitUntilReady(); const inspect = stderr.inspect(); await queue.add('test', { foo: 'bar' }); diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 7cacde17da..c435ff13a9 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -546,7 +546,7 @@ describe('workers', function () { // Check moveToActive was called numJobs + 2 times expect(spy.callCount).to.be.equal(numJobs + 2); - expect(bclientSpy.callCount).to.be.equal(3); + expect(bclientSpy.callCount).to.be.equal(2); await worker.close(); }); From 82da69a190e2798660eb9766ab52e7fee76d399a Mon Sep 17 00:00:00 2001 From: roggervalf Date: Wed, 16 Oct 2024 21:08:26 -0500 Subject: [PATCH 28/28] docs: update v6 clarification for skipMigrationsExecution --- docs/gitbook/guide/migrations/v6.md | 2 ++ tests/test_worker.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/gitbook/guide/migrations/v6.md b/docs/gitbook/guide/migrations/v6.md index 63fbc9b2dc..21718736ff 100644 --- a/docs/gitbook/guide/migrations/v6.md +++ b/docs/gitbook/guide/migrations/v6.md @@ -4,6 +4,8 @@ description: Tips and hints on how to migrate to v6. # Migration to v6 +Make sure to pass **skipMigrationsExecution** option in any of our instances as false in order to execute all necessary changes when coming from an older version + ## Migration of deprecated paused key If you have paused queues after upgrading to this version. These jobs will be moved to wait state when initializing any of our instances (Worker, Queue, QueueEvents or FlowProducer). diff --git a/tests/test_worker.ts b/tests/test_worker.ts index c435ff13a9..7cacde17da 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -546,7 +546,7 @@ describe('workers', function () { // Check moveToActive was called numJobs + 2 times expect(spy.callCount).to.be.equal(numJobs + 2); - expect(bclientSpy.callCount).to.be.equal(2); + expect(bclientSpy.callCount).to.be.equal(3); await worker.close(); });