将表用作没有 sp_getapplock/sp_releaseapplock 的队列

Question

我有一个需要执行的命令列表，所有这些命令都包含在我命名为 的表中myQueue。这个表有点独特，因为一些命令应该组合在一起，以便它们的执行按顺序执行，而不是并发执行，因为同时执行它们会导致不需要的数据工件和错误。因此，队列不能以典型的FIFO / LIFO方式分类，因为出队顺序是在运行时确定的。

总结一下：

1. 命名的表myQueue将充当命令队列（其中出队顺序在运行时确定）
2. 命令以随机方式添加到表中
3. 命令可以归入组，如果是，则必须由单个工作线程以有序、顺序的方式执行
4. 当命令出列时，可以运行任意数量的工作线程
5. 出列是通过 aUPDATE而不是 a执行的，DELETE因为此表用于所述命令的历史性能报告

我目前的方法是通过sp_getapplock/sp_releaseapplock调用使用显式互斥逻辑来迭代这个表。虽然这按预期工作，但该方法会生成足够的锁定，因此在任何给定时间都无法在队列上迭代大量工作线程。在阅读了 Remus Rusanu关于该主题的优秀博客文章后，我决定尝试使用表格提示，希望可以进一步优化我的方法。

我将包含下面的测试代码，但总结一下我的结果，使用表提示和消除对sp_getapplock/ 的调用的缺点sp_releaseapplock最多会导致以下三种不良行为：

1. 死锁
2. 多个线程执行包含在单个组中的命令
3. 一组命令中缺少线程分配

不过，从积极的方面来说，当代码适应死锁时（例如，重试当前包含的违规操作），不使用sp_getapplock/sp_releaseapplock且不会表现出不良行为的方法 2 和 3 的执行速度至少是两倍，如果不是更快的话.

我希望有人会指出我没有正确构建出列语句，这样我仍然可以继续使用表提示。 如果那不起作用，那就这样吧，但我想看看它是否可以做到同样的事情。

可以使用以下代码设置测试。

该myQueue表的创建和人口与命令类似于够我的工作量：

CREATE TABLE myQueue
(
    ID INT  IDENTITY (1,1) PRIMARY KEY CLUSTERED,
    Main    INT,
    Sub     INT,
    Detail  INT,
    Command VARCHAR(MAX),
    Thread  INT,
    StartDT DATETIME2,
    EndDT   DATETIME2
)
GO
INSERT INTO myQueue WITH (TABLOCKX) (Main, Sub, Detail, Command)
SELECT  ABS(CHECKSUM(NEWID()) % 200),
        ABS(CHECKSUM(NEWID()) % 1280),
        ABS(CHECKSUM(NEWID())),
        'WAITFOR DELAY ''00:00:00.01'''
FROM sys.types t1 CROSS JOIN 
     sys.types t2 CROSS JOIN
     sys.types t3 CROSS JOIN
     (VALUES (1), (2)) t4(x)
GO

CREATE NONCLUSTERED INDEX [IX_myQueue_Update]
ON [dbo].[myQueue] ([Main],[Sub])
INCLUDE (Thread, EndDT)
GO

工作线程都遵循相同的逻辑。我建议如果你在本地运行这个，你只需将这段代码复制到单独的查询窗口中并同时运行每个查询，确保所有工作线程都遵循相同的锁定方法（有 7 个埋在注释中并被注释包围）块）：

SET NOCOUNT ON
DECLARE @updOUT TABLE
(
    Main    INT,
    Sub     INT
)
-- Update @CurrentThread as a unique ID, I tend to
SET NOCOUNT ON
DECLARE @updOUT TABLE
(
    Main    INT,
    Sub     INT
)
-- @CurrentThread should be a unique ID, which I'm assigning as @@SPID
DECLARE @CurrentThread INT = @@SPID, 
        @main INT, @sub INT,
        @id INT, @command VARCHAR(MAX), 
        @ErrorMessage NVARCHAR(4000)
WHILE   EXISTS(SELECT TOP 1 ID FROM myQueue WHERE EndDT IS NULL)
BEGIN
    BEGIN TRY

        --/*
        -- Method 1: Top 1 WITH TIES within CTE, direct update against CTE, Contained with sp_getapplock/sp_releaseapplock
        -- works
        -- high volume of xp_userlock waits
        BEGIN TRY
            BEGIN TRAN

                EXEC sp_getapplock @Resource = 'myQueue', @LockMode = 'Update'

                ;WITH dequeue AS
                (
                    SELECT TOP 1 WITH TIES
                        Main, Sub, Thread
                    FROM    myQueue
                    WHERE   EndDT IS NULL
                        AND (Thread IS NULL OR Thread = @CurrentThread)
                    ORDER BY Main, Sub
                )
                UPDATE  dequeue
                SET Thread = @CurrentThread
                OUTPUT  DELETED.Main,
                        DELETED.Sub
                INTO @updOUT

                EXEC sp_releaseapplock @Resource = 'myQueue'
            COMMIT
        END TRY
        BEGIN CATCH
            EXEC sp_releaseapplock @Resource = 'myQueue'
            ROLLBACK TRAN
        END CATCH
        --*/

        /*
        -- Method 2: Top 1 WITH TIES within CTE, direct update against CTE
        -- does not work
        -- some groupings contain multiple worker threads 
        -- missing thread assignments (e.g. NULL value in Thread Column)
        -- deadlocking experienced
        ;WITH dequeue AS
        (
            SELECT TOP 1 WITH TIES
                Main, Sub, Thread
            FROM    myQueue WITH (ROWLOCK, UPDLOCK, READPAST)
            WHERE   EndDT IS NULL
                AND (Thread IS NULL OR Thread = @CurrentThread)
            ORDER BY Main, Sub
        )
        UPDATE  dequeue
        SET Thread = @CurrentThread
        OUTPUT  DELETED.Main,
                DELETED.Sub
        INTO @updOUT
        */

        /*
        -- Method 3: Top 1 WITH TIES within CTE, join to myQueue table
        -- does not work
        -- some groupings contain multiple worker threads 
        -- missing thread assignments (e.g. NULL value in Thread Column)
        -- deadlocking experienced
        ;WITH dequeue AS
        (
            SELECT TOP 1 WITH TIES
                Main, Sub, Thread
            FROM    myQueue WITH (ROWLOCK, UPDLOCK, READPAST)
            WHERE   EndDT IS NULL
                AND (Thread IS NULL OR Thread = @CurrentThread)
            ORDER BY Main, Sub
        )
        UPDATE  myQ
        SET Thread = @CurrentThread
        OUTPUT  DELETED.Main,
                DELETED.Sub
        INTO @updOUT
        FROM    myQueue myQ WITH (ROWLOCK, UPDLOCK, READPAST)
                    INNER JOIN dequeue
                        ON myQ.Main = dequeue.Main
                        AND myQ.Sub = dequeue.Sub 
        */

        /*
        -- Method 4: Top 1 within CTE, join to myQueue table
        -- does not work
        -- some groupings contain multiple worker threads
        ;WITH dequeue AS
        (
            SELECT TOP 1
                Main, Sub, Thread
            FROM    myQueue WITH (ROWLOCK, UPDLOCK, READPAST)
            WHERE   EndDT IS NULL
                AND (Thread IS NULL OR Thread = @CurrentThread)
            ORDER BY Main, Sub
        )
        UPDATE  myQ
        SET Thread = @CurrentThread
        OUTPUT  DELETED.Main,
                DELETED.Sub
        INTO @updOUT
        FROM    myQueue myQ WITH (ROWLOCK, UPDLOCK, READPAST)
                    INNER JOIN dequeue
                        ON myQ.Main = dequeue.Main
                        AND myQ.Sub = dequeue.Sub 
        */

        /*
        -- Method 5: Top 1 WITH TIES within CTE, join to myQueue table, PAGLOCK hint instead of ROWLOCK
        -- works*
        -- deadlocking experienced
        ;WITH dequeue AS
        (
            SELECT TOP 1 WITH TIES
                Main, Sub, Thread
            FROM    myQueue WITH (PAGLOCK, UPDLOCK, READPAST)
            WHERE   EndDT IS NULL
                AND (Thread IS NULL OR Thread = @CurrentThread)
            ORDER BY Main, Sub
        )
        UPDATE  myQ
        SET Thread = @CurrentThread
        OUTPUT  DELETED.Main,
                DELETED.Sub
        INTO @updOUT
        FROM    myQueue myQ WITH (PAGLOCK, UPDLOCK, READPAST)
                    INNER JOIN dequeue
                        ON myQ.Main = dequeue.Main
                        AND myQ.Sub = dequeue.Sub 
        */

        /*
        -- Method 6: Top 1 WITH TIES within CTE, direct update against CTE, PAGLOCK hint instead of ROWLOCK
        -- works*
        -- deadlocking experienced
        ;WITH dequeue AS
        (
            SELECT TOP 1 WITH TIES
                Main, Sub, Thread
            FROM    myQueue WITH (PAGLOCK, UPDLOCK, READPAST)
            WHERE   EndDT IS NULL
                AND (Thread IS NULL OR Thread = @CurrentThread)
            ORDER BY Main, Sub
        )
        UPDATE  dequeue
        SET Thread = @CurrentThread
        OUTPUT  DELETED.Main,
                DELETED.Sub
        INTO @updOUT

        */

        /*
        -- Method 7: Top 1 within CTE, join to myQueue table, PAGLOCK hint instead of ROWLOCK
        -- works*
        -- deadlocking experienced
        ;WITH dequeue AS
        (
            SELECT TOP 1
                Main, Sub, Thread
            FROM    myQueue WITH (PAGLOCK, UPDLOCK, READPAST)
            WHERE   EndDT IS NULL
                AND (Thread IS NULL OR Thread = @CurrentThread)
            ORDER BY Main, Sub
        )
        UPDATE  myQ
        SET Thread = @CurrentThread
        OUTPUT  DELETED.Main,
                DELETED.Sub
        INTO @updOUT
        FROM    myQueue myQ WITH (PAGLOCK, UPDLOCK, READPAST)
                    INNER JOIN dequeue
                        ON myQ.Main = dequeue.Main
                        AND myQ.Sub = dequeue.Sub 
        */

        SELECT  TOP 1 
              @main = Main
            , @sub = Sub
        FROM @updOUT

        END TRY
        BEGIN CATCH
            SELECT @ErrorMessage = 'Msg ' + CAST(ERROR_NUMBER() AS VARCHAR(10)) + ', Level ' + CAST(ERROR_SEVERITY() AS VARCHAR(10)) 
            + ', State ' + CAST(ERROR_STATE() AS VARCHAR(10)) + ', Line ' + CAST(ERROR_LINE() AS VARCHAR(10))
            + CHAR(13) + CHAR(10) + ERROR_MESSAGE()

            RAISERROR(@ErrorMessage, 1, 1) WITH NOWAIT

            -- Set to Uselss values so cursor doesn't fire
            SELECT @main = -1, @sub = -1
        END CATCH

        DELETE FROM @updOUT

        DECLARE WorkQueueCur INSENSITIVE CURSOR
        FOR
            SELECT  ID, Command
            FROM    myQueue
            WHERE   Main = @main
                AND Sub = @sub
            ORDER BY Detail

        OPEN WorkQueueCur

        FETCH NEXT FROM WorkQueueCur
        INTO @id, @command

        WHILE @@FETCH_STATUS = 0
        BEGIN

            RETRY1:

            BEGIN TRY
                UPDATE  myQueue
                SET StartDT = GETDATE()
                WHERE ID = @id
            END TRY
            BEGIN CATCH
                SELECT @ErrorMessage = 'Retry1: Msg ' + CAST(ERROR_NUMBER() AS VARCHAR(10)) + ', Level ' + CAST(ERROR_SEVERITY() AS VARCHAR(10)) 
                + ', State ' + CAST(ERROR_STATE() AS VARCHAR(10)) + ', Line ' + CAST(ERROR_LINE() AS VARCHAR(10))
                + CHAR(13) + CHAR(10) + ERROR_MESSAGE()

                RAISERROR(@ErrorMessage, 1, 1) WITH NOWAIT

                GOTO RETRY1
            END CATCH

            EXEC(@command)

            RETRY2:

            BEGIN TRY
                UPDATE  myQueue
                Set EndDT = GETDATE()
                WHERE ID = @id
            END TRY
            BEGIN CATCH
                SELECT @ErrorMessage = 'Retry2: Msg ' + CAST(ERROR_NUMBER() AS VARCHAR(10)) + ', Level ' + CAST(ERROR_SEVERITY() AS VARCHAR(10)) 
                + ', State ' + CAST(ERROR_STATE() AS VARCHAR(10)) + ', Line ' + CAST(ERROR_LINE() AS VARCHAR(10))
                + CHAR(13) + CHAR(10) + ERROR_MESSAGE()

                RAISERROR(@ErrorMessage, 1, 1) WITH NOWAIT

                GOTO RETRY2
            END CATCH

            FETCH NEXT FROM WorkQueueCur
            INTO @id, @command
        END

        CLOSE WorkQueueCur
        DEALLOCATE WorkQueueCur


END

可以通过运行以下语句来确定上述不良行为 2 和 3（或不存在）的确认：

;WITH invalidMThread AS (
    SELECT  *, DENSE_RANK() OVER (PARTITION BY Main, Sub ORDER BY Thread) AS ThreadCount
    FROM    dbo.myQueue WITH (NOLOCK)
    WHERE   StartDT IS NOT NULL
), invalidNThread AS (
    SELECT  *
    FROM    dbo.myQueue WITH (NOLOCK)
    WHERE   Thread IS NULL
            AND StartDT IS NOT NULL
)
SELECT  t1.*, 'Multiple Threads' AS Issue
FROM    dbo.myQueue t1 WITH (NOLOCK) 
        INNER JOIN invalidMThread i1
            ON i1.Main = t1.Main
            AND i1.Sub = t1.Sub
WHERE   i1.ThreadCount > 1

UNION

SELECT  t1.*, 'Unassigned Thread(s)' AS Issue
FROM    dbo.myQueue t1 WITH (NOLOCK) 
        INNER JOIN invalidNThread i2
            ON i2.Main = t1.Main
            AND i2.Sub = t1.Sub

ORDER BY t1.Main, t1.Sub

再一次，我完全预料到我错过了 Remus 在博客文章中提出的一些关键点，因此非常感谢指出这一点的任何帮助。

Answer 1

您是否考虑过SQL Server Service Broker，或者您正在使用的 SQL Server 版本是否有限制？