检测/诊断线程饥饿

Question

我正在对IIS应用程序进行一些性能/可伸缩性测试,这些测试偶尔似乎会减慢到生产中的爬行速度.我能够使用NUnit一致地重现慢度.

在测试期间或生产中发生缓慢时,CPU和内存不会出现峰值.我强烈怀疑应用程序正在遭受线程饥饿,因为它似乎不是导致瓶颈的CPU,内存,I/O或数据库访问.我确实看到了看似线索饥饿的迹象; 例如,NLog的异步日志文件写入往往具有长时间的静默,随后是具有较旧时间戳的活动突发(即,较低优先级的线程正在等待线程释放以便写入).

我可以采取哪些步骤来明确确定应用程序确实是线程缺乏的,并且(假设是这种情况)确定导致问题的系统的确切区域？

编辑

我忽略了几乎所有的代码都是同步的(它是遗留系统).

Answer 1

根据Sinatr的评论,我在ThreadPool.SetMinThreads和TaskCreationOptions.LongRunning上做了一些阅读,包括何时使用TaskCreationOptions.LongRunning的答案 ？

将MinThreads设置为更高的默认值会对我的情况产生巨大影响.我创建了一个简单的后台进程,以查看ThreadPool中的可用线程在测试运行过程中是否发生了显着变化,并且超过了MinThreads值(它是).

这是我用来诊断的一些代码.这不适用于生产用途,此处显示的线程使用情况报告只会在最初增加时才会引起注意.另请注意,Timer过去需要一个线程,因此还需要等待一个可用的线程.

静态变量:

    private static Timer _timer;
    private static int _lastActiveThreads;
    private static int _lastAvailableThreads;
    private static int _maxThreads;
    private static int _minThreads;

在启动时运行:

    int completionPortThreads;

    ThreadPool.GetMaxThreads(out _maxThreads, out completionPortThreads);
    ThreadPool.GetMinThreads(out _minThreads, out completionPortThreads);

    _timer = new Timer
    {
        AutoReset = true,
        Interval = 500,
    };

    _timer.Elapsed += TimerElasped;
    _timer.Start();

经过的方法:

    private static void TimerElasped(object sender, ElapsedEventArgs e)
    {
        int minWorkerThreads;
        int availWorkerThreads;
        int completionPortThreads;

        ThreadPool.GetMinThreads(out minWorkerThreads, out completionPortThreads);
        ThreadPool.GetAvailableThreads(out availWorkerThreads, out completionPortThreads);

        var activeThreads = _maxThreads - availWorkerThreads;

        if (availWorkerThreads != _lastAvailableThreads)
        {
            _lastAvailableThreads = availWorkerThreads;
            if (activeThreads > _lastActiveThreads)
            {
                _lastActiveThreads = activeThreads;
                Logger.Log($"+++++ Active Threads is now: {activeThreads}");

                if (activeThreads > _minThreads)
                {
                    var diff = activeThreads - _minThreads;
                    Logger.Log($"+++++ Active threads is now {activeThreads}, which is {diff} more than minThread value of {_minThreads}.  This may be causing delays.");
                }
            }
        }
    }

Answer 2

我想出了这个基于上述

using System;
using System.Threading;
using System.Timers;
using log4net;

using Timer = System.Timers.Timer;

namespace somewhere
{
    public class ThreadStatsLogger : IDisposable
    {
        private const int DEPLETION_WARN_LEVEL = 10;
        private const int HISTERESIS_LEVEL = 10;

    private const double SAMPLE_RATE_MILLISECONDS = 500;
    private bool _workerThreadWarned = false;
    private bool _ioThreadWarned = false;
    private bool _minWorkerThreadLevelWarned = false;
    private bool _minIoThreadLevelWarned = false;

    private readonly int _maxWorkerThreadLevel;
    private readonly int _maxIoThreadLevel;
    private readonly int _minWorkerThreadLevel;
    private readonly int _minWorkerThreadLevelRecovery;
    private readonly int _minIoThreadLevel;
    private readonly int _minIoThreadLevelRecovery;
    private Timer _timer;

    private static readonly ILog _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);

    public ThreadStatsLogger()
    {

        _timer = new Timer
        {
            AutoReset = true,
            Interval = SAMPLE_RATE_MILLISECONDS,
        };

        _timer.Elapsed += TimerElasped;
        _timer.Start();
        ThreadPool.GetMinThreads(out _minWorkerThreadLevel, out _minIoThreadLevel);
        ThreadPool.GetMaxThreads(out _maxWorkerThreadLevel, out _maxIoThreadLevel);
        ThreadPool.GetAvailableThreads(out int workerAvailable, out int ioAvailable);

        _logger.InfoFormat("Thread statistics at startup: minimum worker:{0} io:{1}", _minWorkerThreadLevel, _minIoThreadLevel );
        _logger.InfoFormat("Thread statistics at startup: maximum worker:{0} io:{1}", _maxWorkerThreadLevel, _maxIoThreadLevel);
        _logger.InfoFormat("Thread statistics at startup: available worker:{0} io:{1}", workerAvailable, ioAvailable);

        _minWorkerThreadLevelRecovery = (_minWorkerThreadLevel * 3) / 4;
        _minIoThreadLevelRecovery = (_minIoThreadLevel * 3) / 4;
        if (_minWorkerThreadLevelRecovery == _minWorkerThreadLevel) _minWorkerThreadLevelRecovery = _minWorkerThreadLevel - 1;
        if (_minIoThreadLevelRecovery == _minIoThreadLevel) _minIoThreadLevelRecovery = _minIoThreadLevel - 1;
    }

    private void TimerElasped(object sender, ElapsedEventArgs e)
    {

        ThreadPool.GetAvailableThreads(out int availableWorkerThreads, out int availableIoThreads);

        var activeWorkerThreads = _maxWorkerThreadLevel - availableWorkerThreads;
        var activeIoThreads = _maxIoThreadLevel - availableIoThreads;

        _logger.InfoFormat("Thread statistics: active worker:{0} io:{1}", activeWorkerThreads, activeIoThreads);

        if (activeWorkerThreads > _minWorkerThreadLevel && !_minWorkerThreadLevelWarned)
        {
            _logger.InfoFormat("Thread statistics WARN active worker threads above minimum {0}:{1}", activeWorkerThreads, _minWorkerThreadLevel);
            _minWorkerThreadLevelWarned = !_minWorkerThreadLevelWarned;
        }
        if (activeWorkerThreads < _minWorkerThreadLevelRecovery && _minWorkerThreadLevelWarned)
        {
            _logger.InfoFormat("Thread statistics RECOVERY active worker threads below minimum {0}:{1}", activeWorkerThreads, _minWorkerThreadLevel);
            _minWorkerThreadLevelWarned = !_minWorkerThreadLevelWarned;
        }

        if (activeIoThreads > _minIoThreadLevel && !_minIoThreadLevelWarned)
        {
            _logger.InfoFormat("Thread statistics WARN active io threads above minimum {0}:{1}", activeIoThreads, _minIoThreadLevel);
            _minIoThreadLevelWarned = !_minIoThreadLevelWarned;
        }
        if (activeIoThreads < _minIoThreadLevelRecovery && _minIoThreadLevelWarned)
        {
            _logger.InfoFormat("Thread statistics RECOVERY active io threads below minimum {0}:{1}", activeIoThreads, _minIoThreadLevel);
            _minIoThreadLevelWarned = !_minIoThreadLevelWarned;
        }

        if (availableWorkerThreads < DEPLETION_WARN_LEVEL && !_workerThreadWarned)
        {
            _logger.InfoFormat("Thread statistics WARN available worker threads below warning level {0}:{1}", availableWorkerThreads, DEPLETION_WARN_LEVEL);
            _workerThreadWarned = !_workerThreadWarned;
        }

        if (availableWorkerThreads > (DEPLETION_WARN_LEVEL + HISTERESIS_LEVEL) && _workerThreadWarned)
        {
            _logger.InfoFormat("Thread statistics RECOVERY available worker thread recovery {0}:{1}", availableWorkerThreads, DEPLETION_WARN_LEVEL);
            _workerThreadWarned = !_workerThreadWarned;
        }

        if (availableIoThreads < DEPLETION_WARN_LEVEL && !_ioThreadWarned)
        {
            _logger.InfoFormat("Thread statistics WARN available io threads below warning level {0}:{1}", availableIoThreads, DEPLETION_WARN_LEVEL);
            _ioThreadWarned = !_ioThreadWarned;
        }

        if (availableIoThreads > (DEPLETION_WARN_LEVEL + HISTERESIS_LEVEL) && _ioThreadWarned)
        {
            _logger.InfoFormat("Thread statistics RECOVERY available io thread recovery {0}:{1}", availableIoThreads, DEPLETION_WARN_LEVEL);
            _ioThreadWarned = !_ioThreadWarned;
        }
    }

    public void Dispose()
    {
        if (_timer == null) return;
        _timer.Close();
        _timer.Dispose();
        _timer = null;
    }
}

}