//******************************************************************************************************
// ThreadPoolMonitor.cs - Gbtc
//
// Copyright © 2016, Grid Protection Alliance. All Rights Reserved.
//
// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See
// the NOTICE file distributed with this work for additional information regarding copyright ownership.
// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may
// not use this file except in compliance with the License. You may obtain a copy of the License at:
//
// http://www.opensource.org/licenses/MIT
//
// Unless agreed to in writing, the subject software distributed under the License is distributed on an
// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the
// License for the specific language governing permissions and limitations.
//
// Code Modification History:
// ----------------------------------------------------------------------------------------------------
// 11/07/2016 - Steven E. Chisholm
// Generated original version of source code.
//
//******************************************************************************************************
using System.Collections.Generic;
using System.Text;
using GSF.Threading;
namespace GSF.Diagnostics
{
///
/// This class will monitor the performance of the ThreadPool and report any indications of system stress.
///
public static class ThreadPoolMonitor
{
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable")]
private class Monitor
{
private int m_delay;
private ScheduledTask m_task;
private ShortTime? m_time;
private LogEventPublisher m_logSmall;
private LogEventPublisher m_logMedium;
private LogEventPublisher m_logLarge;
public Monitor(int delay)
{
m_logSmall = Log.RegisterEvent(MessageLevel.Info, MessageFlags.SystemHealth, $"ThreadPool Small Delay Variance ({delay} ms)", 0, MessageRate.EveryFewSeconds(10), 5);
m_logMedium = Log.RegisterEvent(MessageLevel.Warning, MessageFlags.SystemHealth, $"ThreadPool Medium Delay Variance ({delay} ms)", 0, MessageRate.EveryFewSeconds(10), 5);
m_logLarge = Log.RegisterEvent(MessageLevel.Error, MessageFlags.SystemHealth, $"ThreadPool Large Delay Variance ({delay} ms)", 0, MessageRate.EveryFewSeconds(10), 5);
m_logSmall.ShouldRaiseMessageSupressionNotifications = false;
m_logMedium.ShouldRaiseMessageSupressionNotifications = false;
m_logLarge.ShouldRaiseMessageSupressionNotifications = false;
m_delay = delay;
m_time = null;
m_task = new ScheduledTask();
m_task.Running += task_Running;
}
private void task_Running(object sender, EventArgs e)
{
ShortTime currentTime = ShortTime.Now;
ShortTime queueTime;
lock (this)
{
if (!m_time.HasValue)
{
return; //Should never happen, but just in case;
}
queueTime = m_time.Value;
m_time = null;
}
double delay = (currentTime - queueTime).TotalMilliseconds;
if (delay > m_delay + 1000)
{
m_logLarge.Publish(delay.ToString() + " ms");
}
else if (delay > m_delay + 500)
{
m_logMedium.Publish(delay.ToString() + " ms");
}
else if (delay > m_delay + 100)
{
m_logSmall.Publish(delay.ToString() + " ms");
}
else
{
return;
}
//If I published a message, a long delay could have occurred and messed
//up my timer, therefore reset the current time.
lock (this)
{
if (m_time.HasValue)
{
m_time = ShortTime.Now;
}
}
}
public void Start()
{
lock (this)
{
//Check if the last queue has processed, if not, then quit.
if (m_time.HasValue)
return;
m_time = ShortTime.Now;
if (m_delay == 0)
{
m_task.Start();
}
else
{
m_task.Start(m_delay);
}
}
}
}
private static readonly LogPublisher Log = Logger.CreatePublisher(typeof(ThreadPoolMonitor), MessageClass.Component);
private static List s_monitors;
private static ShortTime s_lastResetTime;
static ThreadPoolMonitor()
{
if (OptimizationOptions.EnableThreadPoolMonitoring)
{
s_monitors = new List();
s_monitors.Add(new Monitor(0));
s_monitors.Add(new Monitor(10));
s_monitors.Add(new Monitor(100));
s_monitors.Add(new Monitor(1000));
s_lastResetTime = ShortTime.Now;
Log.Publish(MessageLevel.Info, "Starting ThreadPool Monitoring.");
LoadingAdjustedTimestamp.OnHighLoad += LoadingAdjustedTimestamp_OnHighLoad;
}
}
///
/// Initializes the static constructor
///
public static void Initialize()
{
//Initializes the static constructor.
}
private static void LoadingAdjustedTimestamp_OnHighLoad()
{
if (s_lastResetTime.ElapsedSeconds() > 10)
{
s_lastResetTime = ShortTime.Now;
MonitorThreads();
}
}
private static void MonitorThreads()
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("--------------------------------------------------------");
sb.AppendLine("Thread Pool Queue");
sb.AppendLine("--------------------------------------------------------");
ThreadPoolTrace.GetTrace(sb);
sb.AppendLine("--------------------------------------------------------");
sb.AppendLine("Timers");
sb.AppendLine("--------------------------------------------------------");
TimerTrace.GetTrace(sb);
//Since current builds of openPDC appears to be doing better about not blocking all threads on the threadpool
//this diagnostics functionality has been disabled but the code has been left in place if it's needed in the future.
// if (OptimizationOptions.EnableThreadStackDumping)
// {
//#if !MONO
// using (var dataTarget = DataTarget.AttachToProcess(Process.GetCurrentProcess().Id, 5000, AttachFlag.Passive))
// {
// foreach (var clr in dataTarget.ClrVersions)
// {
// var runtime = clr.CreateRuntime();
// sb.AppendLine("--------------------------------------------------------");
// sb.AppendLine("Thread Stacks");
// sb.AppendLine("--------------------------------------------------------");
// foreach (var t in runtime.Threads)
// {
// bool hasWrittenData = false;
// foreach (var item in t.StackTrace)
// {
// if (item.Method != null)
// {
// if (!hasWrittenData)
// hasWrittenData = true;
// sb.AppendLine(item.ToString());
// }
// }
// if (hasWrittenData)
// {
// sb.AppendLine("--------------------------------------------------------");
// }
// }
// }
// }
//#endif
// }
Log.Publish(MessageLevel.Warning, "ThreadPool Stack Trace", "Dumped threadpool stack trace", sb.ToString());
}
}
}