//******************************************************************************************************
// ZeroMQServer.cs - Gbtc
//
// Copyright © 2015, Grid Protection Alliance. All Rights Reserved.
//
// Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See
// the NOTICE file distributed with this work for additional information regarding copyright ownership.
// The GPA licenses this file to you under the MIT License (MIT), the "License"; you may
// not use this file except in compliance with the License. You may obtain a copy of the License at:
//
// http://www.opensource.org/licenses/MIT
//
// Unless agreed to in writing, the subject software distributed under the License is distributed on an
// "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the
// License for the specific language governing permissions and limitations.
//
// Code Modification History:
// ----------------------------------------------------------------------------------------------------
// 05/22/2015 - J. Ritchie Carroll
// Original version of source code generated.
//
//******************************************************************************************************
#region [ Contributor License Agreements ]
/*
This code uses zeromq/clrzmq4 library for ZeroMQ implementation (http://zeromq.org).
Source code for the clrzmq4 project can be found here: https://github.com/zeromq/clrzmq4
ZeroMQ and the clrzmq4 library are licensed under the GNU Lesser General Public License v3.0 plus
a static linking exception (the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at:
https://www.gnu.org/licenses/lgpl-3.0.html
Static linking exception: The copyright holders give you permission to link this library with
independent modules to produce an executable, regardless of the license terms of these independent
modules, and to copy and distribute the resulting executable under terms of your choice, provided
that you also meet, for each linked independent module, the terms and conditions of the license of
that module. An independent module is a module which is not derived from or based on this library.
If you modify this library, you must extend this exception to your version of the library.
*/
#endregion
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics.CodeAnalysis;
using System.Net.Sockets;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using GSF.Configuration;
using GSF.IO;
using ZeroMQ;
namespace GSF.Communication
{
///
/// Represents a ZeroMQ ROUTER style socket as a communication server.
///
public class ZeroMQServer : ServerBase
{
#region [ Members ]
// Constants
///
/// Specifies the default value for the property.
///
public const string DefaultConfigurationString = "server=tcp://*:8888";
///
/// Specifies the default value for the property.
///
///
/// Maps to ZeroMQ high water mark setting for outbound messages.
///
public const int DefaultMaxSendQueueSize = 500000;
///
/// Specifies the default value for the property.
///
///
/// Maps to ZeroMQ high water mark setting for inbound messages.
///
public const int DefaultMaxReceiveQueueSize = 500000;
///
/// Regular expression used to validate the format for a ZeroMQ endpoint.
///
///
///
/// Matches the following valid example input:
/// - tcp://127.0.0.1:5959
/// - tcp://[::1]:5959
/// - tcp://dnsname.org:8186
/// - pgm://[FF01:0:0:0:0:0:0:1]:6565
/// - pgm://224.0.0.1:6565
/// - inproc://pipe-name
///
///
public const string EndpointFormatRegex = @"(?.+)\://(?(\[.+\]|[^\:])+)(\:(?\d+$))?";
private ZeroMQTransportProtocol m_zeroMQTransportProtocol;
private readonly ManualResetEventSlim m_completedHandle;
private Dictionary m_configData;
private readonly ConcurrentDictionary> m_clientInfoLookup;
private readonly Timer m_activeClientTimer;
private Thread m_receiveDataThread;
private readonly object m_sendLock;
private bool m_disposed;
#endregion
#region [ Constructors ]
///
/// Initializes a new instance of the class.
///
public ZeroMQServer() : this(DefaultConfigurationString)
{
}
///
/// Initializes a new instance of the class.
///
/// Config string of the . See for format.
public ZeroMQServer(string configString) : base(TransportProtocol.Tcp, configString)
{
m_zeroMQTransportProtocol = ZeroMQTransportProtocol.Tcp;
m_completedHandle = new ManualResetEventSlim(true);
MaxSendQueueSize = DefaultMaxSendQueueSize;
MaxReceiveQueueSize = DefaultMaxReceiveQueueSize;
m_clientInfoLookup = new ConcurrentDictionary>();
m_activeClientTimer = new Timer(MonitorActiveClients, null, TimeSpan.FromMinutes(1.0D), TimeSpan.FromMinutes(1.0D));
m_sendLock = new object();
}
///
/// Initializes a new instance of the class.
///
/// object that contains the .
public ZeroMQServer(IContainer container) : this() => container?.Add(this);
#endregion
#region [ Properties ]
///
/// Gets or sets the maximum size for the send queue before payloads are dumped from the queue.
///
///
/// Maps to ZeroMQ high water mark setting for outbound messages.
///
[Category("Settings")]
[DefaultValue(DefaultMaxSendQueueSize)]
[Description("The maximum size for the send queue before payloads are dumped from the queue.")]
public int MaxSendQueueSize { get; set; }
///
/// Gets or sets the maximum size for the receive queue before payloads are dumped from the queue.
///
///
/// Maps to ZeroMQ high water mark setting for inbound messages.
///
[Category("Settings")]
[DefaultValue(DefaultMaxReceiveQueueSize)]
[Description("The maximum size for the receive queue before payloads are dumped from the queue.")]
public int MaxReceiveQueueSize { get; set; }
///
/// Gets the used by the server for the transportation of data with the clients.
///
public override TransportProtocol TransportProtocol
{
get
{
switch (m_zeroMQTransportProtocol)
{
case ZeroMQTransportProtocol.Tcp:
case ZeroMQTransportProtocol.InProc:
return TransportProtocol.Tcp;
default:
return TransportProtocol.Udp;
}
}
}
///
/// Gets or sets the ZeroMQ transport protocol to use for the .
///
[Category("Settings")]
[DefaultValue(ZeroMQTransportProtocol.Tcp)]
[Description("The ZeroMQ transport protocol to use for the connection.")]
public ZeroMQTransportProtocol ZeroMQTransportProtocol
{
get => m_zeroMQTransportProtocol;
set => m_zeroMQTransportProtocol = value;
}
///
/// Gets the object for the .
///
[Browsable(false)]
public ZSocket Server { get; private set; }
///
/// Gets the descriptive status of the client.
///
public override string Status
{
get
{
StringBuilder statusBuilder = new StringBuilder(base.Status);
if (Server != null)
{
try
{
statusBuilder.AppendFormat(" 0MQ Identity: {0}", new Guid(Server.Identity));
}
catch
{
statusBuilder.AppendFormat(" 0MQ Identity: {0}", Server.IdentityString);
}
statusBuilder.AppendLine();
statusBuilder.AppendFormat(" 0MQ Last Endpoint: {0}", Server.LastEndpoint);
statusBuilder.AppendLine();
}
statusBuilder.AppendFormat(" 0MQ Transport Protocol: {0}", ZeroMQTransportProtocol);
statusBuilder.AppendLine();
statusBuilder.AppendFormat(" 0MQ Binding Endpoint: {0}", m_configData["server"]);
statusBuilder.AppendLine();
statusBuilder.AppendFormat(" Max send queue size: {0}", MaxSendQueueSize);
statusBuilder.AppendLine();
statusBuilder.AppendFormat(" Max receive queue size: {0}", MaxReceiveQueueSize);
statusBuilder.AppendLine();
return statusBuilder.ToString();
}
}
#endregion
#region [ Methods ]
///
/// Releases the unmanaged resources used by the object and optionally releases the managed resources.
///
/// true to release both managed and unmanaged resources; false to release only unmanaged resources.
[SuppressMessage("Microsoft.Usage", "CA2213:DisposableFieldsShouldBeDisposed", MessageId = "m_activeClientTimer", Justification = "Field is properly disposed")]
protected override void Dispose(bool disposing)
{
if (m_disposed)
return;
try
{
if (!disposing)
return;
if (m_completedHandle != null)
{
m_completedHandle.Set();
m_completedHandle.Dispose();
}
m_activeClientTimer?.Dispose();
Server?.Dispose();
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
///
/// Reads a number of bytes from the current received data buffer and writes those bytes into a byte array at the specified offset.
///
/// ID of the client from which data buffer should be read.
/// Destination buffer used to hold copied bytes.
/// 0-based starting index into destination to begin writing data.
/// The number of bytes to read from current received data buffer and write into .
/// The number of bytes read.
///
/// This function should only be called from within the event handler. Calling this method
/// outside this event will have unexpected results.
///
///
/// No received data buffer has been defined to read -or-
/// Specified does not exist, cannot read buffer.
///
/// is null.
///
/// or is less than 0 -or-
/// and will exceed length.
///
public override int Read(Guid clientID, byte[] buffer, int startIndex, int length)
{
buffer.ValidateParameters(startIndex, length);
if (!m_clientInfoLookup.TryGetValue(clientID, out TransportProvider clientInfo))
throw new InvalidOperationException("Specified client ID does not exist, cannot read buffer.");
if (clientInfo.ReceiveBuffer == null)
throw new InvalidOperationException("No received data buffer has been defined to read.");
int readIndex = ReadIndicies[clientID];
int sourceLength = clientInfo.BytesReceived - readIndex;
int readBytes = length > sourceLength ? sourceLength : length;
Buffer.BlockCopy(clientInfo.ReceiveBuffer, readIndex, buffer, startIndex, readBytes);
// Update read index for next call
readIndex += readBytes;
if (readIndex >= clientInfo.BytesReceived)
readIndex = 0;
ReadIndicies[clientID] = readIndex;
return readBytes;
}
///
/// Saves settings to the config file if the property is set to true.
///
public override void SaveSettings()
{
base.SaveSettings();
if (!PersistSettings)
return;
// Save settings under the specified category.
ConfigurationFile config = ConfigurationFile.Current;
CategorizedSettingsElementCollection settings = config.Settings[SettingsCategory];
settings["MaxSendQueueSize", true].Update(MaxSendQueueSize);
settings["MaxReceiveQueueSize", true].Update(MaxReceiveQueueSize);
config.Save();
}
///
/// Loads saved settings from the config file if the property is set to true.
///
public override void LoadSettings()
{
base.LoadSettings();
if (!PersistSettings)
return;
// Load settings from the specified category.
ConfigurationFile config = ConfigurationFile.Current;
CategorizedSettingsElementCollection settings = config.Settings[SettingsCategory];
settings.Add("MaxSendQueueSize", MaxSendQueueSize, "The maximum size of the send queue before payloads are dumped from the queue.");
settings.Add("MaxReceiveQueueSize", MaxReceiveQueueSize, "The maximum size of the receive queue before payloads are dumped from the queue.");
MaxSendQueueSize = settings["MaxSendQueueSize"].ValueAs(MaxSendQueueSize);
MaxReceiveQueueSize = settings["MaxReceiveQueueSize"].ValueAs(MaxReceiveQueueSize);
// When transitioning from socket type to another, be sure to restore default values
if (MaxSendQueueSize == -1)
MaxSendQueueSize = DefaultMaxSendQueueSize;
if (MaxReceiveQueueSize == -1)
MaxReceiveQueueSize = DefaultMaxReceiveQueueSize;
}
///
/// Stops the synchronously and disconnects all connected clients.
///
public override void Stop()
{
if (CurrentState != ServerState.Running)
return;
if (m_receiveDataThread != null)
{
m_receiveDataThread.Abort();
m_receiveDataThread = null;
}
if (Server != null)
{
Server.Dispose();
Server = null;
}
DisconnectAll(); // Disconnection all clients.
OnServerStopped();
}
///
/// Starts the synchronously and begins accepting client connections asynchronously.
///
/// Attempt is made to the when it is running.
public override void Start()
{
if (CurrentState != ServerState.NotRunning)
return;
// Initialize if needed
if (!Initialized)
Initialize();
// Overwrite config file if max client connections exists in connection string.
if (m_configData.ContainsKey("maxClientConnections") && int.TryParse(m_configData["maxClientConnections"], out int maxClientConnections))
MaxClientConnections = maxClientConnections;
// Overwrite config file if max send queue size exists in connection string.
if (m_configData.ContainsKey("maxSendQueueSize") && int.TryParse(m_configData["maxSendQueueSize"], out int maxQueueSize))
MaxSendQueueSize = maxQueueSize;
// Overwrite config file if max receive queue size exists in connection string.
if (m_configData.ContainsKey("maxReceiveQueueSize") && int.TryParse(m_configData["maxReceiveQueueSize"], out maxQueueSize))
MaxReceiveQueueSize = maxQueueSize;
// Create ZeroMQ Router socket - closest match to IServer implementation
Server = new ZSocket(ZContext.Create(), ZSocketType.ROUTER)
{
Identity = ServerID.ToByteArray(),
SendHighWatermark = MaxSendQueueSize,
ReceiveHighWatermark = MaxReceiveQueueSize,
Immediate = true,
IPv6 = Transport.GetDefaultIPStack() == IPStack.IPv6
};
Server.SetOption(ZSocketOption.LINGER, 0);
Server.SetOption(ZSocketOption.SNDTIMEO, 1000);
Server.SetOption(ZSocketOption.RCVTIMEO, -1);
Server.SetOption(ZSocketOption.RECONNECT_IVL, -1);
Server.Bind(m_configData["server"]);
// Notify that the server has been started successfully.
OnServerStarted();
m_receiveDataThread = new Thread(ReceiveDataHandler)
{
IsBackground = true
};
m_receiveDataThread.Start();
}
private void ReceiveDataHandler()
{
while (Enabled)
{
Guid clientID = Guid.Empty;
TransportProvider clientInfo = null;
try
{
// Receive data from the socket
using (ZMessage message = Server.ReceiveMessage())
{
// Router socket should provide identity, delimiter and data payload frames
if (message.Count == 3)
{
// Extract client identity
clientID = new Guid(message[0].ReadStream());
// Lookup client info, adding it if it doesn't exist
clientInfo = GetClient(clientID);
// Get data payload frame
ZFrame frame = message[2];
clientInfo.BytesReceived = (int)frame.Length;
if (clientInfo.ReceiveBufferSize < clientInfo.BytesReceived)
clientInfo.SetReceiveBuffer(clientInfo.BytesReceived);
frame.Read(clientInfo.ReceiveBuffer, 0, clientInfo.BytesReceived);
clientInfo.Statistics.UpdateBytesReceived(clientInfo.BytesReceived);
// Update last client activity time
clientInfo.Provider = DateTime.UtcNow;
}
}
// Notify consumer of received data
if (clientInfo != null)
OnReceiveClientDataComplete(clientID, clientInfo.ReceiveBuffer, clientInfo.BytesReceived);
}
catch (Exception ex)
{
OnReceiveClientDataException(clientID, ex);
}
}
}
///
/// Disconnects the specified connected client.
///
/// ID of the client to be disconnected.
/// Client does not exist for the specified .
public override void DisconnectOne(Guid clientID)
{
if (!m_clientInfoLookup.TryRemove(clientID, out TransportProvider clientInfo))
return;
try
{
OnClientDisconnected(clientID);
clientInfo.Reset();
}
catch (Exception ex)
{
OnSendClientDataException(clientInfo.ID, new InvalidOperationException($"Client disconnection exception: {ex.Message}", ex));
}
}
///
/// Gets the object associated with the specified client ID.
///
/// ID of the client.
/// Client information.
/// true if client exists; otherwise, false.
public bool TryGetClient(Guid clientID, out TransportProvider clientInfo) => m_clientInfoLookup.TryGetValue(clientID, out clientInfo);
// Get client info, creating it if it doesn't exist
private TransportProvider GetClient(Guid clientID)
{
return m_clientInfoLookup.GetOrAdd(clientID, id =>
{
OnClientConnected(id);
return new TransportProvider
{
Provider = DateTime.UtcNow,
ID = id
};
});
}
///
/// Validates the specified .
///
/// Configuration string to be validated.
/// Port property is missing.
/// Port property value is not between and .
protected override void ValidateConfigurationString(string configurationString)
{
m_configData = configurationString.ParseKeyValuePairs();
// Check for "server" property, this is the preferred configuration for ZeroMQServer
if (m_configData.ContainsKey("server"))
{
// Validate ZeroMQ configuration string
Match endpointMatch = Regex.Match(m_configData["server"], EndpointFormatRegex);
string port = null;
bool validConfigurationString = false;
if (endpointMatch.Success)
{
if (!Enum.TryParse(endpointMatch.Groups["protocol"].Value.Trim(), true, out m_zeroMQTransportProtocol))
m_zeroMQTransportProtocol = ZeroMQTransportProtocol.Tcp;
m_configData["interface"] = endpointMatch.Groups["host"].Value.Trim();
port = endpointMatch.Groups["port"].Value.Trim();
validConfigurationString = true;
}
else
{
// Support traditional IClient "server" property
endpointMatch = Regex.Match(m_configData["server"], Transport.EndpointFormatRegex);
if (endpointMatch.Success)
{
m_configData["server"] = $"{m_zeroMQTransportProtocol.ToString().ToLowerInvariant()}://{m_configData["server"]}";
m_configData["interface"] = endpointMatch.Groups["host"].Value.Trim();
port = endpointMatch.Groups["port"].Value.Trim();
validConfigurationString = true;
}
}
if (!validConfigurationString)
throw new FormatException($"Server property is invalid (Example: {DefaultConfigurationString})");
if (m_zeroMQTransportProtocol != ZeroMQTransportProtocol.InProc && !Transport.IsPortNumberValid(port))
throw new ArgumentOutOfRangeException(nameof(configurationString), $"Port number must be between {Transport.PortRangeLow} and {Transport.PortRangeHigh}");
}
else
{
// Fall back on traditional server configuration strings
Transport.GetInterfaceIPStack(m_configData);
if (string.IsNullOrWhiteSpace(m_configData["interface"]) || m_configData["interface"].Equals("0.0.0.0", StringComparison.Ordinal))
m_configData["interface"] = "*";
// For traditional style connection strings, also support a "zeroMQTransportProtocol" setting
if (m_configData.ContainsKey("zeroMQTransportProtocol"))
{
if (Enum.TryParse(m_configData["zeroMQTransportProtocol"].Trim(), true, out ZeroMQTransportProtocol protocol))
m_zeroMQTransportProtocol = protocol;
}
// For traditional IServer connection strings, a "port" property is expected
if (m_configData.ContainsKey("port") && m_zeroMQTransportProtocol != ZeroMQTransportProtocol.InProc)
m_configData["server"] = $"{m_zeroMQTransportProtocol.ToString().ToLowerInvariant()}://{m_configData["interface"]}:{m_configData["port"]}";
else
throw new FormatException($"Server property is invalid (Example: {DefaultConfigurationString})");
}
}
///
/// Sends data to the specified client asynchronously.
///
/// ID of the client to which the data is to be sent.
/// The buffer that contains the binary data to be sent.
/// The zero-based position in the at which to begin sending data.
/// The number of bytes to be sent from starting at the .
/// for the asynchronous operation.
protected override WaitHandle SendDataToAsync(Guid clientID, byte[] data, int offset, int length)
{
if (CurrentState != ServerState.Running)
throw new SocketException((int)SocketError.NotConnected);
try
{
if (Server != null)
{
// Lookup client info, adding it if it doesn't exist
TransportProvider clientInfo = GetClient(clientID);
// Router socket should provide identity, delimiter and data payload frames
using (ZMessage message = new ZMessage())
{
// Add identity, delimiter and data payload frames
message.Add(new ZFrame(clientID.ToByteArray()));
message.Add(new ZFrame());
message.Add(new ZFrame(data, offset, length));
// ZeroMQ send is asynchronous, but API call is not thread-safe
lock (m_sendLock)
Server.Send(message);
}
clientInfo.Statistics.UpdateBytesSent(length);
// Update last client activity time
clientInfo.Provider = DateTime.UtcNow;
}
}
catch (Exception ex)
{
// Log exception during send operation
OnSendClientDataException(clientID, ex);
}
return m_completedHandle.WaitHandle;
}
///
/// Raises the event.
///
/// ID of client to send to event.
/// Exception to send to event.
protected override void OnSendClientDataException(Guid clientID, Exception ex)
{
if (IsThreadAbortException(ex))
return;
if (m_clientInfoLookup.ContainsKey(clientID) && CurrentState == ServerState.Running)
{
if (ex is ZException zmqex && (zmqex.Error.Number == ZError.EAGAIN.Number || zmqex.Error.Number == ZError.ETERM.Number))
ThreadPool.QueueUserWorkItem(state => DisconnectOne(clientID));
else
base.OnSendClientDataException(clientID, ex);
}
}
///
/// Raises the event.
///
/// ID of client to send to event.
/// Exception to send to event.
protected override void OnReceiveClientDataException(Guid clientID, Exception ex)
{
if (IsThreadAbortException(ex))
return;
if (m_clientInfoLookup.ContainsKey(clientID) && CurrentState == ServerState.Running)
base.OnReceiveClientDataException(clientID, ex);
}
///
/// Raises the event.
///
/// The encountered when connecting to the client.
protected override void OnClientConnectingException(Exception ex)
{
if (IsThreadAbortException(ex))
return;
base.OnClientConnectingException(ex);
}
///
/// Raises the event.
///
/// Exception to send to event.
protected override void OnUnhandledUserException(Exception ex)
{
if (IsThreadAbortException(ex))
return;
base.OnUnhandledUserException(ex);
}
private void MonitorActiveClients(object state)
{
if (m_clientInfoLookup == null)
return;
List oldClients = new List();
// Maintain client info lookup table size by removing clients that haven't been active in the last minute
foreach (TransportProvider clientInfo in m_clientInfoLookup.Values)
{
if ((DateTime.UtcNow - clientInfo.Provider).TotalMinutes > 1.0D)
oldClients.Add(clientInfo.ID);
}
foreach (Guid client in oldClients)
{
if (m_clientInfoLookup.TryRemove(client, out TransportProvider _))
OnClientDisconnected(client);
}
}
#endregion
#region [ Static ]
// Static Methods
internal static bool IsThreadAbortException(Exception ex)
{
while (ex != null)
{
if (ex is ThreadAbortException)
return true;
ex = ex.InnerException;
}
return false;
}
#endregion
}
}