//****************************************************************************************************** // MeasurementExtensions.cs - Gbtc // // Copyright © 2015, Grid Protection Alliance. All Rights Reserved. // // Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See // the NOTICE file distributed with this work for additional information regarding copyright ownership. // The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this // file except in compliance with the License. You may obtain a copy of the License at: // // http://opensource.org/licenses/MIT // // Unless agreed to in writing, the subject software distributed under the License is distributed on an // "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the // License for the specific language governing permissions and limitations. // // Code Modification History: // ---------------------------------------------------------------------------------------------------- // 10/22/2015 - Ritchie Carroll // Generated original version of source code. // //****************************************************************************************************** using System; using System.Runtime.CompilerServices; using GSF; using GSF.TimeSeries; using Misakai.Kafka; namespace KafkaAdapters { #region [ Enumerations ] /// /// Defines the data quality flags applied to a Kafka serialized measurement. /// [Flags] public enum DataQualityFlags : byte { /// /// If no flags are set, measurement quality is normal. /// Normal = (byte)Bits.Nil, /// /// If bit 0 is set, measurement value quality is bad - as reported by source. /// ValueQualityIsBad = (byte)Bits.Bit00, /// /// If bit 1 is set, measurement time quality is bad - as reported by source. /// TimeQualityIsBad = (byte)Bits.Bit01, /// /// Decode version mask - remaining bits used for decoding version, allows 64 possible versions (0 to 63). /// DecodeVersionMask = (byte)(Bits.Bit02 | Bits.Bit03 | Bits.Bit04 | Bits.Bit05 | Bits.Bit06 | Bits.Bit07) } #endregion /// /// Defines extension methods for handling Kafka message serialization to and from time-series measurements. /// /// /// The goal of this serialization implementation is to make it very simple to deserialize /// Kafka time-series messages from other language implementations, such as C++ or Java. /// public static class SerializationExtensions { /// /// Defines the size of serialized Kafka measurement key. /// /// /// Key format:
/// - long Time: 64-bit time of measurement in ticks, i.e., 100-nanosecond intervals since 0/0/0000
/// - uint ID: 32-bit measurement identifier (temporal index into metadata) ///
public const int KeySize = sizeof(long) + sizeof(uint); // 12 bytes /// /// Defines the size of serialized Kafka measurement value. /// /// /// Value format:
/// - byte Version: 8-bit metadata serialization version
/// - byte Quality: 8-bit measurement quality - bit 0 = bad value, bit 1 = bad time*
/// - double Value: 64-bit floating-point measurement value /// * Remaining bits used to define decoding version. ///
public const int ValueSize = sizeof(byte) + sizeof(byte) + sizeof(double); // 10 bytes /// /// Defines the current message decoding version. /// public const byte DecodeVersion = 0; /// /// Serializes a time-series instance into key-value buffers for a Kafka message. /// /// instance to serialize. /// Kafka message to hold serialized key-value buffers. /// Current metadata serialization version. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void KakfaSerialize(this IMeasurement measurement, Message message, byte metadataVersion) { if ((object)measurement == null) throw new ArgumentNullException(nameof(measurement)); if ((object)message == null) throw new ArgumentNullException(nameof(message)); // Copy in "key" for Kafka message, for a time-series entity, the "timestamp" and "ID" message.Key = new byte[KeySize]; Buffer.BlockCopy(BitConverter.GetBytes(measurement.Timestamp), 0, message.Key, 0, 8); Buffer.BlockCopy(BitConverter.GetBytes(measurement.Key.ID), 0, message.Key, 8, 4); // Copy in "value" for Kafka message, for a time-series entity, the "version", "quality" and "value" message.Value = new byte[ValueSize]; message.Value[0] = metadataVersion; message.Value[1] = (byte)((int)measurement.StateFlags.DeriveQualityFlags() | (DecodeVersion << 2)); Buffer.BlockCopy(BitConverter.GetBytes(measurement.AdjustedValue), 0, message.Value, 2, 8); } /// /// Deserializes a Kafka message into a time-series . /// /// Time-series serialized Kafka message. /// Numeric ID of measurement deserialized from Kafka message. /// Message metadata serialization version. /// value deserialized from a time-series serialized Kafka message. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Measurement KafkaDeserialize(this Message message, out uint id, out byte metadataVersion) { if ((object)message == null) throw new ArgumentNullException(nameof(message)); if (message.Key.Length < KeySize) throw new ArgumentOutOfRangeException(nameof(message.Key), $"Buffer size too small - expected {KeySize} bytes for Kafka time-series message key"); if (message.Value.Length < ValueSize) throw new ArgumentOutOfRangeException(nameof(message.Value), $"Buffer size too small - expected {ValueSize} bytes for Kafka time-series message value"); byte messageDecodeVersion = (byte)((message.Value[1] & (byte)DataQualityFlags.DecodeVersionMask) >> 2); if (messageDecodeVersion > DecodeVersion) throw new InvalidOperationException($"This Kafka time-series deserialization engine cannot decode version {messageDecodeVersion} serialized messages"); id = BitConverter.ToUInt32(message.Key, 8); metadataVersion = message.Value[0]; return new Measurement { Timestamp = BitConverter.ToInt64(message.Key, 0), Value = BitConverter.ToDouble(message.Value, 2), StateFlags = DeriveStateFlags((DataQualityFlags)message.Value[1] & ~DataQualityFlags.DecodeVersionMask) }; } /// /// Derive Kafka time-seres data quality flags from measurement state flags /// /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static DataQualityFlags DeriveQualityFlags(this MeasurementStateFlags stateFlags) { DataQualityFlags qualityFlags = DataQualityFlags.Normal; if (stateFlags.HasFlag(MeasurementStateFlags.BadData)) qualityFlags |= DataQualityFlags.ValueQualityIsBad; if (stateFlags.HasFlag(MeasurementStateFlags.BadTime)) qualityFlags |= DataQualityFlags.TimeQualityIsBad; return qualityFlags; } /// /// Derive measurement state flags from Kafka time-seres data quality flags /// /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MeasurementStateFlags DeriveStateFlags(this DataQualityFlags qualityFlags) { MeasurementStateFlags stateFlags = MeasurementStateFlags.Normal; if (qualityFlags.HasFlag(DataQualityFlags.ValueQualityIsBad)) stateFlags |= MeasurementStateFlags.BadData; if (qualityFlags.HasFlag(DataQualityFlags.TimeQualityIsBad)) stateFlags |= MeasurementStateFlags.BadTime; return stateFlags; } } }