//******************************************************************************************************//  GrafanaDataSourceBase_FunctionExecution.cs - Gbtc////  Copyright © 2020, Grid Protection Alliance.  All Rights Reserved.////  Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See//  the NOTICE file distributed with this work for additional information regarding copyright ownership.//  The GPA licenses this file to you under the MIT License (MIT), the "License"; you may not use this//  file except in compliance with the License. You may obtain a copy of the License at:////      http://opensource.org/licenses/MIT////  Unless agreed to in writing, the subject software distributed under the License is distributed on an//  "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the//  License for the specific language governing permissions and limitations.////  Code Modification History://  ----------------------------------------------------------------------------------------------------//  09/25/2020 - J. Ritchie Carroll//       Generated original version of source code.////****************************************************************************************************** using System;using System.Collections.Generic;using System.Data;using System.Linq;using System.Reflection;using System.Text.RegularExpressions;using System.Threading;using Ciloci.Flee;using GSF;using GSF.Collections;using GSF.NumericalAnalysis;using GSF.Units;using ParsedFunction = System.Tuple<GrafanaAdapters.SeriesFunction, string, GrafanaAdapters.GroupOperation, string>; // ReSharper disable PossibleMultipleEnumeration// ReSharper disable AccessToModifiedClosure// ReSharper disable SwitchStatementHandlesSomeKnownEnumValuesWithDefaultnamespace GrafanaAdapters{    partial class GrafanaDataSourceBase    {        private class TargetTimeUnit        {            public TimeUnit Unit;            public double Factor = double.NaN;             public static bool TryParse(string value, out TargetTimeUnit targetTimeUnit)            {                if (Enum.TryParse(value, out TimeUnit timeUnit))                {                    targetTimeUnit = new()                    {                        Unit = timeUnit                    };                     return true;                }                 switch (value?.ToLowerInvariant())                {                    case "milliseconds":                        targetTimeUnit = new()                        {                            Unit = TimeUnit.Seconds,                            Factor = SI.Milli                        };                         return true;                    case "microseconds":                        targetTimeUnit = new()                        {                            Unit = TimeUnit.Seconds,                            Factor = SI.Micro                        };                         return true;                    case "nanoseconds":                        targetTimeUnit = new()                        {                            Unit = TimeUnit.Seconds,                            Factor = SI.Nano                        };                         return true;                }                 targetTimeUnit = null;                return false;            }        }         private IEnumerable<DataSourceValueGroup> ExecuteSeriesFunction(Target sourceTarget, ParsedFunction parsedFunction, DateTime startTime, DateTime stopTime, string interval, bool includePeaks, bool dropEmptySeries, CancellationToken cancellationToken)        {            SeriesFunction seriesFunction = parsedFunction.Item1;            string expression = parsedFunction.Item2;            GroupOperation groupOperation = parsedFunction.Item3;            string imports = parsedFunction.Item4;             // Parse out function parameters and target expression            Tuple<string[], string> expressionParameters = TargetCache<Tuple<string[], string>>.GetOrAdd(expression, () =>            {                List<string> parsedParameters = new();                 if (seriesFunction == SeriesFunction.Evaluate)                {                    int lastIndex = 0;                     for (int i = 0; i < 2; i++)                    {                        if (i == 1)                        {                            // Eval expression required to be in braces, i.e., { expression }, to delineate sub-expression that may have its own functions and parameters.                            int startBrace = expression.IndexOf('{', lastIndex);                             if (startBrace < 0)                                break;                             startBrace++;                             int endBrace = expression.IndexOf('}', startBrace);                             if (endBrace < 0)                                break;                             parsedParameters.Add(expression.Substring(startBrace, endBrace - startBrace));                             lastIndex = expression.IndexOf(',', endBrace);                             if (lastIndex < 0)                                throw new FormatException($"Expected 3 parameters, received 2 in: Evaluate({expression})");                             lastIndex++;                        }                        else                        {                            int nextIndex = expression.IndexOf(',', lastIndex);                             if (nextIndex < 0)                                break;                             parsedParameters.Add(expression.Substring(lastIndex, nextIndex - lastIndex));                            lastIndex = nextIndex + 1;                        }                    }                                        if (parsedParameters.Count == 2)                        expression = expression.Substring(lastIndex).Trim();                    else                        throw new FormatException($"Expected 3 parameters, received {parsedParameters.Count + 1} in: Evaluate({expression})");                }                else                {                    // Extract any required function parameters                    int requiredParameters = s_requiredParameters[seriesFunction]; // Safe: no lock needed since content doesn't change                     // Any slice operation adds one required parameter for time tolerance                    if (groupOperation == GroupOperation.Slice)                        requiredParameters++;                     if (requiredParameters > 0)                    {                        int index = 0;                         for (int i = 0; i < requiredParameters && index > -1; i++)                            index = expression.IndexOf(',', index + 1);                         if (index > -1)                            parsedParameters.AddRange(expression.Substring(0, index).Split(','));                         if (parsedParameters.Count == requiredParameters)                            expression = expression.Substring(index + 1).Trim();                        else                            throw new FormatException($"Expected {requiredParameters + 1} parameters, received {parsedParameters.Count + 1} in: {(groupOperation == GroupOperation.None ? "" : groupOperation.ToString())}{seriesFunction}({expression})");                    }                     // Extract any provided optional function parameters                    int optionalParameters = s_optionalParameters[seriesFunction]; // Safe: no lock needed since content doesn't change                     bool hasSubExpression(string target) => target.StartsWith("FILTER", StringComparison.OrdinalIgnoreCase) || target.Contains("(");                     if (optionalParameters > 0)                    {                        int index = expression.IndexOf(',');                         if (index > -1 && !hasSubExpression(expression.Substring(0, index)))                        {                            int lastIndex = index;                             for (int i = 1; i < optionalParameters && index > -1; i++)                            {                                index = expression.IndexOf(',', index + 1);                                 if (index > -1 && hasSubExpression(expression.Substring(lastIndex + 1, index - lastIndex - 1).Trim()))                                {                                    index = lastIndex;                                    break;                                }                                 lastIndex = index;                            }                             if (index > -1)                            {                                parsedParameters.AddRange(expression.Substring(0, index).Split(','));                                expression = expression.Substring(index + 1).Trim();                            }                        }                    }                }                 return new(parsedParameters.ToArray(), expression);            });             string[] parameters = expressionParameters.Item1;            string queryExpression = expressionParameters.Item2;   // Final function parameter is always target expression             // When accurate calculation results are requested, query data source at full resolution            if (seriesFunction == SeriesFunction.Interval && ParseFloat(parameters[0]) == 0.0D)            {                includePeaks = false;                interval = "0s";            }             // Query function expression to get series data            IEnumerable<DataSourceValueGroup> dataset = QueryTarget(sourceTarget, queryExpression, startTime, stopTime, interval, includePeaks, dropEmptySeries, imports, cancellationToken);             switch (seriesFunction)            {                // Handle label function as a special edge case - group operations on label are ignored                case SeriesFunction.Label:                {                    // Derive labels                    string label = parameters[0];                     if (label.StartsWith("\"") || label.StartsWith("'"))                        label = label.Substring(1, label.Length - 2);                     DataSourceValueGroup[] valueGroups = dataset.ToArray();                    string[] seriesLabels = new string[valueGroups.Length];                    string[] components = label.Split('.');                    string table;                     if (components.Length == 2)                    {                        table = components[0].Trim();                        label = components[1].Trim();                    }                    else                    {                        table = "ActiveMeasurements";                    }                     bool labelHasSubstitution = label.IndexOf('{') >= 0;                     for (int i = 0; i < valueGroups.Length; i++)                    {                        string rootTarget = valueGroups[i].RootTarget;                         seriesLabels[i] = TargetCache<string>.GetOrAdd($"{label}@{rootTarget}", () =>                        {                            if (!labelHasSubstitution)                                return label;                             Dictionary<string, string> substitutions = new();                             foreach (string item in rootTarget.Split(';'))                            {                                string target = item.SplitAlias(out string alias);                                 if (substitutions.TryGetValue("alias", out string substitution))                                {                                    if (!string.IsNullOrWhiteSpace(alias))                                        substitutions["alias"] = string.IsNullOrWhiteSpace(substitution) ? alias : $"{substitution}, {alias}";                                }                                else                                {                                    substitutions.Add("alias", alias ?? "");                                }                                 DataRow record = target.MetadataRecordFromTag(Metadata, table);                                 if (record is null)                                {                                    foreach (string fieldName in Metadata.Tables[table].Columns.Cast<DataColumn>().Select(column => column.ColumnName))                                    {                                        if (fieldName.Equals("PointTag", StringComparison.OrdinalIgnoreCase))                                            continue;                                         substitutions.Add(fieldName, "");                                    }                                     if (substitutions.TryGetValue("PointTag", out substitution))                                        substitutions["PointTag"] = $"{substitution}, {target}";                                    else                                        substitutions.Add("PointTag", target);                                }                                else                                {                                    foreach (string fieldName in record.Table.Columns.Cast<DataColumn>().Select(column => column.ColumnName))                                    {                                        string columnValue = record[fieldName].ToString();                                         if (substitutions.TryGetValue(fieldName, out substitution))                                        {                                            if (!string.IsNullOrWhiteSpace(columnValue))                                                substitutions[fieldName] = string.IsNullOrWhiteSpace(substitution) ? columnValue : $"{substitution}, {columnValue}";                                        }                                        else                                        {                                            substitutions.Add(fieldName, columnValue);                                        }                                    }                                }                            }                             string derivedLabel = substitutions.Aggregate(label, (current, kvp) => current.ReplaceCaseInsensitive($"{{{kvp.Key}}}", kvp.Value));                             if (derivedLabel.Equals(label, StringComparison.Ordinal))                                derivedLabel = $"{label}{(valueGroups.Length > 1 ? $" {i + 1}" : "")}";                             return derivedLabel;                        });                    }                     // Verify that all series labels are unique                    if (seriesLabels.Length > 1)                    {                        HashSet<string> uniqueLabelSet = new(StringComparer.OrdinalIgnoreCase);                         for (int i = 0; i < seriesLabels.Length; i++)                        {                            while (uniqueLabelSet.Contains(seriesLabels[i]))                                seriesLabels[i] = $"{seriesLabels[i]}\u00A0"; // Suffixing with non-breaking space for label uniqueness                             uniqueLabelSet.Add(seriesLabels[i]);                        }                    }                     for (int i = 0; i < valueGroups.Length; i++)                    {                        yield return new()                        {                            Target = seriesLabels[i],                            RootTarget = valueGroups[i].RootTarget,                            SourceTarget = sourceTarget,                            Source = valueGroups[i].Source,                            DropEmptySeries = dropEmptySeries,                            refId = sourceTarget.refId                        };                    }                     break;                }                // Handle evaluate function as a special edge case - only slice-based group operation is supported                case SeriesFunction.Evaluate:                {                    TimeSliceScanner scanner = new(dataset, ParseFloat(parameters[0]) / SI.Milli);                    parameters = string.IsNullOrWhiteSpace(imports) ? new[] { parameters[1].Trim() } : new[] { parameters[1].Trim(), imports };                     foreach (DataSourceValueGroup valueGroup in ExecuteSeriesFunctionOverTimeSlices(scanner, SeriesFunction.Evaluate, parameters, cancellationToken))                    {                        yield return new()                        {                            Target = $"Evaluate({{ {parameters[0]} }}, {valueGroup.Target})",                            RootTarget = valueGroup.RootTarget ?? valueGroup.Target,                            SourceTarget = sourceTarget,                            Source = valueGroup.Source,                            DropEmptySeries = dropEmptySeries                        };                    }                     break;                }                default:                    switch (groupOperation)                    {                        case GroupOperation.Set:                        {                            // Flatten all series into a single enumerable                            DataSourceValueGroup valueGroup = new()                            {                                Target = $"Set{seriesFunction}({string.Join(", ", parameters)}{(parameters.Length > 0 ? ", " : "")}{queryExpression})",                                RootTarget = queryExpression,                                SourceTarget = sourceTarget,                                Source = ExecuteSeriesFunctionOverSource(dataset.AsParallel().WithCancellation(cancellationToken).SelectMany(source => source.Source), seriesFunction, parameters),                                DropEmptySeries = dropEmptySeries                            };                             // Handle edge-case set operations - for these functions there is data in the target series as well                            if (seriesFunction is SeriesFunction.Minimum or SeriesFunction.Maximum or SeriesFunction.Median)                            {                                DataSourceValue dataValue = valueGroup.Source.First();                                valueGroup.Target = $"Set{seriesFunction} = {dataValue.Target}";                                valueGroup.RootTarget = dataValue.Target;                            }                             yield return valueGroup;                                                    break;                        }                        case GroupOperation.Slice:                        {                            TimeSliceScanner scanner = new(dataset, ParseFloat(parameters[0]) / SI.Milli);                            parameters = parameters.Skip(1).ToArray();                             foreach (DataSourceValueGroup valueGroup in ExecuteSeriesFunctionOverTimeSlices(scanner, seriesFunction, parameters, cancellationToken))                            {                                yield return new()                                {                                    Target = $"Slice{seriesFunction}({string.Join(", ", parameters)}{(parameters.Length > 0 ? ", " : "")}{valueGroup.Target})",                                    RootTarget = valueGroup.RootTarget ?? valueGroup.Target,                                    SourceTarget = sourceTarget,                                    Source = valueGroup.Source,                                    DropEmptySeries = dropEmptySeries                                };                            }                             break;                        }                        default:                        {                            foreach (DataSourceValueGroup valueGroup in dataset)                            {                                yield return new()                                {                                    Target = $"{seriesFunction}({string.Join(", ", parameters)}{(parameters.Length > 0 ? ", " : "")}{valueGroup.Target})",                                    RootTarget = valueGroup.RootTarget ?? valueGroup.Target,                                    SourceTarget = sourceTarget,                                    Source = ExecuteSeriesFunctionOverSource(valueGroup.Source, seriesFunction, parameters),                                    DropEmptySeries = dropEmptySeries,                                    refId = sourceTarget.refId                                };                            }                             break;                        }                    }                     break;            }        }         // Execute series function over a set of points from each series at the same time-slice        private static IEnumerable<DataSourceValueGroup> ExecuteSeriesFunctionOverTimeSlices(TimeSliceScanner scanner, SeriesFunction seriesFunction, string[] parameters, CancellationToken cancellationToken)        {            IEnumerable<DataSourceValue> readSliceValues()            {                while (!scanner.DataReadComplete && !cancellationToken.IsCancellationRequested)                {                    foreach (DataSourceValue dataValue in ExecuteSeriesFunctionOverSource(scanner.ReadNextTimeSlice(), seriesFunction, parameters, true))                        yield return dataValue;                }            }             foreach (IGrouping<string, DataSourceValue> valueGroup in readSliceValues().GroupBy(dataValue => dataValue.Target))            {                yield return new()                {                    Target = valueGroup.Key,                    RootTarget = valueGroup.Key,                    Source = valueGroup                };            }        }         // Design philosophy: whenever possible this function should delay source enumeration since source data sets could be very large.        private static IEnumerable<DataSourceValue> ExecuteSeriesFunctionOverSource(IEnumerable<DataSourceValue> source, SeriesFunction seriesFunction, string[] parameters, bool isSliceOperation = false)        {            DataSourceValue[] values;            DataSourceValue result = new();            double lastValue = double.NaN;            double lastTime = 0.0D;            string lastTarget = null;             IEnumerable<double> trackedValues = source.Select(dataValue =>            {                lastTime = dataValue.Time;                lastTarget = dataValue.Target;                return dataValue.Value;            });             double baseTime, timeStep, value, low, high;            bool normalizeTime, lowInclusive, highInclusive;            int count;            TargetTimeUnit timeUnit;            AngleUnit angleUnit;             switch (seriesFunction)            {                case SeriesFunction.Minimum:                    DataSourceValue minValue = new() { Value = double.MaxValue };                     foreach (DataSourceValue dataValue in source)                    {                        if (dataValue.Value <= minValue.Value)                            minValue = dataValue;                    }

Lines 1–500 of 1194. Load more