//******************************************************************************************************//  CompletenessReportGenerator.cs - Gbtc////  Copyright © 2014, Grid Protection Alliance.  All Rights Reserved.////  Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See//  the NOTICE file distributed with this work for additional information regarding copyright ownership.//  The GPA licenses this file to you under the MIT License (MIT), the "License"; you may//  not use this file except in compliance with the License. You may obtain a copy of the License at:////      http://www.opensource.org/licenses/MIT////  Unless agreed to in writing, the subject software distributed under the License is distributed on an//  "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the//  License for the specific language governing permissions and limitations.////  Code Modification History://  ----------------------------------------------------------------------------------------------------//  02/11/2014 - Stephen C. Wills//       Generated original version of source code.////****************************************************************************************************** using System;using System.Collections.Generic;using System.Drawing;using System.Drawing.Imaging;using System.IO;using System.Linq;using System.Text;using System.Windows.Forms.DataVisualization.Charting;using GSF.Data;using GSF.Data.Model;using GSF.Diagnostics;using GSF.Historian;using GSF.Historian.Files;using GSF.Units;using Root.Reports;using Encoder = System.Drawing.Imaging.Encoder; // ReSharper disable UnusedMember.Local// ReSharper disable RedundantAssignment// ReSharper disable AssignNullToNotNullAttribute// ReSharper disable UnusedMethodReturnValue.Local// ReSharper disable UnusedAutoPropertyAccessor.Localnamespace StatHistorianReportGenerator{    /// <summary>    /// Generates GSF Data Completeness Report for a specified date.    /// </summary>    public class CompletenessReportGenerator    {        #region [ Members ]         // Nested Types        private class DeviceStats        {            public string Name;            public double[] DataQualityErrors;            public double[] TimeQualityErrors;            public double[] MeasurementsReceived;            public double[] MeasurementsExpected;        }         private class DataAvailability        {            [PrimaryKey(true)]            public int ID { get; set; }            public float GoodAvailableData { get; set; }            public float BadAvailableData { get; set; }            public float TotalAvailableData { get; set; }        }         // Constants         // Note: Many of the calculations in this class use conversion        // factors of 96 dots per inch and 25.4 millimeters per inch         private const int ReportDays = 35;  // Gather data for 35 days - necessary for 5-day history summary        private const int Month = 30;       // 1 month = 30 days for reporting purposes         private const double PageMarginMillimeters = 25.4D;             // 1-inch margin        private const double PageWidthMillimeters = 8.5D * 25.4D;       // 8.5 inch width        private const double PageHeightMillimeters = 11.0D * 25.4D;     // 11 inch height         private const double FooterHeightMillimeters = (10.0D / 72.0D) * 25.4D;        private const double SpacingMillimeters = 6.0D;         // Fields        private string m_archiveLocation;        private string m_titleText;        private string m_companyText;        private DateTime m_reportDate;        private double m_level4Threshold;        private double m_level3Threshold;        private string m_level4Alias;        private string m_level3Alias;        private bool m_generateCsvReport;         private List<DeviceStats> m_deviceStatsList;        private float m_systemUpTime;         private string m_reportFilePath;         #endregion         #region [ Constructors ]         /// <summary>        /// Creates a new instance of the <see cref="CompletenessReportGenerator"/> class.        /// </summary>        public CompletenessReportGenerator()        {            m_titleText = "GSF Data Completeness Report";            m_companyText = "Grid Protection Alliance";            m_reportDate = DateTime.UtcNow.Date;            m_level4Threshold = 99.0D;            m_level3Threshold = 90.0D;        }         #endregion         #region [ Properties ]         /// <summary>        /// Gets or sets the location of the statistics archive that provides data for reporting.        /// </summary>        public string ArchiveLocation        {            get            {                return m_archiveLocation;            }            set            {                m_archiveLocation = value;            }        }         /// <summary>        /// Gets or sets the text that appears in the title section of the generated report.        /// </summary>        public string TitleText        {            get            {                return m_titleText;            }            set            {                m_titleText = value;            }        }         /// <summary>        /// Gets or sets the text that appears in the company section of the generated report.        /// </summary>        public string CompanyText        {            get            {                return m_companyText;            }            set            {                m_companyText = value;            }        }         /// <summary>        /// Gets or sets the date for which the report will be generated.        /// </summary>        public DateTime ReportDate        {            get            {                return m_reportDate;            }            set            {                m_reportDate = value.ToUniversalTime().Date;            }        }         /// <summary>        /// Gets or sets the completeness threshold at which devices will be considered to be in Level 4.        /// </summary>        public double Level4Threshold        {            get            {                return m_level4Threshold;            }            set            {                m_level4Threshold = value;                 if (m_level4Threshold > 100.0D)                    m_level4Threshold = 100.0D;                 if (m_level4Threshold < 0.0D)                    m_level4Threshold = 0.0D;                 if (m_level4Threshold < m_level3Threshold)                    m_level3Threshold = m_level4Threshold;            }        }         /// <summary>        /// Gets or sets the completeness threshold at which devices will be considered to be in Level 3.        /// </summary>        public double Level3Threshold        {            get            {                return m_level3Threshold;            }            set            {                m_level3Threshold = value;                 if (m_level3Threshold > 100.0D)                    m_level3Threshold = 100.0D;                 if (m_level3Threshold < 0.0D)                    m_level3Threshold = 0.0D;                 if (m_level3Threshold > m_level4Threshold)                    m_level4Threshold = m_level3Threshold;            }        }         /// <summary>        /// Gets or sets the alias to be associated with the Level 4 category.        /// </summary>        public string Level4Alias        {            get            {                return m_level4Alias;            }            set            {                m_level4Alias = value;            }        }         /// <summary>        /// Gets or sets the alias to be associated with the Level 3 category.        /// </summary>        public string Level3Alias        {            get            {                return m_level3Alias;            }            set            {                m_level3Alias = value;            }        }         public bool GenerateCsvReport        {            get            {                return m_generateCsvReport;            }            set            {                m_generateCsvReport = value;            }        }         public string ReportFilePath        {            get            {                return m_reportFilePath;            }            set            {                m_reportFilePath = value;            }        }         #endregion         #region [ Methods ]         /// <summary>        /// Generates the report.        /// </summary>        /// <returns>The report that was generated.</returns>        public Report GenerateReport()        {            DateTime now = DateTime.Now;            Report report = new Report();            FontDef fontDefinition = new FontDef(report, "Helvetica");            Page pageOne = CreatePage(report);            Page pageTwo = CreatePage(report);            double verticalMillimeters;             ReadStatistics();             if (m_generateCsvReport && m_reportFilePath != null)                GenerateCsvReportFunction(m_reportFilePath);             PopulateDataAvailabilityTable();             // Page one            verticalMillimeters = PageMarginMillimeters;            verticalMillimeters += InsertTitle(fontDefinition, pageOne, verticalMillimeters) + SpacingMillimeters;            verticalMillimeters += InsertReportDate(fontDefinition, pageOne, verticalMillimeters);            verticalMillimeters += InsertSystemUpTime(fontDefinition, pageOne, verticalMillimeters) + 1.5D * SpacingMillimeters;             verticalMillimeters += InsertSectionHeader(fontDefinition, pageOne, verticalMillimeters, "5-day Device Data Completeness");            verticalMillimeters += InsertFiveDaySummary(report, fontDefinition, pageOne, verticalMillimeters) + SpacingMillimeters;             verticalMillimeters += InsertSectionHeader(fontDefinition, pageOne, verticalMillimeters, "Percent of Devices with Acceptable Availability (30 days)");            verticalMillimeters += InsertBarChart(pageOne, verticalMillimeters) + SpacingMillimeters;             verticalMillimeters += InsertSectionHeader(fontDefinition, pageOne, verticalMillimeters, "Definitions");            verticalMillimeters += InsertDefinitions(fontDefinition, pageOne, verticalMillimeters) + SpacingMillimeters;            InsertFooter(fontDefinition, pageOne, now, 1);             // Page two            verticalMillimeters = PageMarginMillimeters;            verticalMillimeters += InsertReportDate(fontDefinition, pageTwo, verticalMillimeters) + SpacingMillimeters;             verticalMillimeters += InsertSectionHeader(fontDefinition, pageTwo, verticalMillimeters, "Data Completeness Breakdown");            verticalMillimeters += InsertPieChart(fontDefinition, pageTwo, verticalMillimeters) + SpacingMillimeters;             InsertDetailsList(report, fontDefinition, pageTwo, verticalMillimeters, now, 2);             return report;        }         private void PopulateDataAvailabilityTable()        {            const int DefaultRecordDepth = 14;             try            {                using (AdoDataConnection connection = Program.GetDatabaseConnection())                {                    if ((object)connection == null)                        return;                     TableOperations<DataAvailability> dataAvailabilityTable = new TableOperations<DataAvailability>(connection);                    DataAvailability[] records = dataAvailabilityTable.QueryRecords().ToArray();                     if (records.Length == 0)                    {                        records = new DataAvailability[DefaultRecordDepth];                         for (int i = 0; i < records.Length; i++)                            records[i] = dataAvailabilityTable.NewRecord();                    }                     for (int i = records.Length - 1; i >= 0; i--)                    {                        List<DeviceStats>[] levels = GetLevels(ReportDays - (i + 1));                        DataAvailability record = records[i];                         record.GoodAvailableData = levels[4].Count;                        record.BadAvailableData = levels.Take(4).Sum(level => level.Count);                        record.TotalAvailableData = levels.Sum(level => level.Count);                         dataAvailabilityTable.AddNewOrUpdateRecord(record);                    }                }            }            catch (InvalidOperationException ex)            {                Program.Log.Publish(MessageLevel.Info, "PopulateDataAvailabilityTable", "Failed to populate data availability table.", "Table may not exist in host configuration.", ex);            }            catch (Exception ex)            {                Program.Log.Publish(MessageLevel.Error, "PopulateDataAvailabilityTable", "Failed to populate data availability table.", exception: ex);            }        }         private void GenerateCsvReportFunction(string filepath)        {            filepath = Path.ChangeExtension(filepath, ".csv");            StreamWriter writer = File.CreateText(filepath);            writer.WriteLine("Name,Completeness,DataErrors,Time Errors,StatDate");             List<DeviceStats>[] levels = GetLevels(ReportDays);             for(int level = 0; level < levels.Length; level++)            {                // Sort the devices in this level by completeness, then data errors descending, then name                levels[level] = levels[level].OrderBy(device => Math.Round(device.MeasurementsReceived[ReportDays - 1] / device.MeasurementsExpected[ReportDays - 1], 4)).ThenByDescending(device => device.DataQualityErrors[ReportDays - 1]).ThenBy(device => device.Name).ToList();                 foreach (DeviceStats stats in levels[level])                {                    writer.WriteLine($"{stats.Name},{(stats.MeasurementsReceived[ReportDays - 1] / stats.MeasurementsExpected[ReportDays - 1]):P2},{stats.DataQualityErrors[ReportDays - 1]},{stats.TimeQualityErrors[ReportDays - 1]},{m_reportDate:d/M/yyyy}");                }            }              writer.Dispose();        }         // Creates a page and sets width and height to standard 8.5x11 inches.        private Page CreatePage(Report report)        {            Page page = new Page(report);            page.rWidthMM = PageWidthMillimeters;            page.rHeightMM = PageHeightMillimeters;            return page;        }         // Reads statistics from the archive and stores them in a format that can        // be easily manipulated to determine which devices belong in which levels.        private void ReadStatistics()        {            Dictionary<string, DeviceStats> deviceStatsLookup = new Dictionary<string, DeviceStats>();            DateTime startTime;            DateTime endTime;             ArchiveLocator locator;            Dictionary<MetadataRecord, IEnumerable<IDataPoint>> measurementsReceived;            Dictionary<MetadataRecord, IEnumerable<IDataPoint>> measurementsExpected;            Dictionary<MetadataRecord, IEnumerable<IDataPoint>> dataQualityErrors;            Dictionary<MetadataRecord, IEnumerable<IDataPoint>> timeQualityErrors;             MetadataRecord dataQualityRecord;            MetadataRecord timeQualityRecord;             DeviceStats deviceStats;            string signalReference;            string deviceName;            int index;             // Create the statistics reader for reading statistics from the archive            using (StatisticsReader statisticsReader = new StatisticsReader())            {                // Create the archive locater to                // determine the location of the archive                locator = new ArchiveLocator()                {                    ArchiveLocation = m_archiveLocation,                    ArchiveLocationName = "Statistics",                    ArchiveName = "STAT"                };                 endTime = m_reportDate + TimeSpan.FromDays(1);                startTime = endTime - TimeSpan.FromDays(ReportDays);                 // Set up and open the statistics reader                statisticsReader.StartTime = startTime;                statisticsReader.EndTime = endTime;                statisticsReader.ArchiveFilePath = locator.ArchiveFilePath;                statisticsReader.Open();                 m_systemUpTime = statisticsReader                    .Read("SYSTEM", 15)                    .SingleOrDefault(kvp => kvp.Key.GeneralFlags.Enabled).Value?                    .LastOrDefault()?.Value ?? 0.0F;                 measurementsReceived = statisticsReader.Read("PMU", 4);                measurementsExpected = statisticsReader.Read("PMU", 5);                dataQualityErrors = statisticsReader.Read("PMU", 1);                timeQualityErrors = statisticsReader.Read("PMU", 2);                 // Determine which devices in the archive have stats for both measurements received and measurements expected                foreach (Tuple<MetadataRecord, MetadataRecord> tuple in measurementsReceived.Keys.Join(measurementsExpected.Keys, GetDeviceName, GetDeviceName, Tuple.Create))                {                    signalReference = tuple.Item1.Synonym1;                    deviceName = GetDeviceName(tuple.Item1);                     // Ignore statistics that were calculated by an intermediate gateway                    if (!signalReference.StartsWith("LOCAL$") && signalReference.Contains("LOCAL$"))                        continue;                     // Make sure LOCAL$ statistics take precedence over other statistics calculated for the same device                    if (deviceStatsLookup.ContainsKey(deviceName) && !signalReference.StartsWith("LOCAL$"))                        continue;                     dataQualityRecord = dataQualityErrors.Keys.FirstOrDefault(record => GetDeviceName(record) == deviceName);                    timeQualityRecord = timeQualityErrors.Keys.FirstOrDefault(record => GetDeviceName(record) == deviceName);                     // Create arrays to hold the total sum of the stats for each day being reported                    deviceStats = new DeviceStats()                    {                        Name = deviceName,                        DataQualityErrors = new double[ReportDays],                        TimeQualityErrors = new double[ReportDays],                        MeasurementsReceived = new double[ReportDays],                        MeasurementsExpected = new double[ReportDays]                    };                     if ((object)dataQualityRecord != null)                    {                        // Calculate the total data quality errors for each day being reported                        foreach (IDataPoint dataPoint in dataQualityErrors[dataQualityRecord])                        {

Lines 1–500 of 1286. Load more