Merge pull request #2 from AdysTech/mvadu-patch-1

Updated Generic file processing
AdysTech · Sep 24, 2015 · 699e475 · 699e475
2 parents 2f21fc4 + 6c989a3
commit 699e475
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 13 deletions.
diff --git a/Influxer/GenericColumn.cs b/Influxer/GenericColumn.cs
@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AdysTech.Influxer
+{
+    class GenericColumn
+    {
+        public int ColumnIndex { get; set; }
+        public string ColumnHeader { get; set; }
+    }
+}
diff --git a/Influxer/Influxer.csproj b/Influxer/Influxer.csproj
@@ -11,6 +11,21 @@
     <AssemblyName>Influxer</AssemblyName>
     <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
+    <PublishUrl>publish\</PublishUrl>
+    <Install>true</Install>
+    <InstallFrom>Disk</InstallFrom>
+    <UpdateEnabled>false</UpdateEnabled>
+    <UpdateMode>Foreground</UpdateMode>
+    <UpdateInterval>7</UpdateInterval>
+    <UpdateIntervalUnits>Days</UpdateIntervalUnits>
+    <UpdatePeriodically>false</UpdatePeriodically>
+    <UpdateRequired>false</UpdateRequired>
+    <MapFileExtensions>true</MapFileExtensions>
+    <ApplicationRevision>0</ApplicationRevision>
+    <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
+    <IsWebBootstrapper>false</IsWebBootstrapper>
+    <UseApplicationTrust>false</UseApplicationTrust>
+    <BootstrapperEnabled>true</BootstrapperEnabled>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>AnyCPU</PlatformTarget>
@@ -31,6 +46,7 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
+  <PropertyGroup />
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -43,13 +59,31 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="ExtensionMethods.cs" />
+    <Compile Include="GenericColumn.cs" />
     <Compile Include="PerfmonCounter.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
   </ItemGroup>
+  <ItemGroup>
+    <BootstrapperPackage Include=".NETFramework,Version=v4.5">
+      <Visible>False</Visible>
+      <ProductName>Microsoft .NET Framework 4.5 %28x86 and x64%29</ProductName>
+      <Install>true</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Net.Client.3.5">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
+      <Install>false</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1</ProductName>
+      <Install>false</Install>
+    </BootstrapperPackage>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

diff --git a/Influxer/Program.cs b/Influxer/Program.cs
@@ -43,6 +43,7 @@ enum FileFormats
         const string TableNameSwitch = "-table";
         const string FilterSwitch = "-filter";
         const string ColumnsSwitch = "-columns";
+        const string UtcOffsetSwitch = "-gmtoffset";
 
         private static string influxUrl;
         private static string influxDB;
@@ -57,6 +58,7 @@ enum FileFormats
         private static Regex pattern;
         private static Filters filter;
         private static string filteredColumns;
+        private static int utcOffsetMin = 0;
 
         private static char[] influxIdentifiers = new char[] { ' ', ';', '_', '(', ')', '%', '#', '.', '/', '[', ']', '{', '}', '"' };
         private static char[] whiteSpace = new char[] { '_' };
@@ -79,6 +81,8 @@ static int Main(string[] args)
                 Console.WriteLine ("-table <table name> :   Influx Table name needed for generic fomat only");
                 Console.WriteLine ("-filter <filter>    :   supported:measurement, field, columns.");
                 Console.WriteLine ("-columns <columns>  :   Comma seperated list of columns from input files");
+                Console.WriteLine ("-utcoffset          :   supported:measurement, field, columns.Offset in minutes to UTC. Applicable to Generic format only");
+
                 Console.WriteLine ("-filter-measurement or field is to restrict the input file to only measurements or fileds that already present in the database");
                 Console.WriteLine ("-filter-columns is to restrict to only few columns from the input irrespective of existing data in database");
                 Console.WriteLine ("-columns will be ignored in other cases. Replace any inline commas in columns names with a space!");
@@ -151,6 +155,7 @@ static int Main(string[] args)
                     return 1;
                 }
             }
+            else
             {
                 fileFormat = FileFormats.Perfmon;
             }
@@ -185,8 +190,8 @@ static int Main(string[] args)
 
                 try
                 {
-                    var temp = ParsePerfMonFileHeader (filteredColumns, false);
-                    if ( temp.Count == 0 )
+                    if ( ( fileFormat == FileFormats.Perfmon && ParsePerfMonFileHeader (filteredColumns, false).Count == 0 ) ||
+                        ( fileFormat == FileFormats.Generic && ParseGenericColumns (filteredColumns).Count == 0 ) )
                     {
                         Console.WriteLine ("No columns filtered!!");
                         return 1;
@@ -199,6 +204,10 @@ static int Main(string[] args)
                 }
             }
 
+            if ( cmdArgs.ContainsKey (UtcOffsetSwitch) )
+            {
+                utcOffsetMin = int.Parse (cmdArgs[UtcOffsetSwitch]);
+            }
 
             #endregion
             try
@@ -531,7 +540,7 @@ private static List<PerfmonCounter> FilterPerfmonLogColumns(List<PerfmonCounter>
                 case Filters.Measurement:
                     return columns.Where (p => dbStructure.ContainsKey (p.PerformanceObject)).ToList ();
                 case Filters.Field:
-                    return columns.Where (p => dbStructure[p.PerformanceObject].Contains (p.CounterName)).ToList ();
+                    return columns.Where (p => dbStructure.ContainsKey (p.PerformanceObject) && dbStructure[p.PerformanceObject].Contains (p.CounterName)).ToList ();
                 case Filters.Columns:
                     return columns.Where (p => filterColumns.Any (f => p.PerformanceObject == f.PerformanceObject && p.CounterName == f.CounterName)).ToList ();
             }
@@ -620,11 +629,23 @@ private static async Task<bool> ProcessGenericFile(string InputFileName, string
                 var failedCount = 0;
 
                 var influxAddress = new Uri (influxUrl + "/write?db=" + influxDB + "&precision=s");
-                client.BaseAddress = influxAddress;
-                List<string> columnHeaders = new List<string> ();
+
+                List<GenericColumn> columnHeaders;
 
                 var firstLine = File.ReadLines (InputFileName).FirstOrDefault ();
-                columnHeaders.AddRange (pattern.Split (firstLine.Replace ("\"", "")));
+                var index = 0;
+                columnHeaders = ParseGenericColumns (firstLine);
+
+                Dictionary<string, List<string>> dbStructure;
+                IEnumerable<IGrouping<string, PerfmonCounter>> perfGroup;
+                if ( filter != Filters.None )
+                {
+                    var filterColumns = ParseGenericColumns (filteredColumns);
+
+                    dbStructure = await GetInfluxDBStructureAsync (client, new Uri (influxUrl + "/query?"), influxDB);
+                    columnHeaders = FilterGenericColumns (columnHeaders, filterColumns, dbStructure);
+
+                }
 
                 //Parallel.ForEach (File.ReadLines (inputFileName).Skip (1), (string line) =>
                 foreach ( var line in File.ReadLines (InputFileName).Skip (1) )
@@ -661,7 +682,29 @@ private static async Task<bool> ProcessGenericFile(string InputFileName, string
             return true;
         }
 
-        private async static Task<bool> ProcessGenericLine(string line, List<string> columnHeaders, Regex pattern, HttpClient client, Uri InfluxPath)
+        private static List<GenericColumn> ParseGenericColumns(string headerLine)
+        {
+            var columns = new List<GenericColumn> ();
+            int index = 0;
+            columns.AddRange (pattern.Split (headerLine).Select (s => new GenericColumn () {ColumnIndex=index++, ColumnHeader = s.Replace (influxIdentifiers, "_") }));
+            return columns;
+        }
+
+        private static List<GenericColumn> FilterGenericColumns(List<GenericColumn> columns, List<GenericColumn> filterColumns, Dictionary<string, List<string>> dbStructure)
+        {
+            switch ( filter )
+            {
+                case Filters.Measurement:
+                    return columns.Where (p => dbStructure.ContainsKey (tableName)).ToList ();
+                case Filters.Field:
+                    return columns.Where (p => dbStructure.ContainsKey (tableName) && dbStructure[tableName].Contains (p.ColumnHeader)).ToList ();
+                case Filters.Columns:
+                    return columns.Where (p => filterColumns.Any (f => f.ColumnHeader == p.ColumnHeader)).ToList ();
+            }
+            return columns;
+        }
+
+        private async static Task<bool> ProcessGenericLine(string line, List<GenericColumn> columnHeaders, Regex pattern, HttpClient client, Uri InfluxPath)
         {
             StringBuilder content = new StringBuilder ();
             DateTime timeStamp;
@@ -672,7 +715,7 @@ private async static Task<bool> ProcessGenericLine(string line, List<string> col
 
                 if ( !DateTime.TryParseExact (columns[0], timeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out timeStamp) )
                     throw new FormatException ("Couldn't parse " + columns[0] + " using format " + timeFormat + ", check -timeformat argument");
-                var epoch = timeStamp.ToEpoch ();
+                var epoch = timeStamp.AddMinutes (utcOffsetMin).ToEpoch ();
 
                 double value = 0.0;
                 content.AppendFormat ("{0}", tableName);
@@ -681,10 +724,10 @@ private async static Task<bool> ProcessGenericLine(string line, List<string> col
                 else
                     content.Append (" ");
 
-                for ( var i = 1; i < columnCount; i++ )
+                foreach ( var c in columnHeaders )
                 {
-                    if ( Double.TryParse (columns[i], out value) )
-                        content.AppendFormat ("{0}={1:0.00},", columnHeaders[i], value);
+                    if ( Double.TryParse (columns[c.ColumnIndex], out value) )
+                        content.AppendFormat ("{0}={1:0.00},", c.ColumnHeader, value);
                 }
 
                 content.AppendFormat (" {0}\n", epoch);

diff --git a/Influxer/Properties/AssemblyInfo.cs b/Influxer/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion ("1.0.0.0")]
-[assembly: AssemblyFileVersion ("1.0.0.0")]
+[assembly: AssemblyVersion ("0.0.1.1")]
+[assembly: AssemblyFileVersion ("0.0.1.1")]