Skip to content

Commit

Permalink
Merge pull request #20 from AdysTech/wip
Browse files Browse the repository at this point in the history
Project file hierarchy updated
  • Loading branch information
mvadu authored Oct 29, 2016
2 parents b21b412 + 3fa3324 commit 8f17470
Show file tree
Hide file tree
Showing 24 changed files with 269 additions and 180 deletions.
15 changes: 10 additions & 5 deletions Influxer.Test/Influxer.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,12 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="AdysTech.InfluxDB.Client.Net, Version=0.5.3.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\AdysTech.InfluxDB.Client.Net.0.5.3\lib\net451\AdysTech.InfluxDB.Client.Net.dll</HintPath>
<Reference Include="AdysTech.InfluxDB.Client.Net, Version=0.5.9.1, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\AdysTech.InfluxDB.Client.Net.0.5.9.1\lib\net451\AdysTech.InfluxDB.Client.Net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
Expand Down Expand Up @@ -65,16 +69,17 @@
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="InfluxerTest.orderedtest">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="TestSetup.ps1" />
<None Include="packages.config" />
<None Include="TestFiles\HeaderlessText.txt" />
<None Include="TestFiles\HeaderlessText.conf" />
<None Include="TestFiles\Perfmon.csv" />
<None Include="TestFiles\Simple-2.csv" />
<None Include="TestFiles\Simple.csv" />
</ItemGroup>
<ItemGroup>
<Content Include="TestFiles\MicroSecSemicolonSeperated.txt" />
</ItemGroup>
<Choose>
<When Condition="'$(VisualStudioVersion)' == '10.0' And '$(IsCodedUITest)' == 'True'">
<ItemGroup>
Expand Down
10 changes: 0 additions & 10 deletions Influxer.Test/InfluxerTest.orderedtest

This file was deleted.

60 changes: 42 additions & 18 deletions Influxer.Test/InfluxerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Threading.Tasks;
using AdysTech.Influxer;
using System.IO;
using System.Diagnostics;

namespace Influxer.Test
{
Expand Down Expand Up @@ -39,14 +40,29 @@ public string TestFilesPath

}

private static async Task<InfluxDBClient> GetClientAsync (InfluxerConfigSection settings)
{
var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password);
var dbNames = await client.GetInfluxDBNamesAsync ();
if (dbNames.Contains (settings.InfluxDB.DatabaseName))
return client;
else
{
await client.CreateDatabaseAsync (settings.InfluxDB.DatabaseName);
return client;
}
}

[TestMethod]
public async Task TestGenericHeaderless ()
{
var settings = InfluxerConfigSection.Load (Path.Combine (TestFilesPath, "HeaderlessText.conf"));
settings.InputFileName = Path.Combine (TestFilesPath, "HeaderlessText.txt");
var client = await GetClientAsync (settings);
var result = await new GenericFile ().ProcessGenericFile (settings.InputFileName, client);
Assert.IsTrue (result.ExitCode == ExitCode.Success || result.ExitCode == ExitCode.ProcessedWithErrors, "Processing a generic text file failed");
var file = new GenericFile ();
var result = await file.ProcessGenericFile (settings.InputFileName, client);
//Debug.WriteLine (result.ToString ());
Assert.IsTrue (result.ExitCode == ExitCode.ProcessedWithErrors && result.PointsFound == 1001 && result.PointsFailed == 29, "Processing a generic text file failed");
}

[TestMethod]
Expand All @@ -57,7 +73,8 @@ public async Task TestPerfmonFile ()
settings.InputFileName = Path.Combine (TestFilesPath, "Perfmon.csv");
var client = await GetClientAsync (settings);
var result = await new PerfmonFile ().ProcessPerfMonLog (settings.InputFileName, client);
Assert.IsTrue (result.ExitCode == ExitCode.Success || result.ExitCode == ExitCode.ProcessedWithErrors, "Processing Perfmon file failed");
//Debug.WriteLine (result.ToString ());
Assert.IsTrue (result.ExitCode == ExitCode.ProcessedWithErrors && result.PointsFound == 144818 && result.PointsFailed == 0, "Processing Perfmon file failed");
}

[TestMethod]
Expand All @@ -71,7 +88,8 @@ public async Task TestPerfmonFileMultiMeasurement ()
settings.InputFileName = Path.Combine (TestFilesPath, "Perfmon.csv");
var client = await GetClientAsync (settings);
var result = await new PerfmonFile ().ProcessPerfMonLog (settings.InputFileName, client);
Assert.IsTrue (result.ExitCode == ExitCode.Success || result.ExitCode == ExitCode.ProcessedWithErrors, "Processing Perfmon file failed");
//Debug.WriteLine (result.ToString ());
Assert.IsTrue (result.ExitCode == ExitCode.ProcessedWithErrors && result.PointsFound == 5347 && result.PointsFailed == 0, "Processing Perfmon file failed");
}

[TestMethod]
Expand All @@ -83,8 +101,10 @@ public async Task TestGenericSimple ()
settings.InfluxDB.RetentionPolicy = "autogen";
settings.GenericFile.TimeFormat = "yyyy-MM-dd m:ss";
var client = await GetClientAsync (settings);
var result = await new GenericFile ().ProcessGenericFile (settings.InputFileName, client);
Assert.IsTrue (result.ExitCode == ExitCode.Success || result.ExitCode == ExitCode.ProcessedWithErrors, "Processing a generic CSV file failed");
var file = new GenericFile ();
var result = await file.ProcessGenericFile (settings.InputFileName, client);
//Debug.WriteLine (result.ToString ());
Assert.IsTrue (result.ExitCode == ExitCode.ProcessedWithErrors && result.PointsFound == 4897 && result.PointsFailed == 1, "Processing a generic CSV file failed");
}

[TestMethod]
Expand All @@ -97,21 +117,25 @@ public async Task TestGenericSimple2 ()
settings.InfluxDB.RetentionPolicy = "autogen";
var client = await GetClientAsync (settings);
var result = await new GenericFile ().ProcessGenericFile (settings.InputFileName, client);
Assert.IsTrue (result.ExitCode == ExitCode.Success || result.ExitCode == ExitCode.ProcessedWithErrors, "Processing a generic CSV file failed");
//Debug.WriteLine (result.ToString ());
Assert.IsTrue (result.ExitCode == ExitCode.Success && result.PointsFound == 226 && result.PointsFailed == 0, "Processing a generic CSV file failed");
}


private static async Task<InfluxDBClient> GetClientAsync (InfluxerConfigSection settings)
[TestMethod]
public async Task TestGenericMicroSecPrecision ()
{
var client = new InfluxDBClient (settings.InfluxDB.InfluxUri, settings.InfluxDB.UserName, settings.InfluxDB.Password);
var dbNames = await client.GetInfluxDBNamesAsync ();
if (dbNames.Contains (settings.InfluxDB.DatabaseName))
return client;
else
{
await client.CreateDatabaseAsync (settings.InfluxDB.DatabaseName);
return client;
}
var args = new string[] { "-input", Path.Combine (TestFilesPath, "MicroSecSemicolonSeperated.txt"),
"-format", "Generic",
"-TimeFormat", "yyyy-MM-dd-hh.mm.ss.ffffff",
"-Precision", "Microseconds",
"-splitter", ";" };
InfluxerConfigSection settings;
CommandLineProcessor.ProcessArguments (args);
settings = CommandLineProcessor.Settings;
var client = await GetClientAsync (settings);
var result = await new GenericFile ().ProcessGenericFile (settings.InputFileName, client);
//Debug.WriteLine (result.ToString ());
Assert.IsTrue (result.ExitCode == ExitCode.Success && result.PointsFound == 4 && result.PointsFailed == 0, "Processing a generic MicroSecSemicolonSeperated file failed");
}
}
}
5 changes: 5 additions & 0 deletions Influxer.Test/TestFiles/MicroSecSemicolonSeperated.txt

Large diffs are not rendered by default.

35 changes: 35 additions & 0 deletions Influxer.Test/TestSetup.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
##################################################################################################
#This script downloads the latest nightly InfluxDB build, starts the db engine, and creates basic
#data structures so that 'show databases' query will not fail during tests if the query gets
#executed before CreateDatabase test
#
##################################################################################################

$source = "https://dl.influxdata.com/influxdb/nightlies/influxdb-nightly_windows_amd64.zip"
$destination = "$env:Temp\influxdb-nightly_windows_amd64.zip"
$influx = "$env:Temp\influxdb"
$influxdata = "$env:UserProfile\.influxdb"

if(!(test-path $destination) -or ((Get-ItemProperty -Path $destination -Name LastWriteTime).lastwritetime -lt $(get-date).AddDays(-1)))
{ Invoke-WebRequest $source -OutFile $destination }

if(test-path $influx)
{ rmdir -recurse $influx}

Add-Type -As System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($destination,$influx)
$influxd = Get-ChildItem $influx -File -filter "influxd.exe" -Recurse | % { $_.FullName }

#$x = 7z e $destination -o"$env:Temp\influxdb" -y
if(test-path $influxdata)
{ rmdir -recurse "$env:UserProfile\.influxdb" }

Start-Process -FilePath $influxd
#let the engine start
Start-Sleep -s 10

$r = Invoke-WebRequest -Method Post -Uri http://localhost:8086/query -Body "q=CREATE DATABASE prereq"
if($r.StatusCode -ne 200)
{
throw "Unable to create DB"
}
3 changes: 2 additions & 1 deletion Influxer.Test/packages.config
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="AdysTech.InfluxDB.Client.Net" version="0.5.3" targetFramework="net452" />
<package id="AdysTech.InfluxDB.Client.Net" version="0.5.9.1" targetFramework="net452" />
<package id="Newtonsoft.Json" version="9.0.1" targetFramework="net452" />
</packages>
131 changes: 131 additions & 0 deletions Influxer/Config/CommandLineProcessor.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
using AdysTech.Influxer.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace AdysTech.Influxer.Config
{
public static class CommandLineProcessor
{
private static InfluxerConfigSection _settings;
public static InfluxerConfigSection Settings
{
get
{
return _settings;
}
}

public static bool ProcessArguments (string[] args)
{
if (args.Length == 0)
{
throw new ArgumentException ("Command line arguments not valid, try --help to see valid ones!");
}
#region Parse command line arguments
Dictionary<string, string> cmdArgs = new Dictionary<string, string> ();
Regex commandSwitch = new Regex ("^-[a-zA-Z+]|^/[a-zA-Z+]", RegexOptions.Compiled);
for (int i = 0; i < args.Length; i++)
{
if (commandSwitch.IsMatch (args[i]))
{
var key = args[i].ToLower ();
if (i + 1 < args.Length && !commandSwitch.IsMatch (args[i + 1]))
{
cmdArgs.Add (key.ToLower (), args[i + 1]);
i++;
}
else
cmdArgs.Add (key.ToLower (), "true");
}
}

var totalArguments = cmdArgs.Count;

if (cmdArgs.ContainsKey ("--help") || cmdArgs.ContainsKey ("/help") || cmdArgs.ContainsKey ("/?"))
{
var help = new StringBuilder ();
help.AppendLine ("Influxer is an application to parse log files, push data to Influx for later visualization.");
help.AppendLine ("It currently supports Windows Perfmon and any generic delimited file formats");
help.AppendLine ("It uses InfluxDB.Client.Net to interact with Influx.");
help.AppendLine (new String ('-', 180));
help.AppendLine ("Supported command line arguments");
help.AppendLine ("--help /? or /help shows this help text\n");
help.AppendLine ();
help.AppendLine ("/export to print possible config section, pipe it to a file to edit and reuse the config");
help.AppendLine ();
help.AppendLine ("-config <configuration file path> to load the config file.");
help.AppendLine ();
help.AppendLine ("Any configuration entries can be overridden by command line switches shown below\n");
help.AppendLine (new String ('-', 180));
help.Append (InfluxerConfigSection.LoadDefault ().PrintHelpText ());
Logger.Log (LogLevel.Info, help.ToString ());
return false;
}

if (cmdArgs.ContainsKey ("-config"))
{
try
{
var configFile = Path.GetFullPath (cmdArgs["-config"]);
_settings = InfluxerConfigSection.Load (configFile);
cmdArgs.Remove ("-config");
totalArguments -= 1;
}
catch (Exception e)
{
throw new FileLoadException ($"Error Loading config file:{e.GetType ().Name},{e.Message}", e);
}
}
else
{
_settings = InfluxerConfigSection.LoadDefault ();
}
#endregion

if (totalArguments >= 1)
{
if (!(cmdArgs.Count == 1 && cmdArgs.ContainsKey ("/export")))
{
try
{
if (!_settings.ProcessCommandLineArguments (cmdArgs))
{
throw new ArgumentException ("Invalid commandline arguments!! Use /help to see valid ones");
}
}
catch (Exception e)
{
throw new ArgumentException ($"Error processing arguments :{e.GetType ().Name}, {e.Message}", e);
}
}
}

if (cmdArgs.ContainsKey ("/export"))
{
if (cmdArgs.ContainsKey ("/autolayout"))
{
if (string.IsNullOrWhiteSpace (_settings.InputFileName))
throw new ArgumentException ("No Input file name mentioned!!");

var g = new GenericFile ();
g.GetFileLayout (_settings.InputFileName);
g.ValidateData (_settings.InputFileName);
}
InfluxerConfigSection.Export (Console.OpenStandardOutput (), totalArguments > 1 ? false : true);
return false;
}


if (cmdArgs.Count > 0)
{
throw new ArgumentException ($"Unknown command line arguments: {String.Join (", ", cmdArgs.Select (c => c.Key))}");
}
return true;
}
}
}
6 changes: 3 additions & 3 deletions Influxer/GenericFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, Influ
if (r.ExitCode != ExitCode.Success)
return r;

InfluxDatabase dbStructure;
IInfluxDatabase dbStructure;
if (settings.GenericFile.Filter != Filters.None)
{
var filterColumns = new List<GenericColumn> ();
Expand All @@ -64,7 +64,7 @@ public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, Influ
}

dbStructure = await client.GetInfluxDBStructureAsync (settings.InfluxDB.DatabaseName);
ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure);
ColumnHeaders = FilterGenericColumns (ColumnHeaders, filterColumns, dbStructure as InfluxDatabase);

}

Expand All @@ -73,7 +73,7 @@ public async Task<ProcessStatus> ProcessGenericFile (string InputFileName, Influ
var failureReasons = new Dictionary<Type, FailureTracker> ();

List<IInfluxDatapoint> points = new List<IInfluxDatapoint> (), retryQueue = new List<IInfluxDatapoint> ();
InfluxRetentionPolicy policy = null;
IInfluxRetentionPolicy policy = null;

if (settings.InfluxDB.RetentionDuration != 0 || !String.IsNullOrWhiteSpace (settings.InfluxDB.RetentionPolicy))
{
Expand Down
Loading

0 comments on commit 8f17470

Please sign in to comment.