diff --git a/csharp/Samples/Microsoft.Spark.CSharp/CommandlineArgumentProcessor.cs b/csharp/Samples/Microsoft.Spark.CSharp/CommandlineArgumentProcessor.cs index b6991212..4e007c20 100644 --- a/csharp/Samples/Microsoft.Spark.CSharp/CommandlineArgumentProcessor.cs +++ b/csharp/Samples/Microsoft.Spark.CSharp/CommandlineArgumentProcessor.cs @@ -2,14 +2,13 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Microsoft.Spark.CSharp.Services; namespace Microsoft.Spark.CSharp.Samples { + /// + /// Simple commandline argument parser + /// internal class CommandlineArgumentProcessor { private static readonly ILoggerService logger = LoggerServiceFactory.GetLogger(typeof(CommandlineArgumentProcessor)); @@ -22,7 +21,7 @@ internal static Configuration ProcessArugments(string[] args) } var configuration = new Configuration(); - logger.LogInfo(string.Format("Arguments to SparkCLRSamples are {0}", string.Join(",", args))); + logger.LogInfo(string.Format("Arguments to SparkCLRSamples.exe are {0}", string.Join(",", args))); for (int i = 0; i < args.Length; i++) { if (args[i].Equals("--help", StringComparison.InvariantCultureIgnoreCase) @@ -69,24 +68,24 @@ internal static Configuration ProcessArugments(string[] args) private static void PrintUsage() { - var p = AppDomain.CurrentDomain.FriendlyName; + const string exeName = "SparkCLRSamples.exe"; Console.WriteLine(" "); - Console.WriteLine(" {0} supports following options:", p); + Console.WriteLine(" {0} supports following options:", exeName); Console.WriteLine(" "); Console.WriteLine(" [--temp | spark.local.dir] TEMP_DIR is the directory used as \"scratch\" space in Spark, including map output files and RDDs that get stored on disk. "); Console.WriteLine(" See http://spark.apache.org/docs/latest/configuration.html for details."); Console.WriteLine(" "); - Console.WriteLine(" [--data | sparkclr.sampledata.loc] SAMPLE_DATA_DIR is the directory where Sample data resides. "); + Console.WriteLine(" [--data | sparkclr.sampledata.loc] SAMPLE_DATA_DIR is the directory where data files used by samples reside. "); Console.WriteLine(" "); Console.WriteLine(" [--torun | sparkclr.samples.torun] SAMPLE_LIST specifies a list of samples to run. "); Console.WriteLine(" Case-insensitive command line wild card matching by default. Or, use \"/\" (forward slash) to enclose regular expression. "); - Console.WriteLine(" "); + Console.WriteLine(" "); Console.WriteLine(" [--cat | sparkclr.samples.category] SAMPLE_CATEGORY can be \"all\", \"default\", \"experimental\" or any new categories. "); Console.WriteLine(" Case-insensitive command line wild card matching by default. Or, use \"/\" (forward slash) to enclose regular expression. "); Console.WriteLine(" "); - Console.WriteLine(" [--validate | sparkclr.enablevalidation] Enable validation. "); + Console.WriteLine(" [--validate | sparkclr.enablevalidation] Enables validation of results produced in each sample. "); Console.WriteLine(" "); - Console.WriteLine(" [--dryrun | sparkclr.dryrun] Dry-run mode. "); + Console.WriteLine(" [--dryrun | sparkclr.dryrun] Dry-run mode. Just lists the samples that will be executed with given parameters without running them"); Console.WriteLine(" "); Console.WriteLine(" [--help | -h | -?] Display usage. "); Console.WriteLine(" "); @@ -95,31 +94,31 @@ private static void PrintUsage() Console.WriteLine(" "); Console.WriteLine(" Example 1 - run default samples:"); Console.WriteLine(" "); - Console.WriteLine(@" {0} --temp C:\gitsrc\SparkCLR\run\Temp --data C:\gitsrc\SparkCLR\run\data ", p); + Console.WriteLine(@" {0} --temp C:\gitsrc\SparkCLR\run\Temp --data C:\gitsrc\SparkCLR\run\data ", exeName); Console.WriteLine(" "); Console.WriteLine(" Example 2 - dryrun default samples:"); Console.WriteLine(" "); - Console.WriteLine(@" {0} --dryrun ", p); + Console.WriteLine(@" {0} --dryrun ", exeName); Console.WriteLine(" "); Console.WriteLine(" Example 3 - dryrun all samples:"); Console.WriteLine(" "); - Console.WriteLine(@" {0} --dryrun --cat all ", p); + Console.WriteLine(@" {0} --dryrun --cat all ", exeName); Console.WriteLine(" "); Console.WriteLine(" Example 4 - dryrun PiSample (commandline wildcard matching, case-insensitive):"); Console.WriteLine(" "); - Console.WriteLine(@" {0} --dryrun --torun pi*", p); + Console.WriteLine(@" {0} --dryrun --torun pi*", exeName); Console.WriteLine(" "); Console.WriteLine(" Example 5 - dryrun all DF* samples (commandline wildcard matching, case-insensitive):"); Console.WriteLine(" "); - Console.WriteLine(@" {0} --dryrun --cat a* --torun DF*", p); + Console.WriteLine(@" {0} --dryrun --cat a* --torun DF*", exeName); Console.WriteLine(" "); Console.WriteLine(" Example 6 - dryrun all RD* samples (regular expression):"); Console.WriteLine(" "); - Console.WriteLine(@" {0} --dryrun --cat a* --torun /\bRD.*Sample.*\b/", p); + Console.WriteLine(@" {0} --dryrun --cat a* --torun /\bRD.*Sample.*\b/", exeName); Console.WriteLine(" "); Console.WriteLine(" Example 7 - dryrun specific samples (case insensitive): "); Console.WriteLine(" "); - Console.WriteLine(" {0} --dryrun --torun \"DFShowSchemaSample,DFHeadSample\"", p); + Console.WriteLine(" {0} --dryrun --torun \"DFShowSchemaSample,DFHeadSample\"", exeName); Console.WriteLine(" "); } } diff --git a/csharp/Samples/Microsoft.Spark.CSharp/MiscSamples.cs b/csharp/Samples/Microsoft.Spark.CSharp/MiscSamples.cs index d5573a15..65aa7c01 100644 --- a/csharp/Samples/Microsoft.Spark.CSharp/MiscSamples.cs +++ b/csharp/Samples/Microsoft.Spark.CSharp/MiscSamples.cs @@ -51,7 +51,7 @@ internal static void PiSample() if (SparkCLRSamples.Configuration.IsValidationEnabled) { - Assert.IsTrue(Math.Abs(approximatePiValue - 3.14) <= 0.01); + Assert.IsTrue(Math.Abs(approximatePiValue - 3.14) <= 0.019); } } diff --git a/csharp/Samples/Microsoft.Spark.CSharp/Program.cs b/csharp/Samples/Microsoft.Spark.CSharp/Program.cs index c6ffd97e..9cdeaee5 100644 --- a/csharp/Samples/Microsoft.Spark.CSharp/Program.cs +++ b/csharp/Samples/Microsoft.Spark.CSharp/Program.cs @@ -2,13 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Reflection; -using System.Text; -using System.Text.RegularExpressions; using Microsoft.Spark.CSharp.Core; using Microsoft.Spark.CSharp.Services; @@ -38,11 +34,14 @@ static void Main(string[] args) { SparkContext = CreateSparkContext(); SparkContext.SetCheckpointDir(Path.GetTempPath()); + SamplesRunner.RunSamples(); PrintLogLocation(); - ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ..."); - ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!"); + ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ..."); + //following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333 + ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!"); + //TODO - add instructions to terminate java process SparkContext.Stop(); } } @@ -60,15 +59,17 @@ private static SparkContext CreateSparkContext() private static void PrintLogLocation() { - ConsoleWriteLine("Main", - string.Format(@"Logs by SparkCLR and Apache Spark are available at {0}\SparkCLRLogs", + ConsoleWriteLine(string.Format(@"Logs by SparkCLR and Apache Spark are available at {0}\SparkCLRLogs", Environment.GetEnvironmentVariable("TEMP"))); } - private static void ConsoleWriteLine(string functionName, string message) + private static void ConsoleWriteLine(string message) { - var p = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine("[{0}.{1}] {2}", p, functionName, message); + const string exeName = "SparkCLRSamples.exe"; + var callingMethod = new StackTrace().GetFrames()[1].GetMethod(); + var callingMethodName = callingMethod.Name; + var typeName = callingMethod.ReflectedType; + Console.WriteLine("[{0}.{1}.{2}] {3}", exeName, typeName, callingMethodName, message); } } } diff --git a/csharp/Samples/Microsoft.Spark.CSharp/SampleAttribute.cs b/csharp/Samples/Microsoft.Spark.CSharp/SampleAttribute.cs index 1368a800..9de2cce4 100644 --- a/csharp/Samples/Microsoft.Spark.CSharp/SampleAttribute.cs +++ b/csharp/Samples/Microsoft.Spark.CSharp/SampleAttribute.cs @@ -1,9 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; using System.Text.RegularExpressions; -using System.Threading.Tasks; namespace Microsoft.Spark.CSharp.Samples { diff --git a/csharp/Samples/Microsoft.Spark.CSharp/SamplesRunner.cs b/csharp/Samples/Microsoft.Spark.CSharp/SamplesRunner.cs index e1ca0a15..af80d578 100644 --- a/csharp/Samples/Microsoft.Spark.CSharp/SamplesRunner.cs +++ b/csharp/Samples/Microsoft.Spark.CSharp/SamplesRunner.cs @@ -1,24 +1,26 @@ -using System; +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Reflection; using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; namespace Microsoft.Spark.CSharp.Samples { - //finds all methods that are marked with [Sample] attribute and - //runs all of them if sparkclr.samples.torun commandline arg is not used - //or just runs the ones that are provided as comma separated list + /// + /// Runs samples + /// internal class SamplesRunner { private static Regex samplesToRunRegex; private static Regex samplesCategoryRegex; private static Stopwatch stopWatch; // track for reporting - private static readonly List> samplesRunResultList = new List>(); + private static readonly List> samplesRunInfoList = new List>(); internal static void RunSamples() { @@ -33,10 +35,10 @@ internal static void RunSamples() foreach (var sample in samples) { - var sampleRunResult = RunSample(sample); - if (sampleRunResult != null) + var sampleRunInfo = RunSample(sample); + if (sampleRunInfo != null) { - samplesRunResultList.Add(sampleRunResult); + samplesRunInfoList.Add(sampleRunInfo); } } stopWatch.Stop(); @@ -135,43 +137,38 @@ private static Regex GetRegex() private static void ReportOutcome() { + var succeededSamples = samplesRunInfoList.Where(x => x.Item3).ToList(); + var failedSamples = samplesRunInfoList.Where(x => !x.Item3).ToList(); + var msg = new StringBuilder(); msg.Append("----- ") .Append("Finished running ") - .Append(string.Format("{0} samples(s)", samplesRunResultList.Count)) + .Append(string.Format("{0} samples(s) [succeeded={1}, failed={2}]", samplesRunInfoList.Count, succeededSamples.Count, failedSamples.Count)) .Append(" in ").Append(stopWatch.Elapsed) .AppendLine(" -----"); - var completed = samplesRunResultList.Where(x => x.Item3).ToList(); - var errors = samplesRunResultList.Where(x => !x.Item3).ToList(); - - msg.Append("----- ") - .Append(" Completion counts:") - .Append(" Success=").Append(completed.Count) - .Append(" Failed=").Append(errors.Count) - .AppendLine(" -----"); - - msg.AppendLine("Successful samples:"); - foreach (var s in completed) + if (succeededSamples.Count > 0) { - msg.Append(" ").AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item3)); + msg.AppendLine("Successfully completed samples:"); + foreach (var s in succeededSamples) + { + msg.Append(" ") + .AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item4)); + } } - msg.AppendLine("Failed samples:"); - foreach (var s in errors) + if (failedSamples.Count > 0) { - msg.Append(" ").AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item3)); + msg.AppendLine("Failed samples:"); + foreach (var s in failedSamples) + { + msg.Append(" ") + .AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item4)); + } } - if (errors.Count == 0) - { - Console.WriteLine(msg.ToString()); - } - else - { - Console.WriteLine("[Warning]{0}", msg); - } + Console.WriteLine(msg.ToString()); } } }