Performance failure mode (#10061)

mjkkirschner · web-flow · commit a97eeee0e094 · 2019-10-16T20:50:13.000-04:00
* temp remove graphs to speed up testing * add comparison failure exit code add log text for result add comparisonData property to each Comparer and use it * fix saved file * remove extra pipe | from log * remove new line * address comments * Revert "temp remove graphs to speed up testing" This reverts commit 6134760.
diff --git a/tools/Performance/DynamoPerformanceTests/PerformanceTestConsoleApp.cs b/tools/Performance/DynamoPerformanceTests/PerformanceTestConsoleApp.cs
@@ -3,11 +3,18 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Running;
 using NDesk.Options;
+using System.Linq;
 
 namespace DynamoPerformanceTests
 {
     public class Program
     {
+        private enum ExitCode
+        {
+            ComparisonOK = 0,
+            ComparisonFailure = 1
+        }
+
         private enum Command
         {
             Benchmark,
@@ -18,6 +25,12 @@ private enum Command
             StandardConfigModelOnlyBenchmark
         }
 
+        private static readonly string modelTestBaseReport = "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv";
+        private static readonly string modelTestComparison = "DynamoPerformanceTests.Comparison-Model.csv";
+
+        private static readonly string viewTestBaseReport = "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv";
+        private static readonly string viewTestComparison = "DynamoPerformanceTests.Comparison-View.csv";
+
         public static void Main(string[] args)
         {
             var showHelp = false;
@@ -118,19 +131,54 @@ private static void Compare(string baseResultsPath, string newResultsPath, strin
                 return;
             }
 
-            // Create Model comparer
-            Console.WriteLine("\nComparison of Model tests: \n");
-            var baseModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv");
-            var newModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv");
-            var modelSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), "DynamoPerformanceTests.Comparison-Model.csv");
-            var modelComparer = new ResultsComparer(baseModelPath, newModelPath, modelSavePath);
-
-            // Create View comparer
-            Console.WriteLine("\nComparison of View tests: \n");
-            var baseViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv");
-            var newViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv");
-            var viewSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), "DynamoPerformanceTests.Comparison-View.csv");
-            var viewComparer = new ResultsComparer(baseViewPath, newViewPath, viewSavePath);
+            ResultsComparer modelComparer = null;
+            ResultsComparer viewComparer = null;
+
+            try
+            {
+
+                // Create Model comparer
+                Console.WriteLine("\nComparison of Model tests: \n");
+                var baseModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), modelTestBaseReport  );
+                var newModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath),modelTestBaseReport);
+                var modelSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath),modelTestComparison );
+
+                modelComparer = new ResultsComparer(baseModelPath, newModelPath, modelSavePath);
+
+                // Create View comparer
+                Console.WriteLine("\nComparison of View tests: \n");
+                var baseViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath),viewTestBaseReport );
+                var newViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), viewTestBaseReport);
+                var viewSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath),viewTestComparison );
+
+                viewComparer = new ResultsComparer(baseViewPath, newViewPath, viewSavePath);
+            }
+            //catch here - likely we could not create a view comparer if view comparison did not run.
+            catch(Exception e)
+            {
+                Console.WriteLine($"exception while comparing results {e} {Environment.NewLine} {e.Message}");
+
+            }
+
+            //return result of comparison
+            if (modelComparer != null)
+            {
+                if(modelComparer.ComparisonData.Any(x=>x.ResultState == ResultsComparer.ComparisonResultState.FAIL))
+                {
+                    Console.WriteLine("Comparison failed, some model performance benchmarks failed. Please see log above for details.");
+                    Environment.Exit((int)ExitCode.ComparisonFailure );
+                }
+            }
+
+            if (viewComparer != null)
+            {
+                if (viewComparer.ComparisonData.Any(x => x.ResultState == ResultsComparer.ComparisonResultState.FAIL))
+                {
+                    Console.WriteLine("Comparison failed, some view performance benchmarks failed. Please see log above for details.");
+                    Environment.Exit((int)ExitCode.ComparisonFailure);
+                }
+            }
+
         }
     }
 }
diff --git a/tools/Performance/DynamoPerformanceTests/ResultsComparer.cs b/tools/Performance/DynamoPerformanceTests/ResultsComparer.cs
@@ -17,7 +17,33 @@ namespace DynamoPerformanceTests
     /// </summary>
     public class ResultsComparer
     {
-        private class BenchmarkResult
+        /// <summary>
+        /// Should this comparison between old and new benchmark result in a failure. 
+        /// </summary>
+        internal enum ComparisonResultState
+        {
+            NA,
+            PASS,
+            FAIL,
+           
+        }
+
+        internal static ComparisonResultState DefaultBenchmarkComparison(BenchmarkComparison comparisonData)
+        {
+            //if the change in mean time
+            if (comparisonData.MeanDelta > 10)
+            {
+                return ComparisonResultState.FAIL;
+            }
+            else
+                return ComparisonResultState.PASS;
+        }
+
+
+        internal List<BenchmarkComparison> ComparisonData {  get; set; }
+
+
+        internal class BenchmarkResult
         {
             // Values
             internal string Method { get; set; }
@@ -60,7 +86,7 @@ internal BenchmarkResult(string method, string graph, double mean, string meanUn
             }
         }
 
-        private class BenchmarkComparison
+        internal class BenchmarkComparison
         {
             /// <summary>
             /// Results of baseline benchmark
@@ -105,12 +131,18 @@ internal bool BaseBenchmarkFound
                 get { return BaseResult != null; }
             }
 
+            /// <summary>
+            /// Marks the result of this comparison - should the comparison between base and new result in a test failure?
+            /// </summary>
+            internal ComparisonResultState ResultState { get; set; }
+
             /// <summary>
             /// Create a Comparison between two benchamrk results
             /// </summary>
             /// <param name="baseData"></param>
             /// <param name="newData"></param>
-            internal BenchmarkComparison(BenchmarkResult baseData, BenchmarkResult newData)
+            internal BenchmarkComparison(BenchmarkResult baseData, BenchmarkResult newData, 
+                Func<BenchmarkComparison, ComparisonResultState> comparisonFunction)
             {
                 BaseResult = baseData;
                 NewResult = newData;
@@ -128,6 +160,8 @@ internal BenchmarkComparison(BenchmarkResult baseData, BenchmarkResult newData)
                     Console.WriteLine("Units do not match beween base and new results.");
                     Console.ForegroundColor = fc;
                 }
+                this.ResultState = comparisonFunction(this);
+
             }
 
             /// <summary>
@@ -162,7 +196,7 @@ internal void LogComparison(List<int> columnWidths)
                 }
                 Console.WriteLine("|" + string.Join("|", newData) + "|");
 
-                // Lof Delta data
+                // Log Delta data
                 var meanDeltaString = MeanDelta >= 0 ? "+" + MeanDelta.ToString() : MeanDelta.ToString();
                 var deltaData = new string[] { "", "", "", meanDeltaString + "%", ErrorDelta.ToString() + "%", StdDevDelta.ToString() + "%" };
                 Console.Write("|");
@@ -182,6 +216,10 @@ internal void LogComparison(List<int> columnWidths)
                         Console.Write(deltaItem + "|");
                     }
                 }
+                // log final result of comparison
+                var resultString = Enum.GetName(typeof(ComparisonResultState), this.ResultState);
+                Console.Write(resultString);
+
                 Console.Write("\n");
             }
 
@@ -195,7 +233,9 @@ internal List<string[]> GetComparisonData()
 
                 var baseData = new string[] { BaseResult.Method, BaseResult.Graph, "Base", BaseResult.MeanString, BaseResult.ErrorString, BaseResult.StdDevString };
                 var newData = new string[] { "", "", "New", NewResult.MeanString, NewResult.ErrorString, NewResult.StdDevString };
-                var deltaData = new string[] { "", "", "", MeanDelta.ToString() + "%", ErrorDelta.ToString() + "%", StdDevDelta.ToString() + "%" };
+                var deltaData = new string[] { "", "", "", MeanDelta.ToString() + "%", ErrorDelta.ToString() + "%", StdDevDelta.ToString() + "%",
+                    Enum.GetName(typeof(ComparisonResultState), ResultState) };
+                var resultData = new string[] { };
                 return new List<string[]> { baseData, newData, deltaData };
             }
         }
@@ -209,7 +249,8 @@ public ResultsComparer(string basePath, string newPath, string saveCSVPath = nul
         {
             // Create BenchmarkComparison objects
             var comparisons = CreateComparisons(basePath, newPath);
-            
+            this.ComparisonData = comparisons;
+
             // Log comparisons to console
             LogResults(comparisons);
 
@@ -233,7 +274,17 @@ private List<BenchmarkComparison> CreateComparisons(string basePath, string newP
             foreach(var result in newResults.Values)
             {
                 var baseResult = GetBaseResult(result, baseResults);
-                comparisons.Add(new BenchmarkComparison(baseResult, result));
+                //NOTE(MJK) - to control determination of pass/fail conditions for each test, pass them here.Example:
+                /*
+                * comparisons.Add( (BenchmarkComparison comparison)=>{
+                if(MeanDelta > 15)
+                {
+                return FAIL;
+                }
+                return PASS;
+                 });
+                 */
+                comparisons.Add(new BenchmarkComparison(baseResult, result, ResultsComparer.DefaultBenchmarkComparison));
             }
             return comparisons;
         }
@@ -246,7 +297,7 @@ private void LogResults(List<BenchmarkComparison> comparisons)
             var columnWidths = GetColumnWidths(comparisons);
 
             // Log header
-            var header = new string[] { "Method", "Graph", "Version", "Mean", "Error", "StdDev" };
+            var header = new string[] { "Method", "Graph", "Version", "Mean", "Error", "StdDev", "Result" };
             var separator = new string[header.Length];
             for (int i = 0; i < header.Length; i++)
             {
@@ -330,7 +381,7 @@ private Dictionary<string, BenchmarkResult> ImportResultsCSV(string csvPath)
         private List<string[]> GetComparisonDataRows(List<BenchmarkComparison> comparisons)
         {
             var rows = new List<string[]>();
-            var header = new string[] { "Method", "Graph", "Version", "Mean", "Error", "StdDev" };
+            var header = new string[] { "Method", "Graph", "Version", "Mean", "Error", "StdDev","Result" };
             rows.Add(header);
 
             foreach (var comparison in comparisons)
