Merge pull request #5714 from alexischr/update_bockbuild

[mono.git] / acceptance-tests / profiler-stress / runner.cs

diff --git a/acceptance-tests/profiler-stress/runner.cs b/acceptance-tests/profiler-stress/runner.cs
index 5ef4ab02a2eb198eeb669e3a1fed781af3376828..e4a609b995f259e5cb45c144fb1a3b0076fc8fe5 100644 (file)
--- a/acceptance-tests/profiler-stress/runner.cs
+++ b/acceptance-tests/profiler-stress/runner.cs
@@ -38,18 +38,65 @@ namespace Mono.Profiling.Tests.Stress {
                public ProcessStartInfo StartInfo { get; set; }
                public Stopwatch Stopwatch { get; set; } = new Stopwatch ();
                public int? ExitCode { get; set; }
-               public StringBuilder StandardOutput { get; set; } = new StringBuilder ();
-               public StringBuilder StandardError { get; set; } = new StringBuilder ();
+               public string StandardOutput { get; set; }
+               public string StandardError { get; set; }
        }
 
        static class Program {
 
-               static readonly TimeSpan _timeout = TimeSpan.FromHours (6);
+               static readonly string[] _options = new [] {
+                       "exception",
+                       "monitor",
+                       "gc",
+                       "gcalloc",
+                       "gcmove",
+                       "gcroot",
+                       "gchandle",
+                       "finalization",
+                       "counter",
+                       "jit",
+               };
+
+               static readonly TimeSpan _timeout = TimeSpan.FromHours (9);
+
+               static readonly Dictionary<string, Predicate<Benchmark>> _filters = new Dictionary<string, Predicate<Benchmark>> {
+                       { "ironjs-v8", FilterArmArchitecture },
+               };
+
+               static readonly Dictionary<string, Action<TestResult>> _processors = new Dictionary<string, Action<TestResult>> {
+                       { "msbiology", Process32BitOutOfMemory },
+               };
 
                static string FilterInvalidXmlChars (string text) {
                        return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", string.Empty);
                }
 
+               static bool FilterArmArchitecture (Benchmark benchmark)
+               {
+#if ARCH_arm || ARCH_arm64
+                       return false;
+#else
+                       return true;
+#endif
+               }
+
+               static void Process32BitOutOfMemory (TestResult result)
+               {
+                       if (Environment.Is64BitProcess)
+                               return;
+
+                       if (result.ExitCode == null || result.ExitCode == 0)
+                               return;
+
+                       if (result.StandardError.Contains ("OutOfMemoryException"))
+                               result.ExitCode = 0;
+               }
+
+               static bool IsSupported (Benchmark benchmark)
+               {
+                       return _filters.TryGetValue (benchmark.Name, out var filter) ? filter (benchmark) : true;
+               }
+
                static int Main ()
                {
                        var depDir = Path.Combine ("..", "external", "benchmarker");
@@ -58,7 +105,7 @@ namespace Mono.Profiling.Tests.Stress {
 
                        var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
                                         .Select (Benchmark.Load)
-                                        .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null)
+                                        .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b))
                                         .OrderBy (b => b.Name)
                                         .ToArray ();
 
@@ -75,14 +122,22 @@ namespace Mono.Profiling.Tests.Stress {
                        for (var i = 0; i < benchmarks.Length; i++) {
                                var bench = benchmarks [i];
 
-                               var sampleFreq = rand.Next (0, 1001);
-                               var sampleMode = rand.Next (0, 2) == 1 ? "real" : "process";
+                               var sampleFreq = rand.Next (-1000, 1001);
+                               var sampleMode = rand.Next (0, 2) == 1 ? "-real" : string.Empty;
                                var maxSamples = rand.Next (0, cpus * 2000 + 1);
-                               var heapShotFreq = rand.Next (0, 11);
+                               var heapShotFreq = rand.Next (-10, 11);
                                var maxFrames = rand.Next (0, 33);
-                               var allocMode = rand.Next (0, 2) == 1 ? "alloc" : "noalloc";
+                               var options = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
+                                                     .Select (x => (x.Value ? string.Empty : "no") + x.Key)
+                                                     .ToArray ();
+
+                               var profOptions = $"maxframes={maxFrames},{string.Join (",", options)},output=/dev/null";
+
+                               if (sampleFreq > 0)
+                                       profOptions += $",sample{sampleMode}={sampleFreq},maxsamples={maxSamples}";
 
-                               var profOptions = $"sample=cycles/{sampleFreq},sampling-{sampleMode},maxsamples={maxSamples},heapshot={heapShotFreq}gc,maxframes={maxFrames},{allocMode},output=/dev/null";
+                               if (heapShotFreq > 0)
+                                       profOptions += $",heapshot={heapShotFreq}gc";
 
                                var info = new ProcessStartInfo {
                                        UseShellExecute = false,
@@ -110,14 +165,19 @@ namespace Mono.Profiling.Tests.Stress {
                                using (var proc = new Process ()) {
                                        proc.StartInfo = info;
 
+                                       var stdout = new StringBuilder ();
+                                       var stderr = new StringBuilder ();
+
                                        proc.OutputDataReceived += (sender, args) => {
                                                if (args.Data != null)
-                                                       result.StandardOutput.AppendLine (args.Data);
+                                                       lock (result)
+                                                               stdout.AppendLine (args.Data);
                                        };
 
                                        proc.ErrorDataReceived += (sender, args) => {
                                                if (args.Data != null)
-                                                       result.StandardError.AppendLine (args.Data);
+                                                       lock (result)
+                                                               stderr.AppendLine (args.Data);
                                        };
 
                                        result.Stopwatch.Start ();
@@ -140,6 +200,11 @@ namespace Mono.Profiling.Tests.Stress {
                                                result.ExitCode = proc.ExitCode;
 
                                        result.Stopwatch.Stop ();
+
+                                       lock (result) {
+                                               result.StandardOutput = stdout.ToString ();
+                                               result.StandardError = stderr.ToString ();
+                                       }
                                }
 
                                var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";
@@ -153,15 +218,18 @@ namespace Mono.Profiling.Tests.Stress {
                                        Console.WriteLine ("===== stdout =====");
                                        Console.ResetColor ();
 
-                                       Console.WriteLine (result.StandardOutput.ToString ());
+                                       Console.WriteLine (result.StandardOutput);
 
                                        Console.ForegroundColor = ConsoleColor.Red;
                                        Console.WriteLine ("===== stderr =====");
                                        Console.ResetColor ();
 
-                                       Console.WriteLine (result.StandardError.ToString ());
+                                       Console.WriteLine (result.StandardError);
                                }
 
+                               if (_processors.TryGetValue (bench.Name, out var processor))
+                                       processor (result);
+
                                results.Add (result);
                        }
 
@@ -239,11 +307,11 @@ namespace Mono.Profiling.Tests.Stress {
                                                writer.WriteStartElement ("failure");
 
                                                writer.WriteStartElement ("message");
-                                               writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput.ToString ()));
+                                               writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput));
                                                writer.WriteEndElement ();
 
                                                writer.WriteStartElement ("stack-trace");
-                                               writer.WriteCData (FilterInvalidXmlChars (result.StandardError.ToString ()));
+                                               writer.WriteCData (FilterInvalidXmlChars (result.StandardError));
                                                writer.WriteEndElement ();
 
                                                writer.WriteEndElement ();