public ProcessStartInfo StartInfo { get; set; }
public Stopwatch Stopwatch { get; set; } = new Stopwatch ();
public int? ExitCode { get; set; }
- public StringBuilder StandardOutput { get; set; } = new StringBuilder ();
- public StringBuilder StandardError { get; set; } = new StringBuilder ();
+ public string StandardOutput { get; set; }
+ public string StandardError { get; set; }
}
static class Program {
static readonly string[] _options = new [] {
- "domain",
- "assembly",
- "module",
- "class",
- "jit",
"exception",
- "gcalloc",
- "gc",
- "thread",
- // "calls", // Way too heavy.
"monitor",
+ "gc",
+ "gcalloc",
"gcmove",
"gcroot",
- "context",
+ "gchandle",
"finalization",
"counter",
- "gchandle",
+ "jit",
};
- static readonly TimeSpan _timeout = TimeSpan.FromHours (6);
+ static readonly TimeSpan _timeout = TimeSpan.FromHours (8);
+
+ static readonly Dictionary<string, Predicate<Benchmark>> _filters = new Dictionary<string, Predicate<Benchmark>> {
+ { "ironjs-v8", FilterArmArchitecture },
+ };
+
+ static readonly Dictionary<string, Action<TestResult>> _processors = new Dictionary<string, Action<TestResult>> {
+ { "msbiology", Process32BitOutOfMemory },
+ };
static string FilterInvalidXmlChars (string text) {
return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", string.Empty);
}
+ static bool FilterArmArchitecture (Benchmark benchmark)
+ {
+#if ARCH_arm || ARCH_arm64
+ return false;
+#else
+ return true;
+#endif
+ }
+
+ static void Process32BitOutOfMemory (TestResult result)
+ {
+ if (Environment.Is64BitProcess)
+ return;
+
+ if (result.ExitCode == null || result.ExitCode == 0)
+ return;
+
+ if (result.StandardError.Contains ("OutOfMemoryException"))
+ result.ExitCode = 0;
+ }
+
+ static bool IsSupported (Benchmark benchmark)
+ {
+ return _filters.TryGetValue (benchmark.Name, out var filter) ? filter (benchmark) : true;
+ }
+
static int Main ()
{
var depDir = Path.Combine ("..", "external", "benchmarker");
var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
.Select (Benchmark.Load)
- .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null)
+ .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b))
.OrderBy (b => b.Name)
.ToArray ();
var bench = benchmarks [i];
var sampleFreq = rand.Next (-1000, 1001);
- var sampleMode = rand.Next (0, 2) == 1 ? "real" : "process";
+ var sampleMode = rand.Next (0, 2) == 1 ? "-real" : string.Empty;
var maxSamples = rand.Next (0, cpus * 2000 + 1);
var heapShotFreq = rand.Next (-10, 11);
var maxFrames = rand.Next (0, 33);
var profOptions = $"maxframes={maxFrames},{string.Join (",", options)},output=/dev/null";
if (sampleFreq > 0)
- profOptions += $",sample={sampleFreq},sampling-{sampleMode},maxsamples={maxSamples}";
+ profOptions += $",sample{sampleMode}={sampleFreq},maxsamples={maxSamples}";
if (heapShotFreq > 0)
profOptions += $",heapshot={heapShotFreq}gc";
using (var proc = new Process ()) {
proc.StartInfo = info;
+ var stdout = new StringBuilder ();
+ var stderr = new StringBuilder ();
+
proc.OutputDataReceived += (sender, args) => {
if (args.Data != null)
- result.StandardOutput.AppendLine (args.Data);
+ lock (result)
+ stdout.AppendLine (args.Data);
};
proc.ErrorDataReceived += (sender, args) => {
if (args.Data != null)
- result.StandardError.AppendLine (args.Data);
+ lock (result)
+ stderr.AppendLine (args.Data);
};
result.Stopwatch.Start ();
result.ExitCode = proc.ExitCode;
result.Stopwatch.Stop ();
+
+ lock (result) {
+ result.StandardOutput = stdout.ToString ();
+ result.StandardError = stderr.ToString ();
+ }
}
var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";
Console.WriteLine ("===== stdout =====");
Console.ResetColor ();
- Console.WriteLine (result.StandardOutput.ToString ());
+ Console.WriteLine (result.StandardOutput);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine ("===== stderr =====");
Console.ResetColor ();
- Console.WriteLine (result.StandardError.ToString ());
+ Console.WriteLine (result.StandardError);
}
+ if (_processors.TryGetValue (bench.Name, out var processor))
+ processor (result);
+
results.Add (result);
}
writer.WriteStartElement ("failure");
writer.WriteStartElement ("message");
- writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput.ToString ()));
+ writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput));
writer.WriteEndElement ();
writer.WriteStartElement ("stack-trace");
- writer.WriteCData (FilterInvalidXmlChars (result.StandardError.ToString ()));
+ writer.WriteCData (FilterInvalidXmlChars (result.StandardError));
writer.WriteEndElement ();
writer.WriteEndElement ();