2 using System.Collections.Generic;
3 using System.Diagnostics;
4 using System.Globalization;
8 using System.Text.RegularExpressions;
9 using System.Threading;
11 using Mono.Unix.Native;
12 using Newtonsoft.Json;
14 // Shut up CLS compliance warnings from Json.NET.
15 [assembly: CLSCompliant (true)]
17 namespace Mono.Profiling.Tests.Stress {
19 // https://github.com/xamarin/benchmarker/blob/master/tools/libdbmodel/Benchmark.cs
20 sealed class Benchmark {
22 public string Name { get; set; }
23 public string TestDirectory { get; set; }
24 public bool OnlyExplicit { get; set; }
25 public string[] CommandLine { get; set; }
26 public string[] ClientCommandLine { get; set; }
27 public string[] AOTAssemblies { get; set; }
29 public static Benchmark Load (string file)
31 return JsonConvert.DeserializeObject<Benchmark> (File.ReadAllText (file));
35 sealed class TestResult {
37 public Benchmark Benchmark { get; set; }
38 public ProcessStartInfo StartInfo { get; set; }
39 public Stopwatch Stopwatch { get; set; } = new Stopwatch ();
40 public int? ExitCode { get; set; }
41 public string StandardOutput { get; set; }
42 public string StandardError { get; set; }
45 static class Program {
47 static readonly string[] _options = new [] {
60 static readonly TimeSpan _timeout = TimeSpan.FromHours (9);
62 static readonly Dictionary<string, Predicate<Benchmark>> _filters = new Dictionary<string, Predicate<Benchmark>> {
63 { "ironjs-v8", FilterArmArchitecture },
66 static readonly Dictionary<string, Action<TestResult>> _processors = new Dictionary<string, Action<TestResult>> {
67 { "msbiology", Process32BitOutOfMemory },
70 static string FilterInvalidXmlChars (string text) {
71 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", string.Empty);
74 static bool FilterArmArchitecture (Benchmark benchmark)
76 #if ARCH_arm || ARCH_arm64
83 static void Process32BitOutOfMemory (TestResult result)
85 if (Environment.Is64BitProcess)
88 if (result.ExitCode == null || result.ExitCode == 0)
91 if (result.StandardError.Contains ("OutOfMemoryException"))
95 static bool IsSupported (Benchmark benchmark)
97 return _filters.TryGetValue (benchmark.Name, out var filter) ? filter (benchmark) : true;
102 var depDir = Path.Combine ("..", "external", "benchmarker");
103 var benchDir = Path.Combine (depDir, "benchmarks");
104 var testDir = Path.Combine (depDir, "tests");
106 var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
107 .Select (Benchmark.Load)
108 .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b))
109 .OrderBy (b => b.Name)
112 var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
113 var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));
115 var rand = new Random ();
116 var cpus = Environment.ProcessorCount;
118 var results = new List<TestResult> (benchmarks.Length);
120 var sw = Stopwatch.StartNew ();
122 for (var i = 0; i < benchmarks.Length; i++) {
123 var bench = benchmarks [i];
125 var sampleFreq = rand.Next (-1000, 1001);
126 var sampleMode = rand.Next (0, 2) == 1 ? "-real" : string.Empty;
127 var maxSamples = rand.Next (0, cpus * 2000 + 1);
128 var heapShotFreq = rand.Next (-10, 11);
129 var maxFrames = rand.Next (0, 33);
130 var options = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
131 .Select (x => (x.Value ? string.Empty : "no") + x.Key)
134 var profOptions = $"maxframes={maxFrames},{string.Join (",", options)},output=/dev/null";
137 profOptions += $",sample{sampleMode}={sampleFreq},maxsamples={maxSamples}";
139 if (heapShotFreq > 0)
140 profOptions += $",heapshot={heapShotFreq}gc";
142 var info = new ProcessStartInfo {
143 UseShellExecute = false,
144 WorkingDirectory = Path.Combine (testDir, bench.TestDirectory),
146 Arguments = $"--debug --profile=log:{profOptions} " + string.Join (" ", bench.CommandLine),
147 RedirectStandardOutput = true,
148 RedirectStandardError = true,
151 info.EnvironmentVariables.Clear ();
152 info.EnvironmentVariables.Add ("MONO_PATH", classDir);
154 var progress = $"({i + 1}/{benchmarks.Length})";
156 Console.ForegroundColor = ConsoleColor.Blue;
157 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
158 Console.ResetColor ();
160 var result = new TestResult {
165 using (var proc = new Process ()) {
166 proc.StartInfo = info;
168 var stdout = new StringBuilder ();
169 var stderr = new StringBuilder ();
171 proc.OutputDataReceived += (sender, args) => {
172 if (args.Data != null)
174 stdout.AppendLine (args.Data);
177 proc.ErrorDataReceived += (sender, args) => {
178 if (args.Data != null)
180 stderr.AppendLine (args.Data);
183 result.Stopwatch.Start ();
187 proc.BeginOutputReadLine ();
188 proc.BeginErrorReadLine ();
190 if (!proc.WaitForExit ((int) _timeout.TotalMilliseconds)) {
191 // Force a thread dump.
192 Syscall.kill (proc.Id, Signum.SIGQUIT);
197 } catch (Exception) {
200 result.ExitCode = proc.ExitCode;
202 result.Stopwatch.Stop ();
205 result.StandardOutput = stdout.ToString ();
206 result.StandardError = stderr.ToString ();
210 var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";
212 Console.ForegroundColor = result.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
213 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {result.Stopwatch.Elapsed.ToString ("G")} and {resultStr}");
214 Console.ResetColor ();
216 if (result.ExitCode != 0) {
217 Console.ForegroundColor = ConsoleColor.Red;
218 Console.WriteLine ("===== stdout =====");
219 Console.ResetColor ();
221 Console.WriteLine (result.StandardOutput);
223 Console.ForegroundColor = ConsoleColor.Red;
224 Console.WriteLine ("===== stderr =====");
225 Console.ResetColor ();
227 Console.WriteLine (result.StandardError);
230 if (_processors.TryGetValue (bench.Name, out var processor))
233 results.Add (result);
238 var successes = results.Count (r => r.ExitCode == 0);
239 var failures = results.Count (r => r.ExitCode != null && r.ExitCode != 0);
240 var timeouts = results.Count (r => r.ExitCode == null);
242 var settings = new XmlWriterSettings {
243 NewLineOnAttributes = true,
247 using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
248 writer.WriteStartDocument ();
249 writer.WriteComment ("This file represents the results of running a test suite");
251 writer.WriteStartElement ("test-results");
252 writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
253 writer.WriteAttributeString ("total", results.Count.ToString ());
254 writer.WriteAttributeString ("failures", failures.ToString ());
255 writer.WriteAttributeString ("not-run", "0");
256 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
257 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
259 writer.WriteStartElement ("environment");
260 writer.WriteAttributeString ("nunit-version", "2.4.8.0");
261 writer.WriteAttributeString ("clr-version", Environment.Version.ToString ());
262 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString ());
263 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString ());
264 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory);
265 writer.WriteAttributeString ("machine-name", Environment.MachineName);
266 writer.WriteAttributeString ("user", Environment.UserName);
267 writer.WriteAttributeString ("user-domain", Environment.UserDomainName);
268 writer.WriteEndElement ();
270 writer.WriteStartElement ("culture-info");
271 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name);
272 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name);
273 writer.WriteEndElement ();
275 writer.WriteStartElement ("test-suite");
276 writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
277 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
278 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
279 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
280 writer.WriteStartElement ("results");
282 writer.WriteStartElement ("test-suite");
283 writer.WriteAttributeString ("name", "MonoTests");
284 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
285 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
286 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
287 writer.WriteStartElement ("results");
289 writer.WriteStartElement ("test-suite");
290 writer.WriteAttributeString ("name", "profiler-stress");
291 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
292 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
293 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
294 writer.WriteStartElement ("results");
296 foreach (var result in results) {
297 var timeoutStr = result.ExitCode == null ? "_timeout" : string.Empty;
299 writer.WriteStartElement ("test-case");
300 writer.WriteAttributeString ("name", $"MonoTests.profiler-stress.{result.Benchmark.Name}{timeoutStr}");
301 writer.WriteAttributeString ("executed", "True");
302 writer.WriteAttributeString ("success", (result.ExitCode == 0).ToString ());
303 writer.WriteAttributeString ("time", ((int) result.Stopwatch.Elapsed.TotalSeconds).ToString ());
304 writer.WriteAttributeString ("asserts", result.ExitCode == 0 ? "0" : "1");
306 if (result.ExitCode != 0) {
307 writer.WriteStartElement ("failure");
309 writer.WriteStartElement ("message");
310 writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput));
311 writer.WriteEndElement ();
313 writer.WriteStartElement ("stack-trace");
314 writer.WriteCData (FilterInvalidXmlChars (result.StandardError));
315 writer.WriteEndElement ();
317 writer.WriteEndElement ();
320 writer.WriteEndElement ();
323 writer.WriteEndElement ();
324 writer.WriteEndElement ();
326 writer.WriteEndElement ();
327 writer.WriteEndElement ();
329 writer.WriteEndElement ();
330 writer.WriteEndElement ();
332 writer.WriteEndElement ();
334 writer.WriteEndDocument ();
337 var failureStr = failures + timeouts != 0 ? $" ({failures} failures, {timeouts} timeouts)" : string.Empty;
339 Console.ForegroundColor = failures + timeouts != 0 ? ConsoleColor.Red : ConsoleColor.Green;
340 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{results.Count} passing tests{failureStr}");
341 Console.ResetColor ();
343 return failures + timeouts;