5 // Zoltan Varga (vargaz@gmail.com)
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
13 using System.Threading;
14 using System.Diagnostics;
15 using System.Collections.Generic;
16 using System.Globalization;
19 using System.Text.RegularExpressions;
21 #if !FULL_AOT_DESKTOP && !MOBILE
22 using Mono.Unix.Native;
26 // This is a simple test runner with support for parallel execution
29 public class TestRunner
31 const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
32 const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
33 const string MONO_PATH = "MONO_PATH";
34 const string MONO_GAC_PREFIX = "MONO_GAC_PREFIX";
38 public StringBuilder stdout, stderr;
39 public object stdoutLock = new object (), stderrLock = new object ();
40 public string stdoutName, stderrName;
44 public string test, opt_set;
47 public static int Main (String[] args) {
50 int timeout = 2 * 60; // in seconds
51 int expectedExitCode = 0;
53 string testsuiteName = null;
54 string inputFile = null;
56 string disabled_tests = null;
57 string runtime = "mono";
59 string mono_path = null;
60 string runtime_args = null;
61 string mono_gac_prefix = null;
62 var opt_sets = new List<string> ();
66 while (i < args.Length) {
67 if (args [i].StartsWith ("-")) {
68 if (args [i] == "-j") {
69 if (i + 1 >= args.Length) {
70 Console.WriteLine ("Missing argument to -j command line option.");
73 if (args [i + 1] == "a")
74 concurrency = Environment.ProcessorCount;
76 concurrency = Int32.Parse (args [i + 1]);
78 } else if (args [i] == "--timeout") {
79 if (i + 1 >= args.Length) {
80 Console.WriteLine ("Missing argument to --timeout command line option.");
83 timeout = Int32.Parse (args [i + 1]);
85 } else if (args [i] == "--disabled") {
86 if (i + 1 >= args.Length) {
87 Console.WriteLine ("Missing argument to --disabled command line option.");
90 disabled_tests = args [i + 1];
92 } else if (args [i] == "--runtime") {
93 if (i + 1 >= args.Length) {
94 Console.WriteLine ("Missing argument to --runtime command line option.");
97 runtime = args [i + 1];
99 } else if (args [i] == "--runtime-args") {
100 if (i + 1 >= args.Length) {
101 Console.WriteLine ("Missing argument to --runtime-args command line option.");
104 runtime_args = (runtime_args ?? "") + " " + args [i + 1];
106 } else if (args [i] == "--config") {
107 if (i + 1 >= args.Length) {
108 Console.WriteLine ("Missing argument to --config command line option.");
111 config = args [i + 1];
113 } else if (args [i] == "--opt-sets") {
114 if (i + 1 >= args.Length) {
115 Console.WriteLine ("Missing argument to --opt-sets command line option.");
118 foreach (var s in args [i + 1].Split ())
121 } else if (args [i] == "--expected-exit-code") {
122 if (i + 1 >= args.Length) {
123 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
126 expectedExitCode = Int32.Parse (args [i + 1]);
128 } else if (args [i] == "--testsuite-name") {
129 if (i + 1 >= args.Length) {
130 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
133 testsuiteName = args [i + 1];
135 } else if (args [i] == "--input-file") {
136 if (i + 1 >= args.Length) {
137 Console.WriteLine ("Missing argument to --input-file command line option.");
140 inputFile = args [i + 1];
142 } else if (args [i] == "--mono-path") {
143 if (i + 1 >= args.Length) {
144 Console.WriteLine ("Missing argument to --mono-path command line option.");
147 mono_path = args [i + 1].Substring(0, args [i + 1].Length);
150 } else if (args [i] == "--mono-gac-prefix") {
151 if (i + 1 >= args.Length) {
152 Console.WriteLine ("Missing argument to --mono-gac-prefix command line option.");
155 mono_gac_prefix = args[i + 1];
157 } else if (args [i] == "--verbose") {
161 Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
169 if (String.IsNullOrEmpty (testsuiteName)) {
170 Console.WriteLine ("Missing the required --testsuite-name command line option.");
174 var disabled = new Dictionary <string, string> ();
176 if (disabled_tests != null) {
177 foreach (string test in disabled_tests.Split ())
178 disabled [test] = test;
181 var tests = new List<string> ();
183 if (!String.IsNullOrEmpty (inputFile)) {
184 tests.AddRange (File.ReadAllLines (inputFile));
186 // The remaining arguments are the tests
187 for (int j = i; j < args.Length; ++j)
188 if (!disabled.ContainsKey (args [j]))
189 tests.Add (args [j]);
192 var passed = new List<ProcessData> ();
193 var failed = new List<ProcessData> ();
194 var timedout = new List<ProcessData> ();
196 object monitor = new object ();
198 Console.WriteLine ("Running tests: ");
200 var test_info = new Queue<TestInfo> ();
201 if (opt_sets.Count == 0) {
202 foreach (string s in tests)
203 test_info.Enqueue (new TestInfo { test = s });
205 foreach (string opt in opt_sets) {
206 foreach (string s in tests)
207 test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
211 /* compute the max length of test names, to have an optimal output width */
212 int output_width = -1;
213 foreach (TestInfo ti in test_info) {
214 if (ti.test.Length > output_width)
215 output_width = Math.Min (120, ti.test.Length);
218 List<Thread> threads = new List<Thread> (concurrency);
220 DateTime test_start_time = DateTime.UtcNow;
222 for (int j = 0; j < concurrency; ++j) {
223 Thread thread = new Thread (() => {
228 if (test_info.Count == 0)
230 ti = test_info.Dequeue ();
233 var output = new StringWriter ();
235 string test = ti.test;
236 string opt_set = ti.opt_set;
239 output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
244 /* Spawn a new process */
246 string process_args = "";
249 process_args += " -O=" + opt_set;
250 if (runtime_args != null)
251 process_args += " " + runtime_args;
253 process_args += " " + test;
255 ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
256 info.UseShellExecute = false;
257 info.RedirectStandardOutput = true;
258 info.RedirectStandardError = true;
259 info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
261 info.EnvironmentVariables["MONO_CONFIG"] = config;
262 if (mono_path != null)
263 info.EnvironmentVariables[MONO_PATH] = mono_path;
264 if (mono_gac_prefix != null)
265 info.EnvironmentVariables[MONO_GAC_PREFIX] = mono_gac_prefix;
266 Process p = new Process ();
269 ProcessData data = new ProcessData ();
272 string log_prefix = "";
274 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
276 data.stdoutName = test + log_prefix + ".stdout";
277 data.stdout = new StringBuilder ();
279 data.stderrName = test + log_prefix + ".stderr";
280 data.stderr = new StringBuilder ();
282 p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
283 lock (data.stdoutLock) {
285 data.stdout.AppendLine (e.Data);
289 p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
290 lock (data.stderrLock) {
292 data.stderr.AppendLine (e.Data);
296 var start = DateTime.UtcNow;
300 p.BeginOutputReadLine ();
301 p.BeginErrorReadLine ();
303 if (!p.WaitForExit (timeout * 1000)) {
308 // Force the process to print a thread dump
309 TryThreadDump (p.Id, data);
312 output.Write ($"timed out ({timeout}s)");
319 } else if (p.ExitCode != expectedExitCode) {
320 var end = DateTime.UtcNow;
327 output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
329 var end = DateTime.UtcNow;
336 output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
343 Console.WriteLine (output.ToString ());
350 threads.Add (thread);
353 for (int j = 0; j < threads.Count; ++j)
356 TimeSpan test_time = DateTime.UtcNow - test_start_time;
358 int npassed = passed.Count;
359 int nfailed = failed.Count;
360 int ntimedout = timedout.Count;
362 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
363 xmlWriterSettings.NewLineOnAttributes = true;
364 xmlWriterSettings.Indent = true;
366 string xmlPath = String.Format ("TestResult-{0}.xml", testsuiteName);
367 using (XmlWriter writer = XmlWriter.Create (xmlPath, xmlWriterSettings)) {
368 // <?xml version="1.0" encoding="utf-8" standalone="no"?>
369 writer.WriteStartDocument ();
370 // <!--This file represents the results of running a test suite-->
371 writer.WriteComment ("This file represents the results of running a test suite");
372 // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
373 writer.WriteStartElement ("test-results");
374 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
375 writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
376 writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
377 writer.WriteAttributeString ("not-run", "0");
378 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
379 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
380 // <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
381 writer.WriteStartElement ("environment");
382 writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
383 writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
384 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
385 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
386 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
387 writer.WriteAttributeString ("machine-name", Environment.MachineName );
388 writer.WriteAttributeString ("user", Environment.UserName );
389 writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
390 writer.WriteEndElement ();
391 // <culture-info current-culture="en-GB" current-uiculture="en-GB" />
392 writer.WriteStartElement ("culture-info");
393 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
394 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
395 writer.WriteEndElement ();
396 // <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
397 writer.WriteStartElement ("test-suite");
398 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
399 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
400 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
401 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
403 writer.WriteStartElement ("results");
404 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
405 writer.WriteStartElement ("test-suite");
406 writer.WriteAttributeString ("name","MonoTests");
407 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
408 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
409 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
411 writer.WriteStartElement ("results");
412 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
413 writer.WriteStartElement ("test-suite");
414 writer.WriteAttributeString ("name", testsuiteName);
415 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
416 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
417 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
419 writer.WriteStartElement ("results");
420 // Dump all passing tests first
421 foreach (ProcessData pd in passed) {
422 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
423 writer.WriteStartElement ("test-case");
424 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
425 writer.WriteAttributeString ("executed", "True");
426 writer.WriteAttributeString ("success", "True");
427 writer.WriteAttributeString ("time", "0");
428 writer.WriteAttributeString ("asserts", "0");
429 writer.WriteEndElement ();
431 // Now dump all failing tests
432 foreach (ProcessData pd in failed) {
433 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
434 writer.WriteStartElement ("test-case");
435 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
436 writer.WriteAttributeString ("executed", "True");
437 writer.WriteAttributeString ("success", "False");
438 writer.WriteAttributeString ("time", "0");
439 writer.WriteAttributeString ("asserts", "1");
440 writer.WriteStartElement ("failure");
441 writer.WriteStartElement ("message");
442 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
443 writer.WriteEndElement ();
444 writer.WriteStartElement ("stack-trace");
445 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
446 writer.WriteEndElement ();
447 writer.WriteEndElement ();
448 writer.WriteEndElement ();
450 // Then dump all timing out tests
451 foreach (ProcessData pd in timedout) {
452 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
453 writer.WriteStartElement ("test-case");
454 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
455 writer.WriteAttributeString ("executed", "True");
456 writer.WriteAttributeString ("success", "False");
457 writer.WriteAttributeString ("time", "0");
458 writer.WriteAttributeString ("asserts", "1");
459 writer.WriteStartElement ("failure");
460 writer.WriteStartElement ("message");
461 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
462 writer.WriteEndElement ();
463 writer.WriteStartElement ("stack-trace");
464 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
465 writer.WriteEndElement ();
466 writer.WriteEndElement ();
467 writer.WriteEndElement ();
470 writer.WriteEndElement ();
472 writer.WriteEndElement ();
474 writer.WriteEndElement ();
476 writer.WriteEndElement ();
478 writer.WriteEndElement ();
480 writer.WriteEndElement ();
482 writer.WriteEndElement ();
483 writer.WriteEndDocument ();
485 string babysitterXmlList = Environment.GetEnvironmentVariable("MONO_BABYSITTER_NUNIT_XML_LIST_FILE");
486 if (!String.IsNullOrEmpty(babysitterXmlList)) {
488 string fullXmlPath = Path.GetFullPath(xmlPath);
489 File.AppendAllText(babysitterXmlList, fullXmlPath + Environment.NewLine);
490 } catch (Exception e) {
491 Console.WriteLine("Attempted to record XML path to file {0} but failed.", babysitterXmlList);
497 Console.WriteLine ();
498 Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
499 Console.WriteLine ();
500 Console.WriteLine ("{0,4} test(s) passed", npassed);
501 Console.WriteLine ("{0,4} test(s) failed", nfailed);
502 Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
504 Console.WriteLine ();
505 Console.WriteLine (String.Format ("{0} test(s) passed, {1} test(s) did not pass.", npassed, nfailed));
509 Console.WriteLine ();
510 Console.WriteLine ("Failed test(s):");
511 foreach (ProcessData pd in failed) {
512 Console.WriteLine ();
513 Console.WriteLine (pd.test);
514 DumpFile (pd.stdoutName, pd.stdout.ToString ());
515 DumpFile (pd.stderrName, pd.stderr.ToString ());
520 Console.WriteLine ();
521 Console.WriteLine ("Timed out test(s):");
522 foreach (ProcessData pd in timedout) {
523 Console.WriteLine ();
524 Console.WriteLine (pd.test);
525 DumpFile (pd.stdoutName, pd.stdout.ToString ());
526 DumpFile (pd.stderrName, pd.stderr.ToString ());
530 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
533 static void DumpFile (string filename, string text) {
534 Console.WriteLine ("=============== {0} ===============", filename);
535 Console.WriteLine (text);
536 Console.WriteLine ("=============== EOF ===============");
539 static string FilterInvalidXmlChars (string text) {
540 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
541 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
542 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
545 static void TryThreadDump (int pid, ProcessData data)
553 #if !FULL_AOT_DESKTOP && !MOBILE
554 /* LLDB cannot produce managed stacktraces for all the threads */
556 Syscall.kill (pid, Signum.SIGQUIT);
569 static void TryLLDB (int pid, ProcessData data)
571 string filename = Path.GetTempFileName ();
573 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
575 sw.WriteLine ("process attach --pid " + pid);
576 sw.WriteLine ("thread list");
577 sw.WriteLine ("thread backtrace all");
578 sw.WriteLine ("detach");
579 sw.WriteLine ("quit");
582 ProcessStartInfo psi = new ProcessStartInfo {
584 Arguments = "--batch --source \"" + filename + "\" --no-lldbinit",
585 UseShellExecute = false,
586 RedirectStandardError = true,
587 RedirectStandardOutput = true,
590 using (Process process = new Process { StartInfo = psi })
592 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
593 lock (data.stdoutLock) {
595 data.stdout.AppendLine (e.Data);
599 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
600 lock (data.stderrLock) {
602 data.stderr.AppendLine (e.Data);
607 process.BeginOutputReadLine ();
608 process.BeginErrorReadLine ();
609 if (!process.WaitForExit (60 * 1000))
615 static void TryGDB (int pid, ProcessData data)
617 string filename = Path.GetTempFileName ();
619 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
621 sw.WriteLine ("attach " + pid);
622 sw.WriteLine ("info threads");
623 sw.WriteLine ("thread apply all p mono_print_thread_dump(0)");
624 sw.WriteLine ("thread apply all backtrace");
627 ProcessStartInfo psi = new ProcessStartInfo {
629 Arguments = "-batch -x \"" + filename + "\" -nx",
630 UseShellExecute = false,
631 RedirectStandardError = true,
632 RedirectStandardOutput = true,
635 using (Process process = new Process { StartInfo = psi })
637 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
638 lock (data.stdoutLock) {
640 data.stdout.AppendLine (e.Data);
644 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
645 lock (data.stderrLock) {
647 data.stderr.AppendLine (e.Data);
652 process.BeginOutputReadLine ();
653 process.BeginErrorReadLine ();
654 if (!process.WaitForExit (60 * 1000))