5 // Zoltan Varga (vargaz@gmail.com)
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
13 using System.Threading;
14 using System.Diagnostics;
15 using System.Collections.Generic;
16 using System.Globalization;
19 using System.Text.RegularExpressions;
22 using Mono.Unix.Native;
26 // This is a simple test runner with support for parallel execution
29 public class TestRunner
31 const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
32 const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
33 const string MONO_PATH = "MONO_PATH";
37 public StringBuilder stdout, stderr;
38 public string stdoutName, stderrName;
42 public string test, opt_set;
45 public static int Main (String[] args) {
48 int timeout = 2 * 60; // in seconds
49 int expectedExitCode = 0;
51 string testsuiteName = null;
52 string inputFile = null;
54 string disabled_tests = null;
55 string runtime = "mono";
57 string mono_path = null;
58 var opt_sets = new List<string> ();
60 string aot_run_flags = null;
61 string aot_build_flags = null;
65 while (i < args.Length) {
66 if (args [i].StartsWith ("-")) {
67 if (args [i] == "-j") {
68 if (i + 1 >= args.Length) {
69 Console.WriteLine ("Missing argument to -j command line option.");
72 if (args [i + 1] == "a")
73 concurrency = Environment.ProcessorCount;
75 concurrency = Int32.Parse (args [i + 1]);
77 } else if (args [i] == "--timeout") {
78 if (i + 1 >= args.Length) {
79 Console.WriteLine ("Missing argument to --timeout command line option.");
82 timeout = Int32.Parse (args [i + 1]);
84 } else if (args [i] == "--disabled") {
85 if (i + 1 >= args.Length) {
86 Console.WriteLine ("Missing argument to --disabled command line option.");
89 disabled_tests = args [i + 1];
91 } else if (args [i] == "--runtime") {
92 if (i + 1 >= args.Length) {
93 Console.WriteLine ("Missing argument to --runtime command line option.");
96 runtime = args [i + 1];
98 } else if (args [i] == "--config") {
99 if (i + 1 >= args.Length) {
100 Console.WriteLine ("Missing argument to --config command line option.");
103 config = args [i + 1];
105 } else if (args [i] == "--opt-sets") {
106 if (i + 1 >= args.Length) {
107 Console.WriteLine ("Missing argument to --opt-sets command line option.");
110 foreach (var s in args [i + 1].Split ())
113 } else if (args [i] == "--expected-exit-code") {
114 if (i + 1 >= args.Length) {
115 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
118 expectedExitCode = Int32.Parse (args [i + 1]);
120 } else if (args [i] == "--testsuite-name") {
121 if (i + 1 >= args.Length) {
122 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
125 testsuiteName = args [i + 1];
127 } else if (args [i] == "--input-file") {
128 if (i + 1 >= args.Length) {
129 Console.WriteLine ("Missing argument to --input-file command line option.");
132 inputFile = args [i + 1];
134 } else if (args [i] == "--runtime") {
135 if (i + 1 >= args.Length) {
136 Console.WriteLine ("Missing argument to --runtime command line option.");
139 runtime = args [i + 1];
141 } else if (args [i] == "--mono-path") {
142 if (i + 1 >= args.Length) {
143 Console.WriteLine ("Missing argument to --mono-path command line option.");
146 mono_path = args [i + 1].Substring(0, args [i + 1].Length);
149 } else if (args [i] == "--aot-run-flags") {
150 if (i + 1 >= args.Length) {
151 Console.WriteLine ("Missing argument to --aot-run-flags command line option.");
154 aot_run_flags = args [i + 1].Substring(0, args [i + 1].Length);
156 } else if (args [i] == "--aot-build-flags") {
157 if (i + 1 >= args.Length) {
158 Console.WriteLine ("Missing argument to --aot-build-flags command line option.");
161 aot_build_flags = args [i + 1].Substring(0, args [i + 1].Length);
163 } else if (args [i] == "--verbose") {
167 Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
175 if (String.IsNullOrEmpty (testsuiteName)) {
176 Console.WriteLine ("Missing the required --testsuite-name command line option.");
180 var disabled = new Dictionary <string, string> ();
182 if (disabled_tests != null) {
183 foreach (string test in disabled_tests.Split ())
184 disabled [test] = test;
187 var tests = new List<string> ();
189 if (!String.IsNullOrEmpty (inputFile)) {
190 tests.AddRange (File.ReadAllLines (inputFile));
192 // The remaining arguments are the tests
193 for (int j = i; j < args.Length; ++j)
194 if (!disabled.ContainsKey (args [j]))
195 tests.Add (args [j]);
198 var passed = new List<ProcessData> ();
199 var failed = new List<ProcessData> ();
200 var timedout = new List<ProcessData> ();
202 object monitor = new object ();
204 Console.WriteLine ("Running tests: ");
206 var test_info = new Queue<TestInfo> ();
207 if (opt_sets.Count == 0) {
208 foreach (string s in tests)
209 test_info.Enqueue (new TestInfo { test = s });
211 foreach (string opt in opt_sets) {
212 foreach (string s in tests)
213 test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
217 /* compute the max length of test names, to have an optimal output width */
218 int output_width = -1;
219 foreach (TestInfo ti in test_info) {
220 if (ti.test.Length > output_width)
221 output_width = Math.Min (120, ti.test.Length);
224 if (aot_build_flags != null) {
225 Console.WriteLine("AOT compiling tests");
227 object aot_monitor = new object ();
228 var aot_queue = new Queue<String> (tests);
230 List<Thread> build_threads = new List<Thread> (concurrency);
232 for (int j = 0; j < concurrency; ++j) {
233 Thread thread = new Thread (() => {
238 if (aot_queue.Count == 0)
240 test_name = aot_queue.Dequeue ();
243 string test_bitcode_output = test_name + "_bitcode_tmp";
244 string test_bitcode_arg = ",temp-path=" + test_bitcode_output;
245 string aot_args = aot_build_flags + test_bitcode_arg + " " + test_name;
247 Directory.CreateDirectory(test_bitcode_output);
249 ProcessStartInfo job = new ProcessStartInfo (runtime, aot_args);
250 job.UseShellExecute = false;
251 job.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
252 job.EnvironmentVariables[MONO_PATH] = mono_path;
253 Process compiler = new Process ();
254 compiler.StartInfo = job;
258 if (!compiler.WaitForExit (timeout * 1000)) {
263 throw new Exception(String.Format("Timeout AOT compiling tests, output in {0}", test_bitcode_output));
264 } else if (compiler.ExitCode != 0) {
265 throw new Exception(String.Format("Error AOT compiling tests, output in {0}", test_bitcode_output));
267 Directory.Delete (test_bitcode_output, true);
274 build_threads.Add (thread);
277 for (int j = 0; j < build_threads.Count; ++j)
278 build_threads [j].Join ();
280 Console.WriteLine("Done compiling");
283 List<Thread> threads = new List<Thread> (concurrency);
285 DateTime test_start_time = DateTime.UtcNow;
287 for (int j = 0; j < concurrency; ++j) {
288 Thread thread = new Thread (() => {
293 if (test_info.Count == 0)
295 ti = test_info.Dequeue ();
298 var output = new StringWriter ();
300 string test = ti.test;
301 string opt_set = ti.opt_set;
304 output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
311 if (aot_run_flags != null)
312 test_invoke = aot_run_flags + " " + test;
316 /* Spawn a new process */
319 process_args = test_invoke;
321 process_args = "-O=" + opt_set + " " + test_invoke;
323 ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
324 info.UseShellExecute = false;
325 info.RedirectStandardOutput = true;
326 info.RedirectStandardError = true;
327 info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
329 info.EnvironmentVariables["MONO_CONFIG"] = config;
330 if (mono_path != null)
331 info.EnvironmentVariables[MONO_PATH] = mono_path;
332 Process p = new Process ();
335 ProcessData data = new ProcessData ();
338 string log_prefix = "";
340 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
342 data.stdoutName = test + log_prefix + ".stdout";
343 data.stdout = new StringBuilder ();
345 data.stderrName = test + log_prefix + ".stderr";
346 data.stderr = new StringBuilder ();
348 p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
349 if (e.Data != null) {
350 data.stdout.AppendLine (e.Data);
354 p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
355 if (e.Data != null) {
356 data.stderr.AppendLine (e.Data);
360 var start = DateTime.UtcNow;
364 p.BeginOutputReadLine ();
365 p.BeginErrorReadLine ();
367 if (!p.WaitForExit (timeout * 1000)) {
373 // Force the process to print a thread dump
375 Syscall.kill (p.Id, Signum.SIGQUIT);
382 output.Write ($"timed out ({timeout}s)");
389 } else if (p.ExitCode != expectedExitCode) {
390 var end = DateTime.UtcNow;
397 output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
399 var end = DateTime.UtcNow;
406 output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
413 Console.WriteLine (output.ToString ());
420 threads.Add (thread);
423 for (int j = 0; j < threads.Count; ++j)
426 TimeSpan test_time = DateTime.UtcNow - test_start_time;
428 int npassed = passed.Count;
429 int nfailed = failed.Count;
430 int ntimedout = timedout.Count;
432 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
433 xmlWriterSettings.NewLineOnAttributes = true;
434 xmlWriterSettings.Indent = true;
436 string xmlPath = String.Format ("TestResult-{0}.xml", testsuiteName);
437 using (XmlWriter writer = XmlWriter.Create (xmlPath, xmlWriterSettings)) {
438 // <?xml version="1.0" encoding="utf-8" standalone="no"?>
439 writer.WriteStartDocument ();
440 // <!--This file represents the results of running a test suite-->
441 writer.WriteComment ("This file represents the results of running a test suite");
442 // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
443 writer.WriteStartElement ("test-results");
444 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
445 writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
446 writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
447 writer.WriteAttributeString ("not-run", "0");
448 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
449 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
450 // <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
451 writer.WriteStartElement ("environment");
452 writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
453 writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
454 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
455 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
456 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
457 writer.WriteAttributeString ("machine-name", Environment.MachineName );
458 writer.WriteAttributeString ("user", Environment.UserName );
459 writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
460 writer.WriteEndElement ();
461 // <culture-info current-culture="en-GB" current-uiculture="en-GB" />
462 writer.WriteStartElement ("culture-info");
463 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
464 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
465 writer.WriteEndElement ();
466 // <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
467 writer.WriteStartElement ("test-suite");
468 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
469 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
470 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
471 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
473 writer.WriteStartElement ("results");
474 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
475 writer.WriteStartElement ("test-suite");
476 writer.WriteAttributeString ("name","MonoTests");
477 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
478 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
479 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
481 writer.WriteStartElement ("results");
482 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
483 writer.WriteStartElement ("test-suite");
484 writer.WriteAttributeString ("name", testsuiteName);
485 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
486 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
487 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
489 writer.WriteStartElement ("results");
490 // Dump all passing tests first
491 foreach (ProcessData pd in passed) {
492 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
493 writer.WriteStartElement ("test-case");
494 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
495 writer.WriteAttributeString ("executed", "True");
496 writer.WriteAttributeString ("success", "True");
497 writer.WriteAttributeString ("time", "0");
498 writer.WriteAttributeString ("asserts", "0");
499 writer.WriteEndElement ();
501 // Now dump all failing tests
502 foreach (ProcessData pd in failed) {
503 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
504 writer.WriteStartElement ("test-case");
505 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
506 writer.WriteAttributeString ("executed", "True");
507 writer.WriteAttributeString ("success", "False");
508 writer.WriteAttributeString ("time", "0");
509 writer.WriteAttributeString ("asserts", "1");
510 writer.WriteStartElement ("failure");
511 writer.WriteStartElement ("message");
512 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
513 writer.WriteEndElement ();
514 writer.WriteStartElement ("stack-trace");
515 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
516 writer.WriteEndElement ();
517 writer.WriteEndElement ();
518 writer.WriteEndElement ();
520 // Then dump all timing out tests
521 foreach (ProcessData pd in timedout) {
522 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
523 writer.WriteStartElement ("test-case");
524 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
525 writer.WriteAttributeString ("executed", "True");
526 writer.WriteAttributeString ("success", "False");
527 writer.WriteAttributeString ("time", "0");
528 writer.WriteAttributeString ("asserts", "1");
529 writer.WriteStartElement ("failure");
530 writer.WriteStartElement ("message");
531 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
532 writer.WriteEndElement ();
533 writer.WriteStartElement ("stack-trace");
534 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
535 writer.WriteEndElement ();
536 writer.WriteEndElement ();
537 writer.WriteEndElement ();
540 writer.WriteEndElement ();
542 writer.WriteEndElement ();
544 writer.WriteEndElement ();
546 writer.WriteEndElement ();
548 writer.WriteEndElement ();
550 writer.WriteEndElement ();
552 writer.WriteEndElement ();
553 writer.WriteEndDocument ();
555 string babysitterXmlList = Environment.GetEnvironmentVariable("MONO_BABYSITTER_NUNIT_XML_LIST_FILE");
556 if (!String.IsNullOrEmpty(babysitterXmlList)) {
558 string fullXmlPath = Path.GetFullPath(xmlPath);
559 File.AppendAllText(babysitterXmlList, fullXmlPath + Environment.NewLine);
560 } catch (Exception e) {
561 Console.WriteLine("Attempted to record XML path to file {0} but failed.", babysitterXmlList);
567 Console.WriteLine ();
568 Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
569 Console.WriteLine ();
570 Console.WriteLine ("{0,4} test(s) passed", npassed);
571 Console.WriteLine ("{0,4} test(s) failed", nfailed);
572 Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
574 Console.WriteLine ();
575 Console.WriteLine (String.Format ("{0} test(s) passed, {1} test(s) did not pass.", npassed, nfailed));
579 Console.WriteLine ();
580 Console.WriteLine ("Failed test(s):");
581 foreach (ProcessData pd in failed) {
582 Console.WriteLine ();
583 Console.WriteLine (pd.test);
584 DumpFile (pd.stdoutName, pd.stdout.ToString ());
585 DumpFile (pd.stderrName, pd.stderr.ToString ());
590 Console.WriteLine ();
591 Console.WriteLine ("Timed out test(s):");
592 foreach (ProcessData pd in timedout) {
593 Console.WriteLine ();
594 Console.WriteLine (pd.test);
595 DumpFile (pd.stdoutName, pd.stdout.ToString ());
596 DumpFile (pd.stderrName, pd.stderr.ToString ());
600 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
603 static void DumpFile (string filename, string text) {
604 Console.WriteLine ("=============== {0} ===============", filename);
605 Console.WriteLine (text);
606 Console.WriteLine ("=============== EOF ===============");
609 static string FilterInvalidXmlChars (string text) {
610 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
611 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
612 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");