5 // Zoltan Varga (vargaz@gmail.com)
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
9 // Permission is hereby granted, free of charge, to any person obtaining
10 // a copy of this software and associated documentation files (the
11 // "Software"), to deal in the Software without restriction, including
12 // without limitation the rights to use, copy, modify, merge, publish,
13 // distribute, sublicense, and/or sell copies of the Software, and to
14 // permit persons to whom the Software is furnished to do so, subject to
15 // the following conditions:
17 // The above copyright notice and this permission notice shall be
18 // included in all copies or substantial portions of the Software.
20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 using System.Threading;
31 using System.Diagnostics;
32 using System.Collections.Generic;
33 using System.Globalization;
35 using System.Text.RegularExpressions;
36 using Mono.Unix.Native;
39 // This is a simple test runner with support for parallel execution
42 public class TestRunner
44 const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
45 const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
49 public StreamWriter stdout, stderr;
50 public string stdoutFile, stderrFile;
54 public string test, opt_set;
57 public static int Main (String[] args) {
60 int timeout = 2 * 60; // in seconds
61 int expectedExitCode = 0;
62 string testsuiteName = null;
63 string inputFile = null;
65 // FIXME: Add support for runtime arguments + env variables
67 string disabled_tests = null;
68 string runtime = "mono";
69 var opt_sets = new List<string> ();
73 while (i < args.Length) {
74 if (args [i].StartsWith ("-")) {
75 if (args [i] == "-j") {
76 if (i + 1 >= args.Length) {
77 Console.WriteLine ("Missing argument to -j command line option.");
80 if (args [i + 1] == "a")
81 concurrency = Environment.ProcessorCount;
83 concurrency = Int32.Parse (args [i + 1]);
85 } else if (args [i] == "--timeout") {
86 if (i + 1 >= args.Length) {
87 Console.WriteLine ("Missing argument to --timeout command line option.");
90 timeout = Int32.Parse (args [i + 1]);
92 } else if (args [i] == "--disabled") {
93 if (i + 1 >= args.Length) {
94 Console.WriteLine ("Missing argument to --disabled command line option.");
97 disabled_tests = args [i + 1];
99 } else if (args [i] == "--runtime") {
100 if (i + 1 >= args.Length) {
101 Console.WriteLine ("Missing argument to --runtime command line option.");
104 runtime = args [i + 1];
106 } else if (args [i] == "--opt-sets") {
107 if (i + 1 >= args.Length) {
108 Console.WriteLine ("Missing argument to --opt-sets command line option.");
111 foreach (var s in args [i + 1].Split ())
114 } else if (args [i] == "--expected-exit-code") {
115 if (i + 1 >= args.Length) {
116 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
119 expectedExitCode = Int32.Parse (args [i + 1]);
121 } else if (args [i] == "--testsuite-name") {
122 if (i + 1 >= args.Length) {
123 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
126 testsuiteName = args [i + 1];
128 } else if (args [i] == "--input-file") {
129 if (i + 1 >= args.Length) {
130 Console.WriteLine ("Missing argument to --input-file command line option.");
133 inputFile = args [i + 1];
136 Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
144 if (String.IsNullOrEmpty (testsuiteName)) {
145 Console.WriteLine ("Missing the required --testsuite-name command line option.");
149 var disabled = new Dictionary <string, string> ();
151 if (disabled_tests != null) {
152 foreach (string test in disabled_tests.Split ())
153 disabled [test] = test;
156 var tests = new List<string> ();
158 if (!String.IsNullOrEmpty (inputFile)) {
159 tests.AddRange (File.ReadAllLines (inputFile));
161 // The remaining arguments are the tests
162 for (int j = i; j < args.Length; ++j)
163 if (!disabled.ContainsKey (args [j]))
164 tests.Add (args [j]);
167 var passed = new List<ProcessData> ();
168 var failed = new List<ProcessData> ();
169 var timedout = new List<ProcessData> ();
171 object monitor = new object ();
173 Console.WriteLine ("Running tests: ");
175 var test_info = new Queue<TestInfo> ();
176 if (opt_sets.Count == 0) {
177 foreach (string s in tests)
178 test_info.Enqueue (new TestInfo { test = s });
180 foreach (string opt in opt_sets) {
181 foreach (string s in tests)
182 test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
186 /* compute the max length of test names, to have an optimal output width */
187 int output_width = -1;
188 foreach (TestInfo ti in test_info) {
189 if (ti.test.Length > output_width)
190 output_width = Math.Min (120, ti.test.Length);
193 List<Thread> threads = new List<Thread> (concurrency);
195 DateTime test_start_time = DateTime.UtcNow;
197 for (int j = 0; j < concurrency; ++j) {
198 Thread thread = new Thread (() => {
203 if (test_info.Count == 0)
205 ti = test_info.Dequeue ();
208 var output = new StringWriter ();
210 string test = ti.test;
211 string opt_set = ti.opt_set;
213 output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
215 /* Spawn a new process */
220 process_args = "-O=" + opt_set + " " + test;
221 ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
222 info.UseShellExecute = false;
223 info.RedirectStandardOutput = true;
224 info.RedirectStandardError = true;
225 info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
226 Process p = new Process ();
229 ProcessData data = new ProcessData ();
232 string log_prefix = "";
234 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
236 data.stdoutFile = test + log_prefix + ".stdout";
237 data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create));
239 data.stderrFile = test + log_prefix + ".stderr";
240 data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create));
242 p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
243 if (e.Data != null) {
244 data.stdout.WriteLine (e.Data);
246 data.stdout.Flush ();
247 data.stdout.Close ();
251 p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
252 if (e.Data != null) {
253 data.stderr.WriteLine (e.Data);
255 data.stderr.Flush ();
256 data.stderr.Close ();
260 var start = DateTime.UtcNow;
264 p.BeginOutputReadLine ();
265 p.BeginErrorReadLine ();
267 if (!p.WaitForExit (timeout * 1000)) {
272 // Force the process to print a thread dump
274 Syscall.kill (p.Id, Signum.SIGQUIT);
279 output.Write ("timed out");
282 } else if (p.ExitCode != expectedExitCode) {
283 var end = DateTime.UtcNow;
289 output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
291 var end = DateTime.UtcNow;
297 output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
303 Console.WriteLine (output.ToString ());
310 threads.Add (thread);
313 for (int j = 0; j < threads.Count; ++j)
316 TimeSpan test_time = DateTime.UtcNow - test_start_time;
318 int npassed = passed.Count;
319 int nfailed = failed.Count;
320 int ntimedout = timedout.Count;
322 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
323 xmlWriterSettings.NewLineOnAttributes = true;
324 xmlWriterSettings.Indent = true;
325 using (XmlWriter writer = XmlWriter.Create (String.Format ("TestResult-{0}.xml", testsuiteName), xmlWriterSettings)) {
326 // <?xml version="1.0" encoding="utf-8" standalone="no"?>
327 writer.WriteStartDocument ();
328 // <!--This file represents the results of running a test suite-->
329 writer.WriteComment ("This file represents the results of running a test suite");
330 // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
331 writer.WriteStartElement ("test-results");
332 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
333 writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
334 writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
335 writer.WriteAttributeString ("not-run", "0");
336 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
337 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
338 // <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
339 writer.WriteStartElement ("environment");
340 writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
341 writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
342 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
343 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
344 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
345 writer.WriteAttributeString ("machine-name", Environment.MachineName );
346 writer.WriteAttributeString ("user", Environment.UserName );
347 writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
348 writer.WriteEndElement ();
349 // <culture-info current-culture="en-GB" current-uiculture="en-GB" />
350 writer.WriteStartElement ("culture-info");
351 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
352 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
353 writer.WriteEndElement ();
354 // <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
355 writer.WriteStartElement ("test-suite");
356 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
357 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
358 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
359 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
361 writer.WriteStartElement ("results");
362 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
363 writer.WriteStartElement ("test-suite");
364 writer.WriteAttributeString ("name","MonoTests");
365 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
366 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
367 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
369 writer.WriteStartElement ("results");
370 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
371 writer.WriteStartElement ("test-suite");
372 writer.WriteAttributeString ("name", testsuiteName);
373 writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
374 writer.WriteAttributeString ("time", test_time.Seconds.ToString());
375 writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
377 writer.WriteStartElement ("results");
378 // Dump all passing tests first
379 foreach (ProcessData pd in passed) {
380 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
381 writer.WriteStartElement ("test-case");
382 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
383 writer.WriteAttributeString ("executed", "True");
384 writer.WriteAttributeString ("success", "True");
385 writer.WriteAttributeString ("time", "0");
386 writer.WriteAttributeString ("asserts", "0");
387 writer.WriteEndElement ();
389 // Now dump all failing tests
390 foreach (ProcessData pd in failed) {
391 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
392 writer.WriteStartElement ("test-case");
393 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
394 writer.WriteAttributeString ("executed", "True");
395 writer.WriteAttributeString ("success", "False");
396 writer.WriteAttributeString ("time", "0");
397 writer.WriteAttributeString ("asserts", "1");
398 writer.WriteStartElement ("failure");
399 writer.WriteStartElement ("message");
400 writer.WriteCData (DumpPseudoTrace (pd.stdoutFile));
401 writer.WriteEndElement ();
402 writer.WriteStartElement ("stack-trace");
403 writer.WriteCData (DumpPseudoTrace (pd.stderrFile));
404 writer.WriteEndElement ();
405 writer.WriteEndElement ();
406 writer.WriteEndElement ();
408 // Then dump all timing out tests
409 foreach (ProcessData pd in timedout) {
410 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
411 writer.WriteStartElement ("test-case");
412 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
413 writer.WriteAttributeString ("executed", "True");
414 writer.WriteAttributeString ("success", "False");
415 writer.WriteAttributeString ("time", "0");
416 writer.WriteAttributeString ("asserts", "1");
417 writer.WriteStartElement ("failure");
418 writer.WriteStartElement ("message");
419 writer.WriteCData (DumpPseudoTrace (pd.stdoutFile));
420 writer.WriteEndElement ();
421 writer.WriteStartElement ("stack-trace");
422 writer.WriteCData (DumpPseudoTrace (pd.stderrFile));
423 writer.WriteEndElement ();
424 writer.WriteEndElement ();
425 writer.WriteEndElement ();
428 writer.WriteEndElement ();
430 writer.WriteEndElement ();
432 writer.WriteEndElement ();
434 writer.WriteEndElement ();
436 writer.WriteEndElement ();
438 writer.WriteEndElement ();
440 writer.WriteEndElement ();
441 writer.WriteEndDocument ();
444 Console.WriteLine ();
445 Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
446 Console.WriteLine ();
447 Console.WriteLine ("{0,4} test(s) passed", npassed);
448 Console.WriteLine ("{0,4} test(s) failed", nfailed);
449 Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
452 Console.WriteLine ();
453 Console.WriteLine ("Failed test(s):");
454 foreach (ProcessData pd in failed) {
455 Console.WriteLine ();
456 Console.WriteLine (pd.test);
457 DumpFile (pd.stdoutFile);
458 DumpFile (pd.stderrFile);
463 Console.WriteLine ();
464 Console.WriteLine ("Timed out test(s):");
465 foreach (ProcessData pd in timedout) {
466 Console.WriteLine ();
467 Console.WriteLine (pd.test);
468 DumpFile (pd.stdoutFile);
469 DumpFile (pd.stderrFile);
473 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
476 static void DumpFile (string filename) {
477 if (File.Exists (filename)) {
478 Console.WriteLine ("=============== {0} ===============", filename);
479 Console.WriteLine (File.ReadAllText (filename));
480 Console.WriteLine ("=============== EOF ===============");
484 static string DumpPseudoTrace (string filename) {
485 if (File.Exists (filename))
486 return FilterInvalidXmlChars (File.ReadAllText (filename));
491 static string FilterInvalidXmlChars (string text) {
492 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
493 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
494 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");