X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Ftests%2Ftest-runner.cs;h=80a294566dfec3e5f4357d1c142976badb608f22;hb=ef0ddf45c3081e799edcb4e95770186514b80cf1;hp=6dc6745d53e2453804c79b8191e4deb1366a0d0f;hpb=93ce056a764e8048f33548a3744ba2bd84072043;p=mono.git diff --git a/mono/tests/test-runner.cs b/mono/tests/test-runner.cs index 6dc6745d53e..80a294566df 100644 --- a/mono/tests/test-runner.cs +++ b/mono/tests/test-runner.cs @@ -6,24 +6,7 @@ // // Copyright (C) 2008 Novell, Inc (http://www.novell.com) // -// Permission is hereby granted, free of charge, to any person obtaining -// a copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so, subject to -// the following conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. // using System; using System.IO; @@ -32,6 +15,7 @@ using System.Diagnostics; using System.Collections.Generic; using System.Globalization; using System.Xml; +using System.Text.RegularExpressions; // // This is a simple test runner with support for parallel execution @@ -39,21 +23,13 @@ using System.Xml; public class TestRunner { + const string TEST_TIME_FORMAT = "mm\\:ss\\.fff"; + const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC"; + class ProcessData { public string test; public StreamWriter stdout, stderr; public string stdoutFile, stderrFile; - - public void CloseStreams () { - if (stdout != null) { - stdout.Close (); - stdout = null; - } - if (stderr != null) { - stderr.Close (); - stderr = null; - } - } } class TestInfo { @@ -64,8 +40,9 @@ public class TestRunner // Defaults int concurrency = 1; int timeout = 2 * 60; // in seconds - - DateTime test_start_time = DateTime.UtcNow; + int expectedExitCode = 0; + string testsuiteName = null; + string inputFile = null; // FIXME: Add support for runtime arguments + env variables @@ -78,7 +55,7 @@ public class TestRunner while (i < args.Length) { if (args [i].StartsWith ("-")) { if (args [i] == "-j") { - if (i + i >= args.Length) { + if (i + 1 >= args.Length) { Console.WriteLine ("Missing argument to -j command line option."); return 1; } @@ -88,34 +65,55 @@ public class TestRunner concurrency = Int32.Parse (args [i + 1]); i += 2; } else if (args [i] == "--timeout") { - if (i + i >= args.Length) { + if (i + 1 >= args.Length) { Console.WriteLine ("Missing argument to --timeout command line option."); return 1; } timeout = Int32.Parse (args [i + 1]); i += 2; } else if (args [i] == "--disabled") { - if (i + i >= args.Length) { + if (i + 1 >= args.Length) { Console.WriteLine ("Missing argument to --disabled command line option."); return 1; } disabled_tests = args [i + 1]; i += 2; } else if (args [i] == "--runtime") { - if (i + i >= args.Length) { + if (i + 1 >= args.Length) { Console.WriteLine ("Missing argument to --runtime command line option."); return 1; } runtime = args [i + 1]; i += 2; } else if (args [i] == "--opt-sets") { - if (i + i >= args.Length) { + if (i + 1 >= args.Length) { Console.WriteLine ("Missing argument to --opt-sets command line option."); return 1; } foreach (var s in args [i + 1].Split ()) opt_sets.Add (s); i += 2; + } else if (args [i] == "--expected-exit-code") { + if (i + 1 >= args.Length) { + Console.WriteLine ("Missing argument to --expected-exit-code command line option."); + return 1; + } + expectedExitCode = Int32.Parse (args [i + 1]); + i += 2; + } else if (args [i] == "--testsuite-name") { + if (i + 1 >= args.Length) { + Console.WriteLine ("Missing argument to --testsuite-name command line option."); + return 1; + } + testsuiteName = args [i + 1]; + i += 2; + } else if (args [i] == "--input-file") { + if (i + 1 >= args.Length) { + Console.WriteLine ("Missing argument to --input-file command line option."); + return 1; + } + inputFile = args [i + 1]; + i += 2; } else { Console.WriteLine ("Unknown command line option: '" + args [i] + "'."); return 1; @@ -125,6 +123,11 @@ public class TestRunner } } + if (String.IsNullOrEmpty (testsuiteName)) { + Console.WriteLine ("Missing the required --testsuite-name command line option."); + return 1; + } + var disabled = new Dictionary (); if (disabled_tests != null) { @@ -132,204 +135,178 @@ public class TestRunner disabled [test] = test; } - // The remaining arguments are the tests var tests = new List (); - for (int j = i; j < args.Length; ++j) - if (!disabled.ContainsKey (args [j])) - tests.Add (args [j]); - int npassed = 0; - int nfailed = 0; + if (!String.IsNullOrEmpty (inputFile)) { + tests.AddRange (File.ReadAllLines (inputFile)); + } else { + // The remaining arguments are the tests + for (int j = i; j < args.Length; ++j) + if (!disabled.ContainsKey (args [j])) + tests.Add (args [j]); + } - var processes = new List (); var passed = new List (); var failed = new List (); - var process_data = new Dictionary (); + var timedout = new List (); object monitor = new object (); - var terminated = new List (); + Console.WriteLine ("Running tests: "); - if (concurrency != 1) - Console.WriteLine ("Running tests: "); - - var test_info = new List (); + var test_info = new Queue (); if (opt_sets.Count == 0) { foreach (string s in tests) - test_info.Add (new TestInfo { test = s }); + test_info.Enqueue (new TestInfo { test = s }); } else { foreach (string opt in opt_sets) { foreach (string s in tests) - test_info.Add (new TestInfo { test = s, opt_set = opt }); + test_info.Enqueue (new TestInfo { test = s, opt_set = opt }); } - } + } + /* compute the max length of test names, to have an optimal output width */ + int output_width = -1; foreach (TestInfo ti in test_info) { - lock (monitor) { - while (processes.Count == concurrency) { - /* Wait for one process to terminate */ - Monitor.Wait (monitor); - } - - /* Cleaup terminated processes */ - foreach (Process dead in terminated) { - if (process_data [dead].stdout != null) - process_data [dead].stdout.Close (); - if (process_data [dead].stderr != null) - process_data [dead].stderr.Close (); - // This is needed to avoid CreateProcess failed errors :( - dead.Close (); - } - terminated.Clear (); - } - - string test = ti.test; - string opt_set = ti.opt_set; - - if (concurrency == 1) - Console.Write ("Testing " + test + "... "); - - /* Spawn a new process */ - string process_args; - if (opt_set == null) - process_args = test; - else - process_args = "-O=" + opt_set + " " + test; - ProcessStartInfo info = new ProcessStartInfo (runtime, process_args); - info.UseShellExecute = false; - info.RedirectStandardOutput = true; - info.RedirectStandardError = true; - Process p = new Process (); - p.StartInfo = info; - p.EnableRaisingEvents = true; - - ProcessData data = new ProcessData (); - data.test = test; - - p.Exited += delegate (object sender, EventArgs e) { - // Anon methods share some of their state, so we can't use - // variables which change during the loop (test, p) - Process dead = (Process)sender; - - lock (monitor) { - if (dead.ExitCode == 0) { - if (concurrency == 1) - Console.WriteLine ("passed."); - else - Console.Write ("."); - passed.Add(process_data [dead]); - npassed ++; - } else { - if (concurrency == 1) - Console.WriteLine ("failed."); - else - Console.Write ("F"); - failed.Add (process_data [dead]); - nfailed ++; - } - processes.Remove (dead); - terminated.Add (dead); - Monitor.Pulse (monitor); - } - }; - - string log_prefix = ""; - if (opt_set != null) - log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_"); - - data.stdoutFile = test + log_prefix + ".stdout"; - data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create)); - - data.stderrFile = test + log_prefix + ".stderr"; - data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create)); + if (ti.test.Length > output_width) + output_width = Math.Min (120, ti.test.Length); + } - p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) { - Process p2 = (Process)sender; + List threads = new List (concurrency); - StreamWriter fs; + DateTime test_start_time = DateTime.UtcNow; - lock (monitor) { - fs = process_data [p2].stdout; + for (int j = 0; j < concurrency; ++j) { + Thread thread = new Thread (() => { + while (true) { + TestInfo ti; - if (String.IsNullOrEmpty (e.Data)) - process_data [p2].stdout = null; - } + lock (monitor) { + if (test_info.Count == 0) + break; + ti = test_info.Dequeue (); + } - if (String.IsNullOrEmpty (e.Data)) { - fs.Close (); - } else { - fs.WriteLine (e.Data); - fs.Flush (); - } - }; + var output = new StringWriter (); - p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) { - Process p2 = (Process)sender; + string test = ti.test; + string opt_set = ti.opt_set; - StreamWriter fs; + output.Write (String.Format ("{{0,-{0}}} ", output_width), test); - lock (monitor) { - fs = process_data [p2].stderr; + /* Spawn a new process */ + string process_args; + if (opt_set == null) + process_args = test; + else + process_args = "-O=" + opt_set + " " + test; + ProcessStartInfo info = new ProcessStartInfo (runtime, process_args); + info.UseShellExecute = false; + info.RedirectStandardOutput = true; + info.RedirectStandardError = true; + info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString(); + Process p = new Process (); + p.StartInfo = info; + + ProcessData data = new ProcessData (); + data.test = test; + + string log_prefix = ""; + if (opt_set != null) + log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_"); + + data.stdoutFile = test + log_prefix + ".stdout"; + data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create)); + + data.stderrFile = test + log_prefix + ".stderr"; + data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create)); + + p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) { + if (e.Data != null) { + data.stdout.WriteLine (e.Data); + } else { + data.stdout.Flush (); + data.stdout.Close (); + } + }; + + p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) { + if (e.Data != null) { + data.stderr.WriteLine (e.Data); + } else { + data.stderr.Flush (); + data.stderr.Close (); + } + }; + + var start = DateTime.UtcNow; + + p.Start (); + + p.BeginOutputReadLine (); + p.BeginErrorReadLine (); + + if (!p.WaitForExit (timeout * 1000)) { + lock (monitor) { + timedout.Add (data); + } + + output.Write ("timed out"); + + p.Kill (); + } else if (p.ExitCode != expectedExitCode) { + var end = DateTime.UtcNow; + + lock (monitor) { + failed.Add (data); + } + + output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode); + } else { + var end = DateTime.UtcNow; - if (String.IsNullOrEmpty (e.Data)) - process_data [p2].stderr = null; + lock (monitor) { + passed.Add (data); + } - } + output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT)); + } - if (String.IsNullOrEmpty (e.Data)) { - fs.Close (); + p.Close (); lock (monitor) { - process_data [p2].stderr = null; + Console.WriteLine (output.ToString ()); } - } else { - fs.WriteLine (e.Data); - fs.Flush (); } - }; + }); - lock (monitor) { - processes.Add (p); - process_data [p] = data; - } - p.Start (); + thread.Start (); - p.BeginOutputReadLine (); - p.BeginErrorReadLine (); + threads.Add (thread); } - bool timed_out = false; - - /* Wait for all processes to terminate */ - while (true) { - lock (monitor) { - int nprocesses = processes.Count; + for (int j = 0; j < threads.Count; ++j) + threads [j].Join (); - if (nprocesses == 0) - break; + TimeSpan test_time = DateTime.UtcNow - test_start_time; - bool res = Monitor.Wait (monitor, 1000 * timeout); - if (!res) { - timed_out = true; - break; - } - } - } + int npassed = passed.Count; + int nfailed = failed.Count; + int ntimedout = timedout.Count; - TimeSpan test_time = DateTime.UtcNow - test_start_time; XmlWriterSettings xmlWriterSettings = new XmlWriterSettings (); xmlWriterSettings.NewLineOnAttributes = true; xmlWriterSettings.Indent = true; - using (XmlWriter writer = XmlWriter.Create ("TestResults_runtime.xml", xmlWriterSettings)) { + using (XmlWriter writer = XmlWriter.Create (String.Format ("TestResult-{0}.xml", testsuiteName), xmlWriterSettings)) { // writer.WriteStartDocument (); // writer.WriteComment ("This file represents the results of running a test suite"); // writer.WriteStartElement ("test-results"); - writer.WriteAttributeString ("name", "runtime-tests.dummy"); - writer.WriteAttributeString ("total", (npassed + nfailed).ToString()); - writer.WriteAttributeString ("failures", nfailed.ToString()); + writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName)); + writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString()); + writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString()); writer.WriteAttributeString ("not-run", "0"); writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd")); writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss")); @@ -351,33 +328,33 @@ public class TestRunner writer.WriteEndElement (); // writer.WriteStartElement ("test-suite"); - writer.WriteAttributeString ("name","runtime-tests.dummy"); - writer.WriteAttributeString ("success", (nfailed == 0).ToString()); + writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName)); + writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString()); writer.WriteAttributeString ("time", test_time.Seconds.ToString()); - writer.WriteAttributeString ("asserts", nfailed.ToString()); + writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString()); // writer.WriteStartElement ("results"); // writer.WriteStartElement ("test-suite"); writer.WriteAttributeString ("name","MonoTests"); - writer.WriteAttributeString ("success", (nfailed == 0).ToString()); + writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString()); writer.WriteAttributeString ("time", test_time.Seconds.ToString()); - writer.WriteAttributeString ("asserts", nfailed.ToString()); + writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString()); // writer.WriteStartElement ("results"); // writer.WriteStartElement ("test-suite"); - writer.WriteAttributeString ("name","runtime"); - writer.WriteAttributeString ("success", (nfailed == 0).ToString()); + writer.WriteAttributeString ("name", testsuiteName); + writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString()); writer.WriteAttributeString ("time", test_time.Seconds.ToString()); - writer.WriteAttributeString ("asserts", nfailed.ToString()); + writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString()); // writer.WriteStartElement ("results"); // Dump all passing tests first foreach (ProcessData pd in passed) { // writer.WriteStartElement ("test-case"); - writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test); + writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test)); writer.WriteAttributeString ("executed", "True"); writer.WriteAttributeString ("success", "True"); writer.WriteAttributeString ("time", "0"); @@ -388,7 +365,26 @@ public class TestRunner foreach (ProcessData pd in failed) { // writer.WriteStartElement ("test-case"); - writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test); + writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test)); + writer.WriteAttributeString ("executed", "True"); + writer.WriteAttributeString ("success", "False"); + writer.WriteAttributeString ("time", "0"); + writer.WriteAttributeString ("asserts", "1"); + writer.WriteStartElement ("failure"); + writer.WriteStartElement ("message"); + writer.WriteCData (DumpPseudoTrace (pd.stdoutFile)); + writer.WriteEndElement (); + writer.WriteStartElement ("stack-trace"); + writer.WriteCData (DumpPseudoTrace (pd.stderrFile)); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteEndElement (); + } + // Then dump all timing out tests + foreach (ProcessData pd in timedout) { + // + writer.WriteStartElement ("test-case"); + writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test)); writer.WriteAttributeString ("executed", "True"); writer.WriteAttributeString ("success", "False"); writer.WriteAttributeString ("time", "0"); @@ -421,35 +417,35 @@ public class TestRunner } Console.WriteLine (); + Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT)); + Console.WriteLine (); + Console.WriteLine ("{0,4} test(s) passed", npassed); + Console.WriteLine ("{0,4} test(s) failed", nfailed); + Console.WriteLine ("{0,4} test(s) timed out", ntimedout); - if (timed_out) { - Console.WriteLine ("\nrunning tests timed out:\n"); - Console.WriteLine (npassed + nfailed); - lock (monitor) { - foreach (Process p in processes) { - ProcessData pd = process_data [p]; - pd.CloseStreams (); - Console.WriteLine (pd.test); - p.Kill (); - DumpFile (pd.stdoutFile); - DumpFile (pd.stderrFile); - } + if (nfailed > 0) { + Console.WriteLine (); + Console.WriteLine ("Failed test(s):"); + foreach (ProcessData pd in failed) { + Console.WriteLine (); + Console.WriteLine (pd.test); + DumpFile (pd.stdoutFile); + DumpFile (pd.stderrFile); } - return 1; } - Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass."); - if (nfailed > 0) { - Console.WriteLine ("\nFailed tests:\n"); - foreach (ProcessData pd in failed) { + if (ntimedout > 0) { + Console.WriteLine (); + Console.WriteLine ("Timed out test(s):"); + foreach (ProcessData pd in timedout) { + Console.WriteLine (); Console.WriteLine (pd.test); DumpFile (pd.stdoutFile); DumpFile (pd.stderrFile); } - return 1; - } else { - return 0; } + + return (ntimedout == 0 && nfailed == 0) ? 0 : 1; } static void DumpFile (string filename) { @@ -462,8 +458,14 @@ public class TestRunner static string DumpPseudoTrace (string filename) { if (File.Exists (filename)) - return File.ReadAllText (filename); + return FilterInvalidXmlChars (File.ReadAllText (filename)); else return string.Empty; } + + static string FilterInvalidXmlChars (string text) { + // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML: + // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */ + return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", ""); + } }