using System.Threading;
using System.Diagnostics;
using System.Collections.Generic;
+using System.Globalization;
+using System.Xml;
//
// This is a simple test runner with support for parallel execution
class ProcessData {
public string test;
public StreamWriter stdout, stderr;
+ public string stdoutFile, stderrFile;
+
+ public void CloseStreams () {
+ if (stdout != null) {
+ stdout.Close ();
+ stdout = null;
+ }
+ if (stderr != null) {
+ stderr.Close ();
+ stderr = null;
+ }
+ }
+ }
+
+ class TestInfo {
+ public string test, opt_set;
}
public static int Main (String[] args) {
int concurrency = 1;
int timeout = 2 * 60; // in seconds
+ DateTime test_start_time = DateTime.UtcNow;
+
// FIXME: Add support for runtime arguments + env variables
string disabled_tests = null;
string runtime = "mono";
+ var opt_sets = new List<string> ();
// Process options
int i = 0;
}
runtime = args [i + 1];
i += 2;
+ } else if (args [i] == "--opt-sets") {
+ if (i + i >= args.Length) {
+ Console.WriteLine ("Missing argument to --opt-sets command line option.");
+ return 1;
+ }
+ foreach (var s in args [i + 1].Split ())
+ opt_sets.Add (s);
+ i += 2;
} else {
Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
return 1;
int nfailed = 0;
var processes = new List<Process> ();
- var failed = new List<string> ();
+ var passed = new List<ProcessData> ();
+ var failed = new List<ProcessData> ();
var process_data = new Dictionary<Process, ProcessData> ();
object monitor = new object ();
if (concurrency != 1)
Console.WriteLine ("Running tests: ");
- foreach (string test in tests) {
+ var test_info = new List<TestInfo> ();
+ if (opt_sets.Count == 0) {
+ foreach (string s in tests)
+ test_info.Add (new TestInfo { test = s });
+ } else {
+ foreach (string opt in opt_sets) {
+ foreach (string s in tests)
+ test_info.Add (new TestInfo { test = s, opt_set = opt });
+ }
+ }
+
+ foreach (TestInfo ti in test_info) {
lock (monitor) {
while (processes.Count == concurrency) {
/* Wait for one process to terminate */
terminated.Clear ();
}
+ string test = ti.test;
+ string opt_set = ti.opt_set;
+
if (concurrency == 1)
Console.Write ("Testing " + test + "... ");
/* Spawn a new process */
- ProcessStartInfo info = new ProcessStartInfo (runtime, test);
+ string process_args;
+ if (opt_set == null)
+ process_args = test;
+ else
+ process_args = "-O=" + opt_set + " " + test;
+ ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
info.UseShellExecute = false;
info.RedirectStandardOutput = true;
info.RedirectStandardError = true;
Console.WriteLine ("passed.");
else
Console.Write (".");
+ passed.Add(process_data [dead]);
npassed ++;
} else {
if (concurrency == 1)
Console.WriteLine ("failed.");
else
Console.Write ("F");
- failed.Add (process_data [dead].test);
+ failed.Add (process_data [dead]);
nfailed ++;
}
processes.Remove (dead);
}
};
- data.stdout = new StreamWriter (new FileStream (test + ".stdout", FileMode.Create));
+ string log_prefix = "";
+ if (opt_set != null)
+ log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
+
+ data.stdoutFile = test + log_prefix + ".stdout";
+ data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create));
- data.stderr = new StreamWriter (new FileStream (test + ".stderr", FileMode.Create));
+ data.stderrFile = test + log_prefix + ".stderr";
+ data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create));
p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
Process p2 = (Process)sender;
process_data [p2].stdout = null;
}
- if (String.IsNullOrEmpty (e.Data))
+ if (String.IsNullOrEmpty (e.Data)) {
fs.Close ();
- else
+ } else {
fs.WriteLine (e.Data);
+ fs.Flush ();
+ }
};
p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
lock (monitor) {
process_data [p2].stderr = null;
}
- }
- else
+ } else {
fs.WriteLine (e.Data);
+ fs.Flush ();
+ }
};
lock (monitor) {
}
}
+ TimeSpan test_time = DateTime.UtcNow - test_start_time;
+ XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
+ xmlWriterSettings.NewLineOnAttributes = true;
+ xmlWriterSettings.Indent = true;
+ using (XmlWriter writer = XmlWriter.Create ("TestResults_runtime.xml", xmlWriterSettings)) {
+ // <?xml version="1.0" encoding="utf-8" standalone="no"?>
+ writer.WriteStartDocument ();
+ // <!--This file represents the results of running a test suite-->
+ writer.WriteComment ("This file represents the results of running a test suite");
+ // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
+ writer.WriteStartElement ("test-results");
+ writer.WriteAttributeString ("name", "runtime-tests.dummy");
+ writer.WriteAttributeString ("total", (npassed + nfailed).ToString());
+ writer.WriteAttributeString ("failures", nfailed.ToString());
+ writer.WriteAttributeString ("not-run", "0");
+ writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
+ writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
+ // <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
+ writer.WriteStartElement ("environment");
+ writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
+ writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
+ writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
+ writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
+ writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
+ writer.WriteAttributeString ("machine-name", Environment.MachineName );
+ writer.WriteAttributeString ("user", Environment.UserName );
+ writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
+ writer.WriteEndElement ();
+ // <culture-info current-culture="en-GB" current-uiculture="en-GB" />
+ writer.WriteStartElement ("culture-info");
+ writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
+ writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
+ writer.WriteEndElement ();
+ // <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
+ writer.WriteStartElement ("test-suite");
+ writer.WriteAttributeString ("name","runtime-tests.dummy");
+ writer.WriteAttributeString ("success", (nfailed == 0).ToString());
+ writer.WriteAttributeString ("time", test_time.Seconds.ToString());
+ writer.WriteAttributeString ("asserts", nfailed.ToString());
+ // <results>
+ writer.WriteStartElement ("results");
+ // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
+ writer.WriteStartElement ("test-suite");
+ writer.WriteAttributeString ("name","MonoTests");
+ writer.WriteAttributeString ("success", (nfailed == 0).ToString());
+ writer.WriteAttributeString ("time", test_time.Seconds.ToString());
+ writer.WriteAttributeString ("asserts", nfailed.ToString());
+ // <results>
+ writer.WriteStartElement ("results");
+ // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
+ writer.WriteStartElement ("test-suite");
+ writer.WriteAttributeString ("name","runtime");
+ writer.WriteAttributeString ("success", (nfailed == 0).ToString());
+ writer.WriteAttributeString ("time", test_time.Seconds.ToString());
+ writer.WriteAttributeString ("asserts", nfailed.ToString());
+ // <results>
+ writer.WriteStartElement ("results");
+ // Dump all passing tests first
+ foreach (ProcessData pd in passed) {
+ // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
+ writer.WriteStartElement ("test-case");
+ writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test);
+ writer.WriteAttributeString ("executed", "True");
+ writer.WriteAttributeString ("success", "True");
+ writer.WriteAttributeString ("time", "0");
+ writer.WriteAttributeString ("asserts", "0");
+ writer.WriteEndElement ();
+ }
+ // Now dump all failing tests
+ foreach (ProcessData pd in failed) {
+ // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
+ writer.WriteStartElement ("test-case");
+ writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test);
+ writer.WriteAttributeString ("executed", "True");
+ writer.WriteAttributeString ("success", "False");
+ writer.WriteAttributeString ("time", "0");
+ writer.WriteAttributeString ("asserts", "1");
+ writer.WriteStartElement ("failure");
+ writer.WriteStartElement ("message");
+ writer.WriteCData (DumpPseudoTrace (pd.stdoutFile));
+ writer.WriteEndElement ();
+ writer.WriteStartElement ("stack-trace");
+ writer.WriteCData (DumpPseudoTrace (pd.stderrFile));
+ writer.WriteEndElement ();
+ writer.WriteEndElement ();
+ writer.WriteEndElement ();
+ }
+ // </results>
+ writer.WriteEndElement ();
+ // </test-suite>
+ writer.WriteEndElement ();
+ // </results>
+ writer.WriteEndElement ();
+ // </test-suite>
+ writer.WriteEndElement ();
+ // </results>
+ writer.WriteEndElement ();
+ // </test-suite>
+ writer.WriteEndElement ();
+ // </test-results>
+ writer.WriteEndElement ();
+ writer.WriteEndDocument ();
+ }
+
Console.WriteLine ();
if (timed_out) {
Console.WriteLine (npassed + nfailed);
lock (monitor) {
foreach (Process p in processes) {
- Console.WriteLine (process_data [p].test);
+ ProcessData pd = process_data [p];
+ pd.CloseStreams ();
+ Console.WriteLine (pd.test);
+ p.Kill ();
+ DumpFile (pd.stdoutFile);
+ DumpFile (pd.stderrFile);
}
}
return 1;
Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
if (nfailed > 0) {
Console.WriteLine ("\nFailed tests:\n");
- foreach (string s in failed)
- Console.WriteLine (s);
+ foreach (ProcessData pd in failed) {
+ Console.WriteLine (pd.test);
+ DumpFile (pd.stdoutFile);
+ DumpFile (pd.stderrFile);
+ }
return 1;
} else {
return 0;
}
}
+
+ static void DumpFile (string filename) {
+ if (File.Exists (filename)) {
+ Console.WriteLine ("=============== {0} ===============", filename);
+ Console.WriteLine (File.ReadAllText (filename));
+ Console.WriteLine ("=============== EOF ===============");
+ }
+ }
+
+ static string DumpPseudoTrace (string filename) {
+ if (File.Exists (filename))
+ return File.ReadAllText (filename);
+ else
+ return string.Empty;
+ }
}
projects / mono.git / blobdiff