From 84a8428de1d2bf032afa25cdac81b74bbbe2a28e Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 11 Feb 2015 12:48:45 +0000 Subject: [PATCH] Add NUnit XML emission to the managed runtime test runner. This makes it easy to plug failures from the runtime tests into a visualisation, such as Jenkins' jUnit display system. --- mono/tests/test-runner.cs | 117 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/mono/tests/test-runner.cs b/mono/tests/test-runner.cs index dda3625061e..6dc6745d53e 100644 --- a/mono/tests/test-runner.cs +++ b/mono/tests/test-runner.cs @@ -30,6 +30,8 @@ using System.IO; using System.Threading; using System.Diagnostics; using System.Collections.Generic; +using System.Globalization; +using System.Xml; // // This is a simple test runner with support for parallel execution @@ -63,6 +65,8 @@ public class TestRunner int concurrency = 1; int timeout = 2 * 60; // in seconds + DateTime test_start_time = DateTime.UtcNow; + // FIXME: Add support for runtime arguments + env variables string disabled_tests = null; @@ -138,6 +142,7 @@ public class TestRunner int nfailed = 0; var processes = new List (); + var passed = new List (); var failed = new List (); var process_data = new Dictionary (); @@ -212,6 +217,7 @@ public class TestRunner Console.WriteLine ("passed."); else Console.Write ("."); + passed.Add(process_data [dead]); npassed ++; } else { if (concurrency == 1) @@ -310,6 +316,110 @@ public class TestRunner } } + TimeSpan test_time = DateTime.UtcNow - test_start_time; + XmlWriterSettings xmlWriterSettings = new XmlWriterSettings (); + xmlWriterSettings.NewLineOnAttributes = true; + xmlWriterSettings.Indent = true; + using (XmlWriter writer = XmlWriter.Create ("TestResults_runtime.xml", xmlWriterSettings)) { + // + writer.WriteStartDocument (); + // + writer.WriteComment ("This file represents the results of running a test suite"); + // + writer.WriteStartElement ("test-results"); + writer.WriteAttributeString ("name", "runtime-tests.dummy"); + writer.WriteAttributeString ("total", (npassed + nfailed).ToString()); + writer.WriteAttributeString ("failures", nfailed.ToString()); + writer.WriteAttributeString ("not-run", "0"); + writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd")); + writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss")); + // + writer.WriteStartElement ("environment"); + writer.WriteAttributeString ("nunit-version", "2.4.8.0" ); + writer.WriteAttributeString ("clr-version", Environment.Version.ToString() ); + writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() ); + writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() ); + writer.WriteAttributeString ("cwd", Environment.CurrentDirectory ); + writer.WriteAttributeString ("machine-name", Environment.MachineName ); + writer.WriteAttributeString ("user", Environment.UserName ); + writer.WriteAttributeString ("user-domain", Environment.UserDomainName ); + writer.WriteEndElement (); + // + writer.WriteStartElement ("culture-info"); + writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name ); + writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name ); + writer.WriteEndElement (); + // + writer.WriteStartElement ("test-suite"); + writer.WriteAttributeString ("name","runtime-tests.dummy"); + writer.WriteAttributeString ("success", (nfailed == 0).ToString()); + writer.WriteAttributeString ("time", test_time.Seconds.ToString()); + writer.WriteAttributeString ("asserts", nfailed.ToString()); + // + writer.WriteStartElement ("results"); + // + writer.WriteStartElement ("test-suite"); + writer.WriteAttributeString ("name","MonoTests"); + writer.WriteAttributeString ("success", (nfailed == 0).ToString()); + writer.WriteAttributeString ("time", test_time.Seconds.ToString()); + writer.WriteAttributeString ("asserts", nfailed.ToString()); + // + writer.WriteStartElement ("results"); + // + writer.WriteStartElement ("test-suite"); + writer.WriteAttributeString ("name","runtime"); + writer.WriteAttributeString ("success", (nfailed == 0).ToString()); + writer.WriteAttributeString ("time", test_time.Seconds.ToString()); + writer.WriteAttributeString ("asserts", nfailed.ToString()); + // + writer.WriteStartElement ("results"); + // Dump all passing tests first + foreach (ProcessData pd in passed) { + // + writer.WriteStartElement ("test-case"); + writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test); + writer.WriteAttributeString ("executed", "True"); + writer.WriteAttributeString ("success", "True"); + writer.WriteAttributeString ("time", "0"); + writer.WriteAttributeString ("asserts", "0"); + writer.WriteEndElement (); + } + // Now dump all failing tests + foreach (ProcessData pd in failed) { + // + writer.WriteStartElement ("test-case"); + writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test); + writer.WriteAttributeString ("executed", "True"); + writer.WriteAttributeString ("success", "False"); + writer.WriteAttributeString ("time", "0"); + writer.WriteAttributeString ("asserts", "1"); + writer.WriteStartElement ("failure"); + writer.WriteStartElement ("message"); + writer.WriteCData (DumpPseudoTrace (pd.stdoutFile)); + writer.WriteEndElement (); + writer.WriteStartElement ("stack-trace"); + writer.WriteCData (DumpPseudoTrace (pd.stderrFile)); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteEndElement (); + } + // + writer.WriteEndElement (); + // + writer.WriteEndElement (); + // + writer.WriteEndElement (); + // + writer.WriteEndElement (); + // + writer.WriteEndElement (); + // + writer.WriteEndElement (); + // + writer.WriteEndElement (); + writer.WriteEndDocument (); + } + Console.WriteLine (); if (timed_out) { @@ -349,4 +459,11 @@ public class TestRunner Console.WriteLine ("=============== EOF ==============="); } } + + static string DumpPseudoTrace (string filename) { + if (File.Exists (filename)) + return File.ReadAllText (filename); + else + return string.Empty; + } } -- 2.25.1