mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
  10 //
  11 using System;
  12 using System.IO;
  13 using System.Threading;
  14 using System.Text;
  15 using System.Diagnostics;
  16 using System.Collections.Generic;
  17 using System.Globalization;
  18 using System.Xml;
  19 using System.Text.RegularExpressions;
  20
  21 #if !MOBILE_STATIC
  22 using Mono.Unix.Native;
  23 #endif
  24
  25 //
  26 // This is a simple test runner with support for parallel execution
  27 //
  28
  29 public class TestRunner
  30 {
  31         const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
  32         const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
  33
  34         class ProcessData {
  35                 public string test;
  36                 public StringBuilder stdout, stderr;
  37                 public string stdoutName, stderrName;
  38         }
  39
  40         class TestInfo {
  41                 public string test, opt_set;
  42         }
  43
  44         public static int Main (String[] args) {
  45                 // Defaults
  46                 int concurrency = 1;
  47                 int timeout = 2 * 60; // in seconds
  48                 int expectedExitCode = 0;
  49                 string testsuiteName = null;
  50                 string inputFile = null;
  51
  52                 // FIXME: Add support for runtime arguments + env variables
  53
  54                 string disabled_tests = null;
  55                 string runtime = "mono";
  56                 string config = null;
  57                 var opt_sets = new List<string> ();
  58
  59                 // Process options
  60                 int i = 0;
  61                 while (i < args.Length) {
  62                         if (args [i].StartsWith ("-")) {
  63                                 if (args [i] == "-j") {
  64                                         if (i + 1 >= args.Length) {
  65                                                 Console.WriteLine ("Missing argument to -j command line option.");
  66                                                 return 1;
  67                                         }
  68                                         if (args [i + 1] == "a")
  69                                                 concurrency = Environment.ProcessorCount;
  70                                         else
  71                                                 concurrency = Int32.Parse (args [i + 1]);
  72                                         i += 2;
  73                                 } else if (args [i] == "--timeout") {
  74                                         if (i + 1 >= args.Length) {
  75                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  76                                                 return 1;
  77                                         }
  78                                         timeout = Int32.Parse (args [i + 1]);
  79                                         i += 2;
  80                                 } else if (args [i] == "--disabled") {
  81                                         if (i + 1 >= args.Length) {
  82                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  83                                                 return 1;
  84                                         }
  85                                         disabled_tests = args [i + 1];
  86                                         i += 2;
  87                                 } else if (args [i] == "--runtime") {
  88                                         if (i + 1 >= args.Length) {
  89                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  90                                                 return 1;
  91                                         }
  92                                         runtime = args [i + 1];
  93                                         i += 2;
  94                                 } else if (args [i] == "--config") {
  95                                         if (i + 1 >= args.Length) {
  96                                                 Console.WriteLine ("Missing argument to --config command line option.");
  97                                                 return 1;
  98                                         }
  99                                         config = args [i + 1];
 100                                         i += 2;
 101                                 } else if (args [i] == "--opt-sets") {
 102                                         if (i + 1 >= args.Length) {
 103                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 104                                                 return 1;
 105                                         }
 106                                         foreach (var s in args [i + 1].Split ())
 107                                                 opt_sets.Add (s);
 108                                         i += 2;
 109                                 } else if (args [i] == "--expected-exit-code") {
 110                                         if (i + 1 >= args.Length) {
 111                                                 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
 112                                                 return 1;
 113                                         }
 114                                         expectedExitCode = Int32.Parse (args [i + 1]);
 115                                         i += 2;
 116                                 } else if (args [i] == "--testsuite-name") {
 117                                         if (i + 1 >= args.Length) {
 118                                                 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
 119                                                 return 1;
 120                                         }
 121                                         testsuiteName = args [i + 1];
 122                                         i += 2;
 123                                 } else if (args [i] == "--input-file") {
 124                                         if (i + 1 >= args.Length) {
 125                                                 Console.WriteLine ("Missing argument to --input-file command line option.");
 126                                                 return 1;
 127                                         }
 128                                         inputFile = args [i + 1];
 129                                         i += 2;
 130                                 } else {
 131                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 132                                         return 1;
 133                                 }
 134                         } else {
 135                                 break;
 136                         }
 137                 }
 138
 139                 if (String.IsNullOrEmpty (testsuiteName)) {
 140                         Console.WriteLine ("Missing the required --testsuite-name command line option.");
 141                         return 1;
 142                 }
 143
 144                 var disabled = new Dictionary <string, string> ();
 145
 146                 if (disabled_tests != null) {
 147                         foreach (string test in disabled_tests.Split ())
 148                                 disabled [test] = test;
 149                 }
 150
 151                 var tests = new List<string> ();
 152
 153                 if (!String.IsNullOrEmpty (inputFile)) {
 154                         tests.AddRange (File.ReadAllLines (inputFile));
 155                 } else {
 156                         // The remaining arguments are the tests
 157                         for (int j = i; j < args.Length; ++j)
 158                                 if (!disabled.ContainsKey (args [j]))
 159                                         tests.Add (args [j]);
 160                 }
 161
 162                 var passed = new List<ProcessData> ();
 163                 var failed = new List<ProcessData> ();
 164                 var timedout = new List<ProcessData> ();
 165
 166                 object monitor = new object ();
 167
 168                 Console.WriteLine ("Running tests: ");
 169
 170                 var test_info = new Queue<TestInfo> ();
 171                 if (opt_sets.Count == 0) {
 172                         foreach (string s in tests)
 173                                 test_info.Enqueue (new TestInfo { test = s });
 174                 } else {
 175                         foreach (string opt in opt_sets) {
 176                                 foreach (string s in tests)
 177                                         test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
 178                         }
 179                 }
 180
 181                 /* compute the max length of test names, to have an optimal output width */
 182                 int output_width = -1;
 183                 foreach (TestInfo ti in test_info) {
 184                         if (ti.test.Length > output_width)
 185                                 output_width = Math.Min (120, ti.test.Length);
 186                 }
 187
 188                 List<Thread> threads = new List<Thread> (concurrency);
 189
 190                 DateTime test_start_time = DateTime.UtcNow;
 191
 192                 for (int j = 0; j < concurrency; ++j) {
 193                         Thread thread = new Thread (() => {
 194                                 while (true) {
 195                                         TestInfo ti;
 196
 197                                         lock (monitor) {
 198                                                 if (test_info.Count == 0)
 199                                                         break;
 200                                                 ti = test_info.Dequeue ();
 201                                         }
 202
 203                                         var output = new StringWriter ();
 204
 205                                         string test = ti.test;
 206                                         string opt_set = ti.opt_set;
 207
 208                                         output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
 209
 210                                         /* Spawn a new process */
 211                                         string process_args;
 212                                         if (opt_set == null)
 213                                                 process_args = test;
 214                                         else
 215                                                 process_args = "-O=" + opt_set + " " + test;
 216                                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 217                                         info.UseShellExecute = false;
 218                                         info.RedirectStandardOutput = true;
 219                                         info.RedirectStandardError = true;
 220                                         info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 221                                         if (config != null)
 222                                                 info.EnvironmentVariables["MONO_CONFIG"] = config;
 223                                         Process p = new Process ();
 224                                         p.StartInfo = info;
 225
 226                                         ProcessData data = new ProcessData ();
 227                                         data.test = test;
 228
 229                                         string log_prefix = "";
 230                                         if (opt_set != null)
 231                                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 232
 233                                         data.stdoutName = test + log_prefix + ".stdout";
 234                                         data.stdout = new StringBuilder ();
 235
 236                                         data.stderrName = test + log_prefix + ".stderr";
 237                                         data.stderr = new StringBuilder ();
 238
 239                                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 240                                                 if (e.Data != null) {
 241                                                         data.stdout.AppendLine (e.Data);
 242                                                 }
 243                                         };
 244
 245                                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 246                                                 if (e.Data != null) {
 247                                                         data.stderr.AppendLine (e.Data);
 248                                                 }
 249                                         };
 250
 251                                         var start = DateTime.UtcNow;
 252
 253                                         p.Start ();
 254
 255                                         p.BeginOutputReadLine ();
 256                                         p.BeginErrorReadLine ();
 257
 258                                         if (!p.WaitForExit (timeout * 1000)) {
 259                                                 lock (monitor) {
 260                                                         timedout.Add (data);
 261                                                 }
 262
 263 #if !MOBILE_STATIC
 264                                                 // Force the process to print a thread dump
 265                                                 try {
 266                                                         Syscall.kill (p.Id, Signum.SIGQUIT);
 267                                                         Thread.Sleep (1000);
 268                                                 } catch {
 269                                                 }
 270 #endif
 271
 272                                                 output.Write ($"timed out ({timeout}s)");
 273
 274                                                 try {
 275                                                         p.Kill ();
 276                                                 } catch {
 277                                                 }
 278                                         } else if (p.ExitCode != expectedExitCode) {
 279                                                 var end = DateTime.UtcNow;
 280
 281                                                 lock (monitor) {
 282                                                         failed.Add (data);
 283                                                 }
 284
 285                                                 output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
 286                                         } else {
 287                                                 var end = DateTime.UtcNow;
 288
 289                                                 lock (monitor) {
 290                                                         passed.Add (data);
 291                                                 }
 292
 293                                                 output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
 294                                         }
 295
 296                                         p.Close ();
 297
 298                                         lock (monitor) {
 299                                                 Console.WriteLine (output.ToString ());
 300                                         }
 301                                 }
 302                         });
 303
 304                         thread.Start ();
 305
 306                         threads.Add (thread);
 307                 }
 308
 309                 for (int j = 0; j < threads.Count; ++j)
 310                         threads [j].Join ();
 311
 312                 TimeSpan test_time = DateTime.UtcNow - test_start_time;
 313
 314                 int npassed = passed.Count;
 315                 int nfailed = failed.Count;
 316                 int ntimedout = timedout.Count;
 317
 318                 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
 319                 xmlWriterSettings.NewLineOnAttributes = true;
 320                 xmlWriterSettings.Indent = true;
 321                 using (XmlWriter writer = XmlWriter.Create (String.Format ("TestResult-{0}.xml", testsuiteName), xmlWriterSettings)) {
 322                         // <?xml version="1.0" encoding="utf-8" standalone="no"?>
 323                         writer.WriteStartDocument ();
 324                         // <!--This file represents the results of running a test suite-->
 325                         writer.WriteComment ("This file represents the results of running a test suite");
 326                         // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
 327                         writer.WriteStartElement ("test-results");
 328                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 329                         writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
 330                         writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
 331                         writer.WriteAttributeString ("not-run", "0");
 332                         writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 333                         writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 334                         //   <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
 335                         writer.WriteStartElement ("environment");
 336                         writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
 337                         writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
 338                         writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
 339                         writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
 340                         writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
 341                         writer.WriteAttributeString ("machine-name", Environment.MachineName );
 342                         writer.WriteAttributeString ("user", Environment.UserName );
 343                         writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
 344                         writer.WriteEndElement ();
 345                         //   <culture-info current-culture="en-GB" current-uiculture="en-GB" />
 346                         writer.WriteStartElement ("culture-info");
 347                         writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
 348                         writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
 349                         writer.WriteEndElement ();
 350                         //   <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
 351                         writer.WriteStartElement ("test-suite");
 352                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 353                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 354                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 355                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 356                         //     <results>
 357                         writer.WriteStartElement ("results");
 358                         //       <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 359                         writer.WriteStartElement ("test-suite");
 360                         writer.WriteAttributeString ("name","MonoTests");
 361                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 362                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 363                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 364                         //         <results>
 365                         writer.WriteStartElement ("results");
 366                         //           <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 367                         writer.WriteStartElement ("test-suite");
 368                         writer.WriteAttributeString ("name", testsuiteName);
 369                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 370                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 371                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 372                         //             <results>
 373                         writer.WriteStartElement ("results");
 374                         // Dump all passing tests first
 375                         foreach (ProcessData pd in passed) {
 376                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 377                                 writer.WriteStartElement ("test-case");
 378                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 379                                 writer.WriteAttributeString ("executed", "True");
 380                                 writer.WriteAttributeString ("success", "True");
 381                                 writer.WriteAttributeString ("time", "0");
 382                                 writer.WriteAttributeString ("asserts", "0");
 383                                 writer.WriteEndElement ();
 384                         }
 385                         // Now dump all failing tests
 386                         foreach (ProcessData pd in failed) {
 387                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 388                                 writer.WriteStartElement ("test-case");
 389                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 390                                 writer.WriteAttributeString ("executed", "True");
 391                                 writer.WriteAttributeString ("success", "False");
 392                                 writer.WriteAttributeString ("time", "0");
 393                                 writer.WriteAttributeString ("asserts", "1");
 394                                 writer.WriteStartElement ("failure");
 395                                 writer.WriteStartElement ("message");
 396                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 397                                 writer.WriteEndElement ();
 398                                 writer.WriteStartElement ("stack-trace");
 399                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 400                                 writer.WriteEndElement ();
 401                                 writer.WriteEndElement ();
 402                                 writer.WriteEndElement ();
 403                         }
 404                         // Then dump all timing out tests
 405                         foreach (ProcessData pd in timedout) {
 406                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 407                                 writer.WriteStartElement ("test-case");
 408                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
 409                                 writer.WriteAttributeString ("executed", "True");
 410                                 writer.WriteAttributeString ("success", "False");
 411                                 writer.WriteAttributeString ("time", "0");
 412                                 writer.WriteAttributeString ("asserts", "1");
 413                                 writer.WriteStartElement ("failure");
 414                                 writer.WriteStartElement ("message");
 415                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 416                                 writer.WriteEndElement ();
 417                                 writer.WriteStartElement ("stack-trace");
 418                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 419                                 writer.WriteEndElement ();
 420                                 writer.WriteEndElement ();
 421                                 writer.WriteEndElement ();
 422                         }
 423                         //             </results>
 424                         writer.WriteEndElement ();
 425                         //           </test-suite>
 426                         writer.WriteEndElement ();
 427                         //         </results>
 428                         writer.WriteEndElement ();
 429                         //       </test-suite>
 430                         writer.WriteEndElement ();
 431                         //     </results>
 432                         writer.WriteEndElement ();
 433                         //   </test-suite>
 434                         writer.WriteEndElement ();
 435                         // </test-results>
 436                         writer.WriteEndElement ();
 437                         writer.WriteEndDocument ();
 438                 }
 439
 440                 Console.WriteLine ();
 441                 Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
 442                 Console.WriteLine ();
 443                 Console.WriteLine ("{0,4} test(s) passed", npassed);
 444                 Console.WriteLine ("{0,4} test(s) failed", nfailed);
 445                 Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
 446
 447                 if (nfailed > 0) {
 448                         Console.WriteLine ();
 449                         Console.WriteLine ("Failed test(s):");
 450                         foreach (ProcessData pd in failed) {
 451                                 Console.WriteLine ();
 452                                 Console.WriteLine (pd.test);
 453                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 454                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 455                         }
 456                 }
 457
 458                 if (ntimedout > 0) {
 459                         Console.WriteLine ();
 460                         Console.WriteLine ("Timed out test(s):");
 461                         foreach (ProcessData pd in timedout) {
 462                                 Console.WriteLine ();
 463                                 Console.WriteLine (pd.test);
 464                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 465                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 466                         }
 467                 }
 468
 469                 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
 470         }
 471
 472         static void DumpFile (string filename, string text) {
 473                 Console.WriteLine ("=============== {0} ===============", filename);
 474                 Console.WriteLine (text);
 475                 Console.WriteLine ("=============== EOF ===============");
 476         }
 477
 478         static string FilterInvalidXmlChars (string text) {
 479                 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
 480                 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]      /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
 481                 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
 482         }
 483 }