mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
  10 //
  11 using System;
  12 using System.IO;
  13 using System.Threading;
  14 using System.Diagnostics;
  15 using System.Collections.Generic;
  16 using System.Globalization;
  17 using System.Xml;
  18 using System.Text;
  19 using System.Text.RegularExpressions;
  20
  21 #if !FULL_AOT_DESKTOP && !MOBILE
  22 using Mono.Unix.Native;
  23 #endif
  24
  25 //
  26 // This is a simple test runner with support for parallel execution
  27 //
  28
  29 public class TestRunner
  30 {
  31         const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
  32         const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
  33         const string MONO_PATH = "MONO_PATH";
  34
  35         class ProcessData {
  36                 public string test;
  37                 public StringBuilder stdout, stderr;
  38                 public object stdoutLock = new object (), stderrLock = new object ();
  39                 public string stdoutName, stderrName;
  40         }
  41
  42         class TestInfo {
  43                 public string test, opt_set;
  44         }
  45
  46         public static int Main (String[] args) {
  47                 // Defaults
  48                 int concurrency = 1;
  49                 int timeout = 2 * 60; // in seconds
  50                 int expectedExitCode = 0;
  51                 bool verbose = false;
  52                 string testsuiteName = null;
  53                 string inputFile = null;
  54
  55                 string disabled_tests = null;
  56                 string runtime = "mono";
  57                 string config = null;
  58                 string mono_path = null;
  59                 var opt_sets = new List<string> ();
  60
  61                 string aot_run_flags = null;
  62                 string aot_build_flags = null;
  63
  64                 // Process options
  65                 int i = 0;
  66                 while (i < args.Length) {
  67                         if (args [i].StartsWith ("-")) {
  68                                 if (args [i] == "-j") {
  69                                         if (i + 1 >= args.Length) {
  70                                                 Console.WriteLine ("Missing argument to -j command line option.");
  71                                                 return 1;
  72                                         }
  73                                         if (args [i + 1] == "a")
  74                                                 concurrency = Environment.ProcessorCount;
  75                                         else
  76                                                 concurrency = Int32.Parse (args [i + 1]);
  77                                         i += 2;
  78                                 } else if (args [i] == "--timeout") {
  79                                         if (i + 1 >= args.Length) {
  80                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  81                                                 return 1;
  82                                         }
  83                                         timeout = Int32.Parse (args [i + 1]);
  84                                         i += 2;
  85                                 } else if (args [i] == "--disabled") {
  86                                         if (i + 1 >= args.Length) {
  87                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  88                                                 return 1;
  89                                         }
  90                                         disabled_tests = args [i + 1];
  91                                         i += 2;
  92                                 } else if (args [i] == "--runtime") {
  93                                         if (i + 1 >= args.Length) {
  94                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  95                                                 return 1;
  96                                         }
  97                                         runtime = args [i + 1];
  98                                         i += 2;
  99                                 } else if (args [i] == "--config") {
 100                                         if (i + 1 >= args.Length) {
 101                                                 Console.WriteLine ("Missing argument to --config command line option.");
 102                                                 return 1;
 103                                         }
 104                                         config = args [i + 1];
 105                                         i += 2;
 106                                 } else if (args [i] == "--opt-sets") {
 107                                         if (i + 1 >= args.Length) {
 108                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 109                                                 return 1;
 110                                         }
 111                                         foreach (var s in args [i + 1].Split ())
 112                                                 opt_sets.Add (s);
 113                                         i += 2;
 114                                 } else if (args [i] == "--expected-exit-code") {
 115                                         if (i + 1 >= args.Length) {
 116                                                 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
 117                                                 return 1;
 118                                         }
 119                                         expectedExitCode = Int32.Parse (args [i + 1]);
 120                                         i += 2;
 121                                 } else if (args [i] == "--testsuite-name") {
 122                                         if (i + 1 >= args.Length) {
 123                                                 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
 124                                                 return 1;
 125                                         }
 126                                         testsuiteName = args [i + 1];
 127                                         i += 2;
 128                                 } else if (args [i] == "--input-file") {
 129                                         if (i + 1 >= args.Length) {
 130                                                 Console.WriteLine ("Missing argument to --input-file command line option.");
 131                                                 return 1;
 132                                         }
 133                                         inputFile = args [i + 1];
 134                                         i += 2;
 135                                 } else if (args [i] == "--runtime") {
 136                                         if (i + 1 >= args.Length) {
 137                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
 138                                                 return 1;
 139                                         }
 140                                         runtime = args [i + 1];
 141                                         i += 2;
 142                                 } else if (args [i] == "--mono-path") {
 143                                         if (i + 1 >= args.Length) {
 144                                                 Console.WriteLine ("Missing argument to --mono-path command line option.");
 145                                                 return 1;
 146                                         }
 147                                         mono_path = args [i + 1].Substring(0, args [i + 1].Length);
 148
 149                                         i += 2;
 150                                 } else if (args [i] == "--aot-run-flags") {
 151                                         if (i + 1 >= args.Length) {
 152                                                 Console.WriteLine ("Missing argument to --aot-run-flags command line option.");
 153                                                 return 1;
 154                                         }
 155                                         aot_run_flags = args [i + 1].Substring(0, args [i + 1].Length);
 156                                         i += 2;
 157                                 } else if (args [i] == "--aot-build-flags") {
 158                                         if (i + 1 >= args.Length) {
 159                                                 Console.WriteLine ("Missing argument to --aot-build-flags command line option.");
 160                                                 return 1;
 161                                         }
 162                                         aot_build_flags = args [i + 1].Substring(0, args [i + 1].Length);
 163                                         i += 2;
 164                                 } else if (args [i] == "--verbose") {
 165                                         verbose = true;
 166                                         i ++;
 167                                 } else {
 168                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 169                                         return 1;
 170                                 }
 171                         } else {
 172                                 break;
 173                         }
 174                 }
 175
 176                 if (String.IsNullOrEmpty (testsuiteName)) {
 177                         Console.WriteLine ("Missing the required --testsuite-name command line option.");
 178                         return 1;
 179                 }
 180
 181                 var disabled = new Dictionary <string, string> ();
 182
 183                 if (disabled_tests != null) {
 184                         foreach (string test in disabled_tests.Split ())
 185                                 disabled [test] = test;
 186                 }
 187
 188                 var tests = new List<string> ();
 189
 190                 if (!String.IsNullOrEmpty (inputFile)) {
 191                         tests.AddRange (File.ReadAllLines (inputFile));
 192                 } else {
 193                         // The remaining arguments are the tests
 194                         for (int j = i; j < args.Length; ++j)
 195                                 if (!disabled.ContainsKey (args [j]))
 196                                         tests.Add (args [j]);
 197                 }
 198
 199                 var passed = new List<ProcessData> ();
 200                 var failed = new List<ProcessData> ();
 201                 var timedout = new List<ProcessData> ();
 202
 203                 object monitor = new object ();
 204
 205                 Console.WriteLine ("Running tests: ");
 206
 207                 var test_info = new Queue<TestInfo> ();
 208                 if (opt_sets.Count == 0) {
 209                         foreach (string s in tests)
 210                                 test_info.Enqueue (new TestInfo { test = s });
 211                 } else {
 212                         foreach (string opt in opt_sets) {
 213                                 foreach (string s in tests)
 214                                         test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
 215                         }
 216                 }
 217
 218                 /* compute the max length of test names, to have an optimal output width */
 219                 int output_width = -1;
 220                 foreach (TestInfo ti in test_info) {
 221                         if (ti.test.Length > output_width)
 222                                 output_width = Math.Min (120, ti.test.Length);
 223                 }
 224
 225                 if (aot_build_flags != null)  {
 226                         Console.WriteLine("AOT compiling tests");
 227
 228                         object aot_monitor = new object ();
 229                         var aot_queue = new Queue<String> (tests);
 230
 231                         List<Thread> build_threads = new List<Thread> (concurrency);
 232
 233                         for (int j = 0; j < concurrency; ++j) {
 234                                 Thread thread = new Thread (() => {
 235                                         while (true) {
 236                                                 String test_name;
 237
 238                                                 lock (aot_monitor) {
 239                                                         if (aot_queue.Count == 0)
 240                                                                 break;
 241                                                         test_name = aot_queue.Dequeue ();
 242                                                 }
 243
 244                                                 string test_bitcode_output = test_name + "_bitcode_tmp";
 245                                                 string test_bitcode_arg = ",temp-path=" + test_bitcode_output;
 246                                                 string aot_args = aot_build_flags + test_bitcode_arg + " " + test_name;
 247
 248                                                 Directory.CreateDirectory(test_bitcode_output);
 249
 250                                                 ProcessStartInfo job = new ProcessStartInfo (runtime, aot_args);
 251                                                 job.UseShellExecute = false;
 252                                                 job.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 253                                                 job.EnvironmentVariables[MONO_PATH] = mono_path;
 254                                                 Process compiler = new Process ();
 255                                                 compiler.StartInfo = job;
 256
 257                                                 compiler.Start ();
 258
 259                                                 if (!compiler.WaitForExit (timeout * 1000)) {
 260                                                         try {
 261                                                                 compiler.Kill ();
 262                                                         } catch {
 263                                                         }
 264                                                         throw new Exception(String.Format("Timeout AOT compiling tests, output in {0}", test_bitcode_output));
 265                                                 } else if (compiler.ExitCode != 0) {
 266                                                         throw new Exception(String.Format("Error AOT compiling tests, output in {0}", test_bitcode_output));
 267                                                 } else {
 268                                                         Directory.Delete (test_bitcode_output, true);
 269                                                 }
 270                                         }
 271                                 });
 272
 273                                 thread.Start ();
 274
 275                                 build_threads.Add (thread);
 276                         }
 277
 278                         for (int j = 0; j < build_threads.Count; ++j)
 279                                 build_threads [j].Join ();
 280
 281                         Console.WriteLine("Done compiling");
 282                 }
 283
 284                 List<Thread> threads = new List<Thread> (concurrency);
 285
 286                 DateTime test_start_time = DateTime.UtcNow;
 287
 288                 for (int j = 0; j < concurrency; ++j) {
 289                         Thread thread = new Thread (() => {
 290                                 while (true) {
 291                                         TestInfo ti;
 292
 293                                         lock (monitor) {
 294                                                 if (test_info.Count == 0)
 295                                                         break;
 296                                                 ti = test_info.Dequeue ();
 297                                         }
 298
 299                                         var output = new StringWriter ();
 300
 301                                         string test = ti.test;
 302                                         string opt_set = ti.opt_set;
 303
 304                                         if (verbose) {
 305                                                 output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
 306                                         } else {
 307                                                 Console.Write (".");
 308                                         }
 309
 310                                         string test_invoke;
 311
 312                                         if (aot_run_flags != null)
 313                                                 test_invoke = aot_run_flags + " " + test;
 314                                         else
 315                                                 test_invoke = test;
 316
 317                                         /* Spawn a new process */
 318                                         string process_args;
 319                                         if (opt_set == null)
 320                                                 process_args = test_invoke;
 321                                         else
 322                                                 process_args = "-O=" + opt_set + " " + test_invoke;
 323
 324                                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 325                                         info.UseShellExecute = false;
 326                                         info.RedirectStandardOutput = true;
 327                                         info.RedirectStandardError = true;
 328                                         info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 329                                         if (config != null)
 330                                                 info.EnvironmentVariables["MONO_CONFIG"] = config;
 331                                         if (mono_path != null)
 332                                                 info.EnvironmentVariables[MONO_PATH] = mono_path;
 333                                         Process p = new Process ();
 334                                         p.StartInfo = info;
 335
 336                                         ProcessData data = new ProcessData ();
 337                                         data.test = test;
 338
 339                                         string log_prefix = "";
 340                                         if (opt_set != null)
 341                                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 342
 343                                         data.stdoutName = test + log_prefix + ".stdout";
 344                                         data.stdout = new StringBuilder ();
 345
 346                                         data.stderrName = test + log_prefix + ".stderr";
 347                                         data.stderr = new StringBuilder ();
 348
 349                                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 350                                                 lock (data.stdoutLock) {
 351                                                         if (e.Data != null)
 352                                                                 data.stdout.AppendLine (e.Data);
 353                                                 }
 354                                         };
 355
 356                                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 357                                                 lock (data.stderrLock) {
 358                                                         if (e.Data != null)
 359                                                                 data.stderr.AppendLine (e.Data);
 360                                                 }
 361                                         };
 362
 363                                         var start = DateTime.UtcNow;
 364
 365                                         p.Start ();
 366
 367                                         p.BeginOutputReadLine ();
 368                                         p.BeginErrorReadLine ();
 369
 370                                         if (!p.WaitForExit (timeout * 1000)) {
 371                                                 lock (monitor) {
 372                                                         timedout.Add (data);
 373                                                 }
 374
 375                                                 // Force the process to print a thread dump
 376                                                 TryThreadDump (p.Id, data);
 377
 378                                                 if (verbose) {
 379                                                         output.Write ($"timed out ({timeout}s)");
 380                                                 }
 381
 382                                                 try {
 383                                                         p.Kill ();
 384                                                 } catch {
 385                                                 }
 386                                         } else if (p.ExitCode != expectedExitCode) {
 387                                                 var end = DateTime.UtcNow;
 388
 389                                                 lock (monitor) {
 390                                                         failed.Add (data);
 391                                                 }
 392
 393                                                 if (verbose)
 394                                                         output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
 395                                         } else {
 396                                                 var end = DateTime.UtcNow;
 397
 398                                                 lock (monitor) {
 399                                                         passed.Add (data);
 400                                                 }
 401
 402                                                 if (verbose)
 403                                                         output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
 404                                         }
 405
 406                                         p.Close ();
 407
 408                                         lock (monitor) {
 409                                                 if (verbose)
 410                                                         Console.WriteLine (output.ToString ());
 411                                         }
 412                                 }
 413                         });
 414
 415                         thread.Start ();
 416
 417                         threads.Add (thread);
 418                 }
 419
 420                 for (int j = 0; j < threads.Count; ++j)
 421                         threads [j].Join ();
 422
 423                 TimeSpan test_time = DateTime.UtcNow - test_start_time;
 424
 425                 int npassed = passed.Count;
 426                 int nfailed = failed.Count;
 427                 int ntimedout = timedout.Count;
 428
 429                 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
 430                 xmlWriterSettings.NewLineOnAttributes = true;
 431                 xmlWriterSettings.Indent = true;
 432
 433                 string xmlPath = String.Format ("TestResult-{0}.xml", testsuiteName);
 434                 using (XmlWriter writer = XmlWriter.Create (xmlPath, xmlWriterSettings)) {
 435                         // <?xml version="1.0" encoding="utf-8" standalone="no"?>
 436                         writer.WriteStartDocument ();
 437                         // <!--This file represents the results of running a test suite-->
 438                         writer.WriteComment ("This file represents the results of running a test suite");
 439                         // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
 440                         writer.WriteStartElement ("test-results");
 441                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 442                         writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
 443                         writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
 444                         writer.WriteAttributeString ("not-run", "0");
 445                         writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 446                         writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 447                         //   <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
 448                         writer.WriteStartElement ("environment");
 449                         writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
 450                         writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
 451                         writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
 452                         writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
 453                         writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
 454                         writer.WriteAttributeString ("machine-name", Environment.MachineName );
 455                         writer.WriteAttributeString ("user", Environment.UserName );
 456                         writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
 457                         writer.WriteEndElement ();
 458                         //   <culture-info current-culture="en-GB" current-uiculture="en-GB" />
 459                         writer.WriteStartElement ("culture-info");
 460                         writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
 461                         writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
 462                         writer.WriteEndElement ();
 463                         //   <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
 464                         writer.WriteStartElement ("test-suite");
 465                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 466                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 467                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 468                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 469                         //     <results>
 470                         writer.WriteStartElement ("results");
 471                         //       <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 472                         writer.WriteStartElement ("test-suite");
 473                         writer.WriteAttributeString ("name","MonoTests");
 474                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 475                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 476                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 477                         //         <results>
 478                         writer.WriteStartElement ("results");
 479                         //           <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 480                         writer.WriteStartElement ("test-suite");
 481                         writer.WriteAttributeString ("name", testsuiteName);
 482                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 483                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 484                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 485                         //             <results>
 486                         writer.WriteStartElement ("results");
 487                         // Dump all passing tests first
 488                         foreach (ProcessData pd in passed) {
 489                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 490                                 writer.WriteStartElement ("test-case");
 491                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 492                                 writer.WriteAttributeString ("executed", "True");
 493                                 writer.WriteAttributeString ("success", "True");
 494                                 writer.WriteAttributeString ("time", "0");
 495                                 writer.WriteAttributeString ("asserts", "0");
 496                                 writer.WriteEndElement ();
 497                         }
 498                         // Now dump all failing tests
 499                         foreach (ProcessData pd in failed) {
 500                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 501                                 writer.WriteStartElement ("test-case");
 502                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 503                                 writer.WriteAttributeString ("executed", "True");
 504                                 writer.WriteAttributeString ("success", "False");
 505                                 writer.WriteAttributeString ("time", "0");
 506                                 writer.WriteAttributeString ("asserts", "1");
 507                                 writer.WriteStartElement ("failure");
 508                                 writer.WriteStartElement ("message");
 509                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 510                                 writer.WriteEndElement ();
 511                                 writer.WriteStartElement ("stack-trace");
 512                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 513                                 writer.WriteEndElement ();
 514                                 writer.WriteEndElement ();
 515                                 writer.WriteEndElement ();
 516                         }
 517                         // Then dump all timing out tests
 518                         foreach (ProcessData pd in timedout) {
 519                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 520                                 writer.WriteStartElement ("test-case");
 521                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
 522                                 writer.WriteAttributeString ("executed", "True");
 523                                 writer.WriteAttributeString ("success", "False");
 524                                 writer.WriteAttributeString ("time", "0");
 525                                 writer.WriteAttributeString ("asserts", "1");
 526                                 writer.WriteStartElement ("failure");
 527                                 writer.WriteStartElement ("message");
 528                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 529                                 writer.WriteEndElement ();
 530                                 writer.WriteStartElement ("stack-trace");
 531                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 532                                 writer.WriteEndElement ();
 533                                 writer.WriteEndElement ();
 534                                 writer.WriteEndElement ();
 535                         }
 536                         //             </results>
 537                         writer.WriteEndElement ();
 538                         //           </test-suite>
 539                         writer.WriteEndElement ();
 540                         //         </results>
 541                         writer.WriteEndElement ();
 542                         //       </test-suite>
 543                         writer.WriteEndElement ();
 544                         //     </results>
 545                         writer.WriteEndElement ();
 546                         //   </test-suite>
 547                         writer.WriteEndElement ();
 548                         // </test-results>
 549                         writer.WriteEndElement ();
 550                         writer.WriteEndDocument ();
 551
 552                         string babysitterXmlList = Environment.GetEnvironmentVariable("MONO_BABYSITTER_NUNIT_XML_LIST_FILE");
 553                         if (!String.IsNullOrEmpty(babysitterXmlList)) {
 554                                 try {
 555                                         string fullXmlPath = Path.GetFullPath(xmlPath);
 556                                         File.AppendAllText(babysitterXmlList, fullXmlPath + Environment.NewLine);
 557                                 } catch (Exception e) {
 558                                         Console.WriteLine("Attempted to record XML path to file {0} but failed.", babysitterXmlList);
 559                                 }
 560                         }
 561                 }
 562
 563                 if (verbose) {
 564                         Console.WriteLine ();
 565                         Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
 566                         Console.WriteLine ();
 567                         Console.WriteLine ("{0,4} test(s) passed", npassed);
 568                         Console.WriteLine ("{0,4} test(s) failed", nfailed);
 569                         Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
 570                 } else {
 571                         Console.WriteLine ();
 572                         Console.WriteLine (String.Format ("{0} test(s) passed, {1} test(s) did not pass.", npassed, nfailed));
 573                 }
 574
 575                 if (nfailed > 0) {
 576                         Console.WriteLine ();
 577                         Console.WriteLine ("Failed test(s):");
 578                         foreach (ProcessData pd in failed) {
 579                                 Console.WriteLine ();
 580                                 Console.WriteLine (pd.test);
 581                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 582                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 583                         }
 584                 }
 585
 586                 if (ntimedout > 0) {
 587                         Console.WriteLine ();
 588                         Console.WriteLine ("Timed out test(s):");
 589                         foreach (ProcessData pd in timedout) {
 590                                 Console.WriteLine ();
 591                                 Console.WriteLine (pd.test);
 592                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 593                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 594                         }
 595                 }
 596
 597                 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
 598         }
 599
 600         static void DumpFile (string filename, string text) {
 601                 Console.WriteLine ("=============== {0} ===============", filename);
 602                 Console.WriteLine (text);
 603                 Console.WriteLine ("=============== EOF ===============");
 604         }
 605
 606         static string FilterInvalidXmlChars (string text) {
 607                 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
 608                 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]      /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
 609                 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
 610         }
 611
 612         static void TryThreadDump (int pid, ProcessData data)
 613         {
 614                 try {
 615                         TryGDB (pid, data);
 616                         return;
 617                 } catch {
 618                 }
 619
 620 #if !FULL_AOT_DESKTOP && !MOBILE
 621                 /* LLDB cannot produce managed stacktraces for all the threads */
 622                 try {
 623                         Syscall.kill (pid, Signum.SIGQUIT);
 624                         Thread.Sleep (1000);
 625                 } catch {
 626                 }
 627 #endif
 628
 629                 try {
 630                         TryLLDB (pid, data);
 631                         return;
 632                 } catch {
 633                 }
 634         }
 635
 636         static void TryLLDB (int pid, ProcessData data)
 637         {
 638                 string filename = Path.GetTempFileName ();
 639
 640                 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
 641                 {
 642                         sw.WriteLine ("process attach --pid " + pid);
 643                         sw.WriteLine ("thread list");
 644                         sw.WriteLine ("thread backtrace all");
 645                         sw.WriteLine ("detach");
 646                         sw.WriteLine ("quit");
 647                         sw.Flush ();
 648
 649                         ProcessStartInfo psi = new ProcessStartInfo {
 650                                 FileName = "lldb",
 651                                 Arguments = "--batch --source \"" + filename + "\" --no-lldbinit",
 652                                 UseShellExecute = false,
 653                                 RedirectStandardError = true,
 654                                 RedirectStandardOutput = true,
 655                         };
 656
 657                         using (Process process = new Process { StartInfo = psi })
 658                         {
 659                                 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 660                                         lock (data.stdoutLock) {
 661                                                 if (e.Data != null)
 662                                                         data.stdout.AppendLine (e.Data);
 663                                         }
 664                                 };
 665
 666                                 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 667                                         lock (data.stderrLock) {
 668                                                 if (e.Data != null)
 669                                                         data.stderr.AppendLine (e.Data);
 670                                         }
 671                                 };
 672
 673                                 process.Start ();
 674                                 process.BeginOutputReadLine ();
 675                                 process.BeginErrorReadLine ();
 676                                 if (!process.WaitForExit (60 * 1000))
 677                                         process.Kill ();
 678                         }
 679                 }
 680         }
 681
 682         static void TryGDB (int pid, ProcessData data)
 683         {
 684                 string filename = Path.GetTempFileName ();
 685
 686                 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
 687                 {
 688                         sw.WriteLine ("attach " + pid);
 689                         sw.WriteLine ("info threads");
 690                         sw.WriteLine ("thread apply all p mono_print_thread_dump(0)");
 691                         sw.WriteLine ("thread apply all backtrace");
 692                         sw.Flush ();
 693
 694                         ProcessStartInfo psi = new ProcessStartInfo {
 695                                 FileName = "gdb",
 696                                 Arguments = "-batch -x \"" + filename + "\" -nx",
 697                                 UseShellExecute = false,
 698                                 RedirectStandardError = true,
 699                                 RedirectStandardOutput = true,
 700                         };
 701
 702                         using (Process process = new Process { StartInfo = psi })
 703                         {
 704                                 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 705                                         lock (data.stdoutLock) {
 706                                                 if (e.Data != null)
 707                                                         data.stdout.AppendLine (e.Data);
 708                                         }
 709                                 };
 710
 711                                 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 712                                         lock (data.stderrLock) {
 713                                                 if (e.Data != null)
 714                                                         data.stderr.AppendLine (e.Data);
 715                                         }
 716                                 };
 717
 718                                 process.Start ();
 719                                 process.BeginOutputReadLine ();
 720                                 process.BeginErrorReadLine ();
 721                                 if (!process.WaitForExit (60 * 1000))
 722                                         process.Kill ();
 723                         }
 724                 }
 725         }
 726 }