mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
  10 //
  11 using System;
  12 using System.IO;
  13 using System.Threading;
  14 using System.Diagnostics;
  15 using System.Collections.Generic;
  16 using System.Globalization;
  17 using System.Xml;
  18 using System.Text;
  19 using System.Text.RegularExpressions;
  20
  21 #if !FULL_AOT_DESKTOP && !MOBILE
  22 using Mono.Unix.Native;
  23 #endif
  24
  25 //
  26 // This is a simple test runner with support for parallel execution
  27 //
  28
  29 public class TestRunner
  30 {
  31         const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
  32         const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
  33         const string MONO_PATH = "MONO_PATH";
  34         const string MONO_GAC_PREFIX = "MONO_GAC_PREFIX";
  35
  36         class ProcessData {
  37                 public string test;
  38                 public StringBuilder stdout, stderr;
  39                 public object stdoutLock = new object (), stderrLock = new object ();
  40                 public string stdoutName, stderrName;
  41         }
  42
  43         class TestInfo {
  44                 public string test, opt_set;
  45         }
  46
  47         public static int Main (String[] args) {
  48                 // Defaults
  49                 int concurrency = 1;
  50                 int timeout = 2 * 60; // in seconds
  51                 int expectedExitCode = 0;
  52                 bool verbose = false;
  53                 string testsuiteName = null;
  54                 string inputFile = null;
  55
  56                 string disabled_tests = null;
  57                 string runtime = "mono";
  58                 string config = null;
  59                 string mono_path = null;
  60                 string runtime_args = null;
  61                 string mono_gac_prefix = null;
  62                 var opt_sets = new List<string> ();
  63
  64                 string aot_run_flags = null;
  65                 string aot_build_flags = null;
  66
  67                 // Process options
  68                 int i = 0;
  69                 while (i < args.Length) {
  70                         if (args [i].StartsWith ("-")) {
  71                                 if (args [i] == "-j") {
  72                                         if (i + 1 >= args.Length) {
  73                                                 Console.WriteLine ("Missing argument to -j command line option.");
  74                                                 return 1;
  75                                         }
  76                                         if (args [i + 1] == "a")
  77                                                 concurrency = Environment.ProcessorCount;
  78                                         else
  79                                                 concurrency = Int32.Parse (args [i + 1]);
  80                                         i += 2;
  81                                 } else if (args [i] == "--timeout") {
  82                                         if (i + 1 >= args.Length) {
  83                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  84                                                 return 1;
  85                                         }
  86                                         timeout = Int32.Parse (args [i + 1]);
  87                                         i += 2;
  88                                 } else if (args [i] == "--disabled") {
  89                                         if (i + 1 >= args.Length) {
  90                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  91                                                 return 1;
  92                                         }
  93                                         disabled_tests = args [i + 1];
  94                                         i += 2;
  95                                 } else if (args [i] == "--runtime") {
  96                                         if (i + 1 >= args.Length) {
  97                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  98                                                 return 1;
  99                                         }
 100                                         runtime = args [i + 1];
 101                                         i += 2;
 102                                 } else if (args [i] == "--runtime-args") {
 103                                         if (i + 1 >= args.Length) {
 104                                                 Console.WriteLine ("Missing argument to --runtime-args command line option.");
 105                                                 return 1;
 106                                         }
 107                                         runtime_args = args [i + 1];
 108                                         i += 2;
 109                                 } else if (args [i] == "--config") {
 110                                         if (i + 1 >= args.Length) {
 111                                                 Console.WriteLine ("Missing argument to --config command line option.");
 112                                                 return 1;
 113                                         }
 114                                         config = args [i + 1];
 115                                         i += 2;
 116                                 } else if (args [i] == "--opt-sets") {
 117                                         if (i + 1 >= args.Length) {
 118                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 119                                                 return 1;
 120                                         }
 121                                         foreach (var s in args [i + 1].Split ())
 122                                                 opt_sets.Add (s);
 123                                         i += 2;
 124                                 } else if (args [i] == "--expected-exit-code") {
 125                                         if (i + 1 >= args.Length) {
 126                                                 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
 127                                                 return 1;
 128                                         }
 129                                         expectedExitCode = Int32.Parse (args [i + 1]);
 130                                         i += 2;
 131                                 } else if (args [i] == "--testsuite-name") {
 132                                         if (i + 1 >= args.Length) {
 133                                                 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
 134                                                 return 1;
 135                                         }
 136                                         testsuiteName = args [i + 1];
 137                                         i += 2;
 138                                 } else if (args [i] == "--input-file") {
 139                                         if (i + 1 >= args.Length) {
 140                                                 Console.WriteLine ("Missing argument to --input-file command line option.");
 141                                                 return 1;
 142                                         }
 143                                         inputFile = args [i + 1];
 144                                         i += 2;
 145                                 } else if (args [i] == "--mono-path") {
 146                                         if (i + 1 >= args.Length) {
 147                                                 Console.WriteLine ("Missing argument to --mono-path command line option.");
 148                                                 return 1;
 149                                         }
 150                                         mono_path = args [i + 1].Substring(0, args [i + 1].Length);
 151
 152                                         i += 2;
 153                                 } else if (args [i] == "--mono-gac-prefix") {
 154                                         if (i + 1 >= args.Length) {
 155                                                 Console.WriteLine ("Missing argument to --mono-gac-prefix command line option.");
 156                                                 return 1;
 157                                         }
 158                                         mono_gac_prefix = args[i + 1];
 159                                         i += 2;
 160                                 } else if (args [i] == "--aot-run-flags") {
 161                                         if (i + 1 >= args.Length) {
 162                                                 Console.WriteLine ("Missing argument to --aot-run-flags command line option.");
 163                                                 return 1;
 164                                         }
 165                                         aot_run_flags = args [i + 1].Substring(0, args [i + 1].Length);
 166                                         i += 2;
 167                                 } else if (args [i] == "--aot-build-flags") {
 168                                         if (i + 1 >= args.Length) {
 169                                                 Console.WriteLine ("Missing argument to --aot-build-flags command line option.");
 170                                                 return 1;
 171                                         }
 172                                         aot_build_flags = args [i + 1].Substring(0, args [i + 1].Length);
 173                                         i += 2;
 174                                 } else if (args [i] == "--verbose") {
 175                                         verbose = true;
 176                                         i ++;
 177                                 } else {
 178                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 179                                         return 1;
 180                                 }
 181                         } else {
 182                                 break;
 183                         }
 184                 }
 185
 186                 if (String.IsNullOrEmpty (testsuiteName)) {
 187                         Console.WriteLine ("Missing the required --testsuite-name command line option.");
 188                         return 1;
 189                 }
 190
 191                 var disabled = new Dictionary <string, string> ();
 192
 193                 if (disabled_tests != null) {
 194                         foreach (string test in disabled_tests.Split ())
 195                                 disabled [test] = test;
 196                 }
 197
 198                 var tests = new List<string> ();
 199
 200                 if (!String.IsNullOrEmpty (inputFile)) {
 201                         tests.AddRange (File.ReadAllLines (inputFile));
 202                 } else {
 203                         // The remaining arguments are the tests
 204                         for (int j = i; j < args.Length; ++j)
 205                                 if (!disabled.ContainsKey (args [j]))
 206                                         tests.Add (args [j]);
 207                 }
 208
 209                 var passed = new List<ProcessData> ();
 210                 var failed = new List<ProcessData> ();
 211                 var timedout = new List<ProcessData> ();
 212
 213                 object monitor = new object ();
 214
 215                 Console.WriteLine ("Running tests: ");
 216
 217                 var test_info = new Queue<TestInfo> ();
 218                 if (opt_sets.Count == 0) {
 219                         foreach (string s in tests)
 220                                 test_info.Enqueue (new TestInfo { test = s });
 221                 } else {
 222                         foreach (string opt in opt_sets) {
 223                                 foreach (string s in tests)
 224                                         test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
 225                         }
 226                 }
 227
 228                 /* compute the max length of test names, to have an optimal output width */
 229                 int output_width = -1;
 230                 foreach (TestInfo ti in test_info) {
 231                         if (ti.test.Length > output_width)
 232                                 output_width = Math.Min (120, ti.test.Length);
 233                 }
 234
 235                 if (aot_build_flags != null)  {
 236                         Console.WriteLine("AOT compiling tests");
 237
 238                         object aot_monitor = new object ();
 239                         var aot_queue = new Queue<String> (tests);
 240
 241                         List<Thread> build_threads = new List<Thread> (concurrency);
 242
 243                         for (int j = 0; j < concurrency; ++j) {
 244                                 Thread thread = new Thread (() => {
 245                                         while (true) {
 246                                                 String test_name;
 247
 248                                                 lock (aot_monitor) {
 249                                                         if (aot_queue.Count == 0)
 250                                                                 break;
 251                                                         test_name = aot_queue.Dequeue ();
 252                                                 }
 253
 254                                                 string test_bitcode_output = test_name + "_bitcode_tmp";
 255                                                 string test_bitcode_arg = ",temp-path=" + test_bitcode_output;
 256                                                 string aot_args = aot_build_flags + test_bitcode_arg + " " + test_name;
 257
 258                                                 Directory.CreateDirectory(test_bitcode_output);
 259
 260                                                 ProcessStartInfo job = new ProcessStartInfo (runtime, aot_args);
 261                                                 job.UseShellExecute = false;
 262                                                 job.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 263                                                 if (mono_path != null)
 264                                                         job.EnvironmentVariables[MONO_PATH] = mono_path;
 265                                                 if (mono_gac_prefix != null)
 266                                                         job.EnvironmentVariables[MONO_GAC_PREFIX] = mono_gac_prefix;
 267                                                 Process compiler = new Process ();
 268                                                 compiler.StartInfo = job;
 269
 270                                                 compiler.Start ();
 271
 272                                                 if (!compiler.WaitForExit (timeout * 1000)) {
 273                                                         try {
 274                                                                 compiler.Kill ();
 275                                                         } catch {
 276                                                         }
 277                                                         throw new Exception(String.Format("Timeout AOT compiling tests, output in {0}", test_bitcode_output));
 278                                                 } else if (compiler.ExitCode != 0) {
 279                                                         throw new Exception(String.Format("Error AOT compiling tests, output in {0}", test_bitcode_output));
 280                                                 } else {
 281                                                         Directory.Delete (test_bitcode_output, true);
 282                                                 }
 283                                         }
 284                                 });
 285
 286                                 thread.Start ();
 287
 288                                 build_threads.Add (thread);
 289                         }
 290
 291                         for (int j = 0; j < build_threads.Count; ++j)
 292                                 build_threads [j].Join ();
 293
 294                         Console.WriteLine("Done compiling");
 295                 }
 296
 297                 List<Thread> threads = new List<Thread> (concurrency);
 298
 299                 DateTime test_start_time = DateTime.UtcNow;
 300
 301                 for (int j = 0; j < concurrency; ++j) {
 302                         Thread thread = new Thread (() => {
 303                                 while (true) {
 304                                         TestInfo ti;
 305
 306                                         lock (monitor) {
 307                                                 if (test_info.Count == 0)
 308                                                         break;
 309                                                 ti = test_info.Dequeue ();
 310                                         }
 311
 312                                         var output = new StringWriter ();
 313
 314                                         string test = ti.test;
 315                                         string opt_set = ti.opt_set;
 316
 317                                         if (verbose) {
 318                                                 output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
 319                                         } else {
 320                                                 Console.Write (".");
 321                                         }
 322
 323                                         string test_invoke;
 324
 325                                         if (aot_run_flags != null)
 326                                                 test_invoke = aot_run_flags + " " + test;
 327                                         else
 328                                                 test_invoke = test;
 329
 330                                         /* Spawn a new process */
 331                                         string process_args = test_invoke;
 332                                         if (opt_set != null)
 333                                                 process_args = "-O=" + opt_set + " " + process_args;
 334                                         if (runtime_args != null)
 335                                                 process_args = runtime_args + " " + process_args;
 336
 337                                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 338                                         info.UseShellExecute = false;
 339                                         info.RedirectStandardOutput = true;
 340                                         info.RedirectStandardError = true;
 341                                         info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 342                                         if (config != null)
 343                                                 info.EnvironmentVariables["MONO_CONFIG"] = config;
 344                                         if (mono_path != null)
 345                                                 info.EnvironmentVariables[MONO_PATH] = mono_path;
 346                                         if (mono_gac_prefix != null)
 347                                                 info.EnvironmentVariables[MONO_GAC_PREFIX] = mono_gac_prefix;
 348                                         Process p = new Process ();
 349                                         p.StartInfo = info;
 350
 351                                         ProcessData data = new ProcessData ();
 352                                         data.test = test;
 353
 354                                         string log_prefix = "";
 355                                         if (opt_set != null)
 356                                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 357
 358                                         data.stdoutName = test + log_prefix + ".stdout";
 359                                         data.stdout = new StringBuilder ();
 360
 361                                         data.stderrName = test + log_prefix + ".stderr";
 362                                         data.stderr = new StringBuilder ();
 363
 364                                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 365                                                 lock (data.stdoutLock) {
 366                                                         if (e.Data != null)
 367                                                                 data.stdout.AppendLine (e.Data);
 368                                                 }
 369                                         };
 370
 371                                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 372                                                 lock (data.stderrLock) {
 373                                                         if (e.Data != null)
 374                                                                 data.stderr.AppendLine (e.Data);
 375                                                 }
 376                                         };
 377
 378                                         var start = DateTime.UtcNow;
 379
 380                                         p.Start ();
 381
 382                                         p.BeginOutputReadLine ();
 383                                         p.BeginErrorReadLine ();
 384
 385                                         if (!p.WaitForExit (timeout * 1000)) {
 386                                                 lock (monitor) {
 387                                                         timedout.Add (data);
 388                                                 }
 389
 390                                                 // Force the process to print a thread dump
 391                                                 TryThreadDump (p.Id, data);
 392
 393                                                 if (verbose) {
 394                                                         output.Write ($"timed out ({timeout}s)");
 395                                                 }
 396
 397                                                 try {
 398                                                         p.Kill ();
 399                                                 } catch {
 400                                                 }
 401                                         } else if (p.ExitCode != expectedExitCode) {
 402                                                 var end = DateTime.UtcNow;
 403
 404                                                 lock (monitor) {
 405                                                         failed.Add (data);
 406                                                 }
 407
 408                                                 if (verbose)
 409                                                         output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
 410                                         } else {
 411                                                 var end = DateTime.UtcNow;
 412
 413                                                 lock (monitor) {
 414                                                         passed.Add (data);
 415                                                 }
 416
 417                                                 if (verbose)
 418                                                         output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
 419                                         }
 420
 421                                         p.Close ();
 422
 423                                         lock (monitor) {
 424                                                 if (verbose)
 425                                                         Console.WriteLine (output.ToString ());
 426                                         }
 427                                 }
 428                         });
 429
 430                         thread.Start ();
 431
 432                         threads.Add (thread);
 433                 }
 434
 435                 for (int j = 0; j < threads.Count; ++j)
 436                         threads [j].Join ();
 437
 438                 TimeSpan test_time = DateTime.UtcNow - test_start_time;
 439
 440                 int npassed = passed.Count;
 441                 int nfailed = failed.Count;
 442                 int ntimedout = timedout.Count;
 443
 444                 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
 445                 xmlWriterSettings.NewLineOnAttributes = true;
 446                 xmlWriterSettings.Indent = true;
 447
 448                 string xmlPath = String.Format ("TestResult-{0}.xml", testsuiteName);
 449                 using (XmlWriter writer = XmlWriter.Create (xmlPath, xmlWriterSettings)) {
 450                         // <?xml version="1.0" encoding="utf-8" standalone="no"?>
 451                         writer.WriteStartDocument ();
 452                         // <!--This file represents the results of running a test suite-->
 453                         writer.WriteComment ("This file represents the results of running a test suite");
 454                         // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
 455                         writer.WriteStartElement ("test-results");
 456                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 457                         writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
 458                         writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
 459                         writer.WriteAttributeString ("not-run", "0");
 460                         writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 461                         writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 462                         //   <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
 463                         writer.WriteStartElement ("environment");
 464                         writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
 465                         writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
 466                         writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
 467                         writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
 468                         writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
 469                         writer.WriteAttributeString ("machine-name", Environment.MachineName );
 470                         writer.WriteAttributeString ("user", Environment.UserName );
 471                         writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
 472                         writer.WriteEndElement ();
 473                         //   <culture-info current-culture="en-GB" current-uiculture="en-GB" />
 474                         writer.WriteStartElement ("culture-info");
 475                         writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
 476                         writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
 477                         writer.WriteEndElement ();
 478                         //   <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
 479                         writer.WriteStartElement ("test-suite");
 480                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 481                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 482                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 483                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 484                         //     <results>
 485                         writer.WriteStartElement ("results");
 486                         //       <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 487                         writer.WriteStartElement ("test-suite");
 488                         writer.WriteAttributeString ("name","MonoTests");
 489                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 490                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 491                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 492                         //         <results>
 493                         writer.WriteStartElement ("results");
 494                         //           <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 495                         writer.WriteStartElement ("test-suite");
 496                         writer.WriteAttributeString ("name", testsuiteName);
 497                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 498                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 499                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 500                         //             <results>
 501                         writer.WriteStartElement ("results");
 502                         // Dump all passing tests first
 503                         foreach (ProcessData pd in passed) {
 504                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 505                                 writer.WriteStartElement ("test-case");
 506                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 507                                 writer.WriteAttributeString ("executed", "True");
 508                                 writer.WriteAttributeString ("success", "True");
 509                                 writer.WriteAttributeString ("time", "0");
 510                                 writer.WriteAttributeString ("asserts", "0");
 511                                 writer.WriteEndElement ();
 512                         }
 513                         // Now dump all failing tests
 514                         foreach (ProcessData pd in failed) {
 515                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 516                                 writer.WriteStartElement ("test-case");
 517                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 518                                 writer.WriteAttributeString ("executed", "True");
 519                                 writer.WriteAttributeString ("success", "False");
 520                                 writer.WriteAttributeString ("time", "0");
 521                                 writer.WriteAttributeString ("asserts", "1");
 522                                 writer.WriteStartElement ("failure");
 523                                 writer.WriteStartElement ("message");
 524                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 525                                 writer.WriteEndElement ();
 526                                 writer.WriteStartElement ("stack-trace");
 527                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 528                                 writer.WriteEndElement ();
 529                                 writer.WriteEndElement ();
 530                                 writer.WriteEndElement ();
 531                         }
 532                         // Then dump all timing out tests
 533                         foreach (ProcessData pd in timedout) {
 534                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 535                                 writer.WriteStartElement ("test-case");
 536                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
 537                                 writer.WriteAttributeString ("executed", "True");
 538                                 writer.WriteAttributeString ("success", "False");
 539                                 writer.WriteAttributeString ("time", "0");
 540                                 writer.WriteAttributeString ("asserts", "1");
 541                                 writer.WriteStartElement ("failure");
 542                                 writer.WriteStartElement ("message");
 543                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 544                                 writer.WriteEndElement ();
 545                                 writer.WriteStartElement ("stack-trace");
 546                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 547                                 writer.WriteEndElement ();
 548                                 writer.WriteEndElement ();
 549                                 writer.WriteEndElement ();
 550                         }
 551                         //             </results>
 552                         writer.WriteEndElement ();
 553                         //           </test-suite>
 554                         writer.WriteEndElement ();
 555                         //         </results>
 556                         writer.WriteEndElement ();
 557                         //       </test-suite>
 558                         writer.WriteEndElement ();
 559                         //     </results>
 560                         writer.WriteEndElement ();
 561                         //   </test-suite>
 562                         writer.WriteEndElement ();
 563                         // </test-results>
 564                         writer.WriteEndElement ();
 565                         writer.WriteEndDocument ();
 566
 567                         string babysitterXmlList = Environment.GetEnvironmentVariable("MONO_BABYSITTER_NUNIT_XML_LIST_FILE");
 568                         if (!String.IsNullOrEmpty(babysitterXmlList)) {
 569                                 try {
 570                                         string fullXmlPath = Path.GetFullPath(xmlPath);
 571                                         File.AppendAllText(babysitterXmlList, fullXmlPath + Environment.NewLine);
 572                                 } catch (Exception e) {
 573                                         Console.WriteLine("Attempted to record XML path to file {0} but failed.", babysitterXmlList);
 574                                 }
 575                         }
 576                 }
 577
 578                 if (verbose) {
 579                         Console.WriteLine ();
 580                         Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
 581                         Console.WriteLine ();
 582                         Console.WriteLine ("{0,4} test(s) passed", npassed);
 583                         Console.WriteLine ("{0,4} test(s) failed", nfailed);
 584                         Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
 585                 } else {
 586                         Console.WriteLine ();
 587                         Console.WriteLine (String.Format ("{0} test(s) passed, {1} test(s) did not pass.", npassed, nfailed));
 588                 }
 589
 590                 if (nfailed > 0) {
 591                         Console.WriteLine ();
 592                         Console.WriteLine ("Failed test(s):");
 593                         foreach (ProcessData pd in failed) {
 594                                 Console.WriteLine ();
 595                                 Console.WriteLine (pd.test);
 596                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 597                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 598                         }
 599                 }
 600
 601                 if (ntimedout > 0) {
 602                         Console.WriteLine ();
 603                         Console.WriteLine ("Timed out test(s):");
 604                         foreach (ProcessData pd in timedout) {
 605                                 Console.WriteLine ();
 606                                 Console.WriteLine (pd.test);
 607                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 608                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 609                         }
 610                 }
 611
 612                 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
 613         }
 614
 615         static void DumpFile (string filename, string text) {
 616                 Console.WriteLine ("=============== {0} ===============", filename);
 617                 Console.WriteLine (text);
 618                 Console.WriteLine ("=============== EOF ===============");
 619         }
 620
 621         static string FilterInvalidXmlChars (string text) {
 622                 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
 623                 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]      /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
 624                 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
 625         }
 626
 627         static void TryThreadDump (int pid, ProcessData data)
 628         {
 629                 try {
 630                         TryGDB (pid, data);
 631                         return;
 632                 } catch {
 633                 }
 634
 635 #if !FULL_AOT_DESKTOP && !MOBILE
 636                 /* LLDB cannot produce managed stacktraces for all the threads */
 637                 try {
 638                         Syscall.kill (pid, Signum.SIGQUIT);
 639                         Thread.Sleep (1000);
 640                 } catch {
 641                 }
 642 #endif
 643
 644                 try {
 645                         TryLLDB (pid, data);
 646                         return;
 647                 } catch {
 648                 }
 649         }
 650
 651         static void TryLLDB (int pid, ProcessData data)
 652         {
 653                 string filename = Path.GetTempFileName ();
 654
 655                 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
 656                 {
 657                         sw.WriteLine ("process attach --pid " + pid);
 658                         sw.WriteLine ("thread list");
 659                         sw.WriteLine ("thread backtrace all");
 660                         sw.WriteLine ("detach");
 661                         sw.WriteLine ("quit");
 662                         sw.Flush ();
 663
 664                         ProcessStartInfo psi = new ProcessStartInfo {
 665                                 FileName = "lldb",
 666                                 Arguments = "--batch --source \"" + filename + "\" --no-lldbinit",
 667                                 UseShellExecute = false,
 668                                 RedirectStandardError = true,
 669                                 RedirectStandardOutput = true,
 670                         };
 671
 672                         using (Process process = new Process { StartInfo = psi })
 673                         {
 674                                 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 675                                         lock (data.stdoutLock) {
 676                                                 if (e.Data != null)
 677                                                         data.stdout.AppendLine (e.Data);
 678                                         }
 679                                 };
 680
 681                                 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 682                                         lock (data.stderrLock) {
 683                                                 if (e.Data != null)
 684                                                         data.stderr.AppendLine (e.Data);
 685                                         }
 686                                 };
 687
 688                                 process.Start ();
 689                                 process.BeginOutputReadLine ();
 690                                 process.BeginErrorReadLine ();
 691                                 if (!process.WaitForExit (60 * 1000))
 692                                         process.Kill ();
 693                         }
 694                 }
 695         }
 696
 697         static void TryGDB (int pid, ProcessData data)
 698         {
 699                 string filename = Path.GetTempFileName ();
 700
 701                 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
 702                 {
 703                         sw.WriteLine ("attach " + pid);
 704                         sw.WriteLine ("info threads");
 705                         sw.WriteLine ("thread apply all p mono_print_thread_dump(0)");
 706                         sw.WriteLine ("thread apply all backtrace");
 707                         sw.Flush ();
 708
 709                         ProcessStartInfo psi = new ProcessStartInfo {
 710                                 FileName = "gdb",
 711                                 Arguments = "-batch -x \"" + filename + "\" -nx",
 712                                 UseShellExecute = false,
 713                                 RedirectStandardError = true,
 714                                 RedirectStandardOutput = true,
 715                         };
 716
 717                         using (Process process = new Process { StartInfo = psi })
 718                         {
 719                                 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 720                                         lock (data.stdoutLock) {
 721                                                 if (e.Data != null)
 722                                                         data.stdout.AppendLine (e.Data);
 723                                         }
 724                                 };
 725
 726                                 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 727                                         lock (data.stderrLock) {
 728                                                 if (e.Data != null)
 729                                                         data.stderr.AppendLine (e.Data);
 730                                         }
 731                                 };
 732
 733                                 process.Start ();
 734                                 process.BeginOutputReadLine ();
 735                                 process.BeginErrorReadLine ();
 736                                 if (!process.WaitForExit (60 * 1000))
 737                                         process.Kill ();
 738                         }
 739                 }
 740         }
 741 }