mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
  10 //
  11 using System;
  12 using System.IO;
  13 using System.Threading;
  14 using System.Text;
  15 using System.Diagnostics;
  16 using System.Collections.Generic;
  17 using System.Globalization;
  18 using System.Xml;
  19 using System.Text;
  20 using System.Text.RegularExpressions;
  21
  22 #if !MOBILE_STATIC
  23 using Mono.Unix.Native;
  24 #endif
  25
  26 //
  27 // This is a simple test runner with support for parallel execution
  28 //
  29
  30 public class TestRunner
  31 {
  32         const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
  33         const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
  34         const string MONO_PATH = "MONO_PATH";
  35
  36         class ProcessData {
  37                 public string test;
  38                 public StringBuilder stdout, stderr;
  39                 public string stdoutName, stderrName;
  40         }
  41
  42         class TestInfo {
  43                 public string test, opt_set;
  44         }
  45
  46         public static int Main (String[] args) {
  47                 // Defaults
  48                 int concurrency = 1;
  49                 int timeout = 2 * 60; // in seconds
  50                 int expectedExitCode = 0;
  51                 string testsuiteName = null;
  52                 string inputFile = null;
  53
  54                 string disabled_tests = null;
  55                 string runtime = "mono";
  56                 string config = null;
  57                 string mono_path = null;
  58                 var opt_sets = new List<string> ();
  59
  60                 string aot_run_flags = null;
  61                 string aot_build_flags = null;
  62
  63                 // Process options
  64                 int i = 0;
  65                 while (i < args.Length) {
  66                         if (args [i].StartsWith ("-")) {
  67                                 if (args [i] == "-j") {
  68                                         if (i + 1 >= args.Length) {
  69                                                 Console.WriteLine ("Missing argument to -j command line option.");
  70                                                 return 1;
  71                                         }
  72                                         if (args [i + 1] == "a")
  73                                                 concurrency = Environment.ProcessorCount;
  74                                         else
  75                                                 concurrency = Int32.Parse (args [i + 1]);
  76                                         i += 2;
  77                                 } else if (args [i] == "--timeout") {
  78                                         if (i + 1 >= args.Length) {
  79                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  80                                                 return 1;
  81                                         }
  82                                         timeout = Int32.Parse (args [i + 1]);
  83                                         i += 2;
  84                                 } else if (args [i] == "--disabled") {
  85                                         if (i + 1 >= args.Length) {
  86                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  87                                                 return 1;
  88                                         }
  89                                         disabled_tests = args [i + 1];
  90                                         i += 2;
  91                                 } else if (args [i] == "--runtime") {
  92                                         if (i + 1 >= args.Length) {
  93                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  94                                                 return 1;
  95                                         }
  96                                         runtime = args [i + 1];
  97                                         i += 2;
  98                                 } else if (args [i] == "--config") {
  99                                         if (i + 1 >= args.Length) {
 100                                                 Console.WriteLine ("Missing argument to --config command line option.");
 101                                                 return 1;
 102                                         }
 103                                         config = args [i + 1];
 104                                         i += 2;
 105                                 } else if (args [i] == "--opt-sets") {
 106                                         if (i + 1 >= args.Length) {
 107                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 108                                                 return 1;
 109                                         }
 110                                         foreach (var s in args [i + 1].Split ())
 111                                                 opt_sets.Add (s);
 112                                         i += 2;
 113                                 } else if (args [i] == "--expected-exit-code") {
 114                                         if (i + 1 >= args.Length) {
 115                                                 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
 116                                                 return 1;
 117                                         }
 118                                         expectedExitCode = Int32.Parse (args [i + 1]);
 119                                         i += 2;
 120                                 } else if (args [i] == "--testsuite-name") {
 121                                         if (i + 1 >= args.Length) {
 122                                                 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
 123                                                 return 1;
 124                                         }
 125                                         testsuiteName = args [i + 1];
 126                                         i += 2;
 127                                 } else if (args [i] == "--input-file") {
 128                                         if (i + 1 >= args.Length) {
 129                                                 Console.WriteLine ("Missing argument to --input-file command line option.");
 130                                                 return 1;
 131                                         }
 132                                         inputFile = args [i + 1];
 133                                         i += 2;
 134                                 } else if (args [i] == "--runtime") {
 135                                         if (i + 1 >= args.Length) {
 136                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
 137                                                 return 1;
 138                                         }
 139                                         runtime = args [i + 1];
 140                                         i += 2;
 141                                 } else if (args [i] == "--mono-path") {
 142                                         if (i + 1 >= args.Length) {
 143                                                 Console.WriteLine ("Missing argument to --mono-path command line option.");
 144                                                 return 1;
 145                                         }
 146                                         mono_path = args [i + 1].Substring(0, args [i + 1].Length);
 147
 148                                         i += 2;
 149                                 } else if (args [i] == "--aot-run-flags") {
 150                                         if (i + 1 >= args.Length) {
 151                                                 Console.WriteLine ("Missing argument to --aot-run-flags command line option.");
 152                                                 return 1;
 153                                         }
 154                                         aot_run_flags = args [i + 1].Substring(0, args [i + 1].Length);
 155                                         i += 2;
 156                                 } else if (args [i] == "--aot-build-flags") {
 157                                         if (i + 1 >= args.Length) {
 158                                                 Console.WriteLine ("Missing argument to --aot-build-flags command line option.");
 159                                                 return 1;
 160                                         }
 161                                         aot_build_flags = args [i + 1].Substring(0, args [i + 1].Length);
 162                                         i += 2;
 163                                 } else {
 164                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 165                                         return 1;
 166                                 }
 167                         } else {
 168                                 break;
 169                         }
 170                 }
 171
 172                 if (String.IsNullOrEmpty (testsuiteName)) {
 173                         Console.WriteLine ("Missing the required --testsuite-name command line option.");
 174                         return 1;
 175                 }
 176
 177                 var disabled = new Dictionary <string, string> ();
 178
 179                 if (disabled_tests != null) {
 180                         foreach (string test in disabled_tests.Split ())
 181                                 disabled [test] = test;
 182                 }
 183
 184                 var tests = new List<string> ();
 185
 186                 if (!String.IsNullOrEmpty (inputFile)) {
 187                         tests.AddRange (File.ReadAllLines (inputFile));
 188                 } else {
 189                         // The remaining arguments are the tests
 190                         for (int j = i; j < args.Length; ++j)
 191                                 if (!disabled.ContainsKey (args [j]))
 192                                         tests.Add (args [j]);
 193                 }
 194
 195                 var passed = new List<ProcessData> ();
 196                 var failed = new List<ProcessData> ();
 197                 var timedout = new List<ProcessData> ();
 198
 199                 object monitor = new object ();
 200
 201                 Console.WriteLine ("Running tests: ");
 202
 203                 var test_info = new Queue<TestInfo> ();
 204                 if (opt_sets.Count == 0) {
 205                         foreach (string s in tests)
 206                                 test_info.Enqueue (new TestInfo { test = s });
 207                 } else {
 208                         foreach (string opt in opt_sets) {
 209                                 foreach (string s in tests)
 210                                         test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
 211                         }
 212                 }
 213
 214                 /* compute the max length of test names, to have an optimal output width */
 215                 int output_width = -1;
 216                 foreach (TestInfo ti in test_info) {
 217                         if (ti.test.Length > output_width)
 218                                 output_width = Math.Min (120, ti.test.Length);
 219                 }
 220
 221                 if (aot_build_flags != null)  {
 222                         Console.WriteLine("AOT compiling tests");
 223
 224                         object aot_monitor = new object ();
 225                         var aot_queue = new Queue<String> (tests);
 226
 227                         List<Thread> build_threads = new List<Thread> (concurrency);
 228
 229                         for (int j = 0; j < concurrency; ++j) {
 230                                 Thread thread = new Thread (() => {
 231                                         while (true) {
 232                                                 String test_name;
 233
 234                                                 lock (aot_monitor) {
 235                                                         if (aot_queue.Count == 0)
 236                                                                 break;
 237                                                         test_name = aot_queue.Dequeue ();
 238                                                 }
 239
 240                                                 string test_bitcode_output = test_name + "_bitcode_tmp";
 241                                                 string test_bitcode_arg = ",temp-path=" + test_bitcode_output;
 242                                                 string aot_args = aot_build_flags + test_bitcode_arg + " " + test_name;
 243
 244                                                 Directory.CreateDirectory(test_bitcode_output);
 245
 246                                                 ProcessStartInfo job = new ProcessStartInfo (runtime, aot_args);
 247                                                 job.UseShellExecute = false;
 248                                                 job.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 249                                                 job.EnvironmentVariables[MONO_PATH] = mono_path;
 250                                                 Process compiler = new Process ();
 251                                                 compiler.StartInfo = job;
 252
 253                                                 compiler.Start ();
 254
 255                                                 if (!compiler.WaitForExit (timeout * 1000)) {
 256                                                         try {
 257                                                                 compiler.Kill ();
 258                                                         } catch {
 259                                                         }
 260                                                         throw new Exception(String.Format("Timeout AOT compiling tests, output in {0}", test_bitcode_output));
 261                                                 } else if (compiler.ExitCode != 0) {
 262                                                         throw new Exception(String.Format("Error AOT compiling tests, output in {0}", test_bitcode_output));
 263                                                 } else {
 264                                                         Directory.Delete (test_bitcode_output, true);
 265                                                 }
 266                                         }
 267                                 });
 268
 269                                 thread.Start ();
 270
 271                                 build_threads.Add (thread);
 272                         }
 273
 274                         for (int j = 0; j < build_threads.Count; ++j)
 275                                 build_threads [j].Join ();
 276
 277                         Console.WriteLine("Done compiling");
 278                 }
 279
 280                 List<Thread> threads = new List<Thread> (concurrency);
 281
 282                 DateTime test_start_time = DateTime.UtcNow;
 283
 284                 for (int j = 0; j < concurrency; ++j) {
 285                         Thread thread = new Thread (() => {
 286                                 while (true) {
 287                                         TestInfo ti;
 288
 289                                         lock (monitor) {
 290                                                 if (test_info.Count == 0)
 291                                                         break;
 292                                                 ti = test_info.Dequeue ();
 293                                         }
 294
 295                                         var output = new StringWriter ();
 296
 297                                         string test = ti.test;
 298                                         string opt_set = ti.opt_set;
 299
 300                                         output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
 301
 302                                         string test_invoke;
 303
 304                                         if (aot_run_flags != null)
 305                                                 test_invoke = aot_run_flags + " " + test;
 306                                         else
 307                                                 test_invoke = test;
 308
 309                                         /* Spawn a new process */
 310                                         string process_args;
 311                                         if (opt_set == null)
 312                                                 process_args = test_invoke;
 313                                         else
 314                                                 process_args = "-O=" + opt_set + " " + test_invoke;
 315
 316                                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 317                                         info.UseShellExecute = false;
 318                                         info.RedirectStandardOutput = true;
 319                                         info.RedirectStandardError = true;
 320                                         info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
 321                                         if (config != null)
 322                                                 info.EnvironmentVariables["MONO_CONFIG"] = config;
 323                                         if (mono_path != null)
 324                                                 info.EnvironmentVariables[MONO_PATH] = mono_path;
 325                                         Process p = new Process ();
 326                                         p.StartInfo = info;
 327
 328                                         ProcessData data = new ProcessData ();
 329                                         data.test = test;
 330
 331                                         string log_prefix = "";
 332                                         if (opt_set != null)
 333                                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 334
 335                                         data.stdoutName = test + log_prefix + ".stdout";
 336                                         data.stdout = new StringBuilder ();
 337
 338                                         data.stderrName = test + log_prefix + ".stderr";
 339                                         data.stderr = new StringBuilder ();
 340
 341                                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 342                                                 if (e.Data != null) {
 343                                                         data.stdout.AppendLine (e.Data);
 344                                                 }
 345                                         };
 346
 347                                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 348                                                 if (e.Data != null) {
 349                                                         data.stderr.AppendLine (e.Data);
 350                                                 }
 351                                         };
 352
 353                                         var start = DateTime.UtcNow;
 354
 355                                         p.Start ();
 356
 357                                         p.BeginOutputReadLine ();
 358                                         p.BeginErrorReadLine ();
 359
 360                                         if (!p.WaitForExit (timeout * 1000)) {
 361                                                 lock (monitor) {
 362                                                         timedout.Add (data);
 363                                                 }
 364
 365 #if !MOBILE_STATIC
 366                                                 // Force the process to print a thread dump
 367                                                 try {
 368                                                         Syscall.kill (p.Id, Signum.SIGQUIT);
 369                                                         Thread.Sleep (1000);
 370                                                 } catch {
 371                                                 }
 372 #endif
 373
 374                                                 output.Write ($"timed out ({timeout}s)");
 375
 376                                                 try {
 377                                                         p.Kill ();
 378                                                 } catch {
 379                                                 }
 380                                         } else if (p.ExitCode != expectedExitCode) {
 381                                                 var end = DateTime.UtcNow;
 382
 383                                                 lock (monitor) {
 384                                                         failed.Add (data);
 385                                                 }
 386
 387                                                 output.Write ("failed, time: {0}, exit code: {1}", (end - start).ToString (TEST_TIME_FORMAT), p.ExitCode);
 388                                         } else {
 389                                                 var end = DateTime.UtcNow;
 390
 391                                                 lock (monitor) {
 392                                                         passed.Add (data);
 393                                                 }
 394
 395                                                 output.Write ("passed, time: {0}", (end - start).ToString (TEST_TIME_FORMAT));
 396                                         }
 397
 398                                         p.Close ();
 399
 400                                         lock (monitor) {
 401                                                 Console.WriteLine (output.ToString ());
 402                                         }
 403                                 }
 404                         });
 405
 406                         thread.Start ();
 407
 408                         threads.Add (thread);
 409                 }
 410
 411                 for (int j = 0; j < threads.Count; ++j)
 412                         threads [j].Join ();
 413
 414                 TimeSpan test_time = DateTime.UtcNow - test_start_time;
 415
 416                 int npassed = passed.Count;
 417                 int nfailed = failed.Count;
 418                 int ntimedout = timedout.Count;
 419
 420                 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
 421                 xmlWriterSettings.NewLineOnAttributes = true;
 422                 xmlWriterSettings.Indent = true;
 423                 using (XmlWriter writer = XmlWriter.Create (String.Format ("TestResult-{0}.xml", testsuiteName), xmlWriterSettings)) {
 424                         // <?xml version="1.0" encoding="utf-8" standalone="no"?>
 425                         writer.WriteStartDocument ();
 426                         // <!--This file represents the results of running a test suite-->
 427                         writer.WriteComment ("This file represents the results of running a test suite");
 428                         // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
 429                         writer.WriteStartElement ("test-results");
 430                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 431                         writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
 432                         writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
 433                         writer.WriteAttributeString ("not-run", "0");
 434                         writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 435                         writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 436                         //   <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
 437                         writer.WriteStartElement ("environment");
 438                         writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
 439                         writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
 440                         writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
 441                         writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
 442                         writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
 443                         writer.WriteAttributeString ("machine-name", Environment.MachineName );
 444                         writer.WriteAttributeString ("user", Environment.UserName );
 445                         writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
 446                         writer.WriteEndElement ();
 447                         //   <culture-info current-culture="en-GB" current-uiculture="en-GB" />
 448                         writer.WriteStartElement ("culture-info");
 449                         writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
 450                         writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
 451                         writer.WriteEndElement ();
 452                         //   <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
 453                         writer.WriteStartElement ("test-suite");
 454                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 455                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 456                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 457                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 458                         //     <results>
 459                         writer.WriteStartElement ("results");
 460                         //       <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 461                         writer.WriteStartElement ("test-suite");
 462                         writer.WriteAttributeString ("name","MonoTests");
 463                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 464                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 465                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 466                         //         <results>
 467                         writer.WriteStartElement ("results");
 468                         //           <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 469                         writer.WriteStartElement ("test-suite");
 470                         writer.WriteAttributeString ("name", testsuiteName);
 471                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 472                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 473                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 474                         //             <results>
 475                         writer.WriteStartElement ("results");
 476                         // Dump all passing tests first
 477                         foreach (ProcessData pd in passed) {
 478                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 479                                 writer.WriteStartElement ("test-case");
 480                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 481                                 writer.WriteAttributeString ("executed", "True");
 482                                 writer.WriteAttributeString ("success", "True");
 483                                 writer.WriteAttributeString ("time", "0");
 484                                 writer.WriteAttributeString ("asserts", "0");
 485                                 writer.WriteEndElement ();
 486                         }
 487                         // Now dump all failing tests
 488                         foreach (ProcessData pd in failed) {
 489                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 490                                 writer.WriteStartElement ("test-case");
 491                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 492                                 writer.WriteAttributeString ("executed", "True");
 493                                 writer.WriteAttributeString ("success", "False");
 494                                 writer.WriteAttributeString ("time", "0");
 495                                 writer.WriteAttributeString ("asserts", "1");
 496                                 writer.WriteStartElement ("failure");
 497                                 writer.WriteStartElement ("message");
 498                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 499                                 writer.WriteEndElement ();
 500                                 writer.WriteStartElement ("stack-trace");
 501                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 502                                 writer.WriteEndElement ();
 503                                 writer.WriteEndElement ();
 504                                 writer.WriteEndElement ();
 505                         }
 506                         // Then dump all timing out tests
 507                         foreach (ProcessData pd in timedout) {
 508                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 509                                 writer.WriteStartElement ("test-case");
 510                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
 511                                 writer.WriteAttributeString ("executed", "True");
 512                                 writer.WriteAttributeString ("success", "False");
 513                                 writer.WriteAttributeString ("time", "0");
 514                                 writer.WriteAttributeString ("asserts", "1");
 515                                 writer.WriteStartElement ("failure");
 516                                 writer.WriteStartElement ("message");
 517                                 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
 518                                 writer.WriteEndElement ();
 519                                 writer.WriteStartElement ("stack-trace");
 520                                 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
 521                                 writer.WriteEndElement ();
 522                                 writer.WriteEndElement ();
 523                                 writer.WriteEndElement ();
 524                         }
 525                         //             </results>
 526                         writer.WriteEndElement ();
 527                         //           </test-suite>
 528                         writer.WriteEndElement ();
 529                         //         </results>
 530                         writer.WriteEndElement ();
 531                         //       </test-suite>
 532                         writer.WriteEndElement ();
 533                         //     </results>
 534                         writer.WriteEndElement ();
 535                         //   </test-suite>
 536                         writer.WriteEndElement ();
 537                         // </test-results>
 538                         writer.WriteEndElement ();
 539                         writer.WriteEndDocument ();
 540                 }
 541
 542                 Console.WriteLine ();
 543                 Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
 544                 Console.WriteLine ();
 545                 Console.WriteLine ("{0,4} test(s) passed", npassed);
 546                 Console.WriteLine ("{0,4} test(s) failed", nfailed);
 547                 Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
 548
 549                 if (nfailed > 0) {
 550                         Console.WriteLine ();
 551                         Console.WriteLine ("Failed test(s):");
 552                         foreach (ProcessData pd in failed) {
 553                                 Console.WriteLine ();
 554                                 Console.WriteLine (pd.test);
 555                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 556                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 557                         }
 558                 }
 559
 560                 if (ntimedout > 0) {
 561                         Console.WriteLine ();
 562                         Console.WriteLine ("Timed out test(s):");
 563                         foreach (ProcessData pd in timedout) {
 564                                 Console.WriteLine ();
 565                                 Console.WriteLine (pd.test);
 566                                 DumpFile (pd.stdoutName, pd.stdout.ToString ());
 567                                 DumpFile (pd.stderrName, pd.stderr.ToString ());
 568                         }
 569                 }
 570
 571                 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
 572         }
 573
 574         static void DumpFile (string filename, string text) {
 575                 Console.WriteLine ("=============== {0} ===============", filename);
 576                 Console.WriteLine (text);
 577                 Console.WriteLine ("=============== EOF ===============");
 578         }
 579
 580         static string FilterInvalidXmlChars (string text) {
 581                 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
 582                 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]      /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
 583                 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
 584         }
 585 }