mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Threading;
  31 using System.Diagnostics;
  32 using System.Collections.Generic;
  33 using System.Globalization;
  34 using System.Xml;
  35
  36 //
  37 // This is a simple test runner with support for parallel execution
  38 //
  39
  40 public class TestRunner
  41 {
  42         class ProcessData {
  43                 public string test;
  44                 public StreamWriter stdout, stderr;
  45                 public string stdoutFile, stderrFile;
  46
  47                 public void CloseStreams () {
  48                         if (stdout != null) {
  49                                 stdout.Close ();
  50                                 stdout = null;
  51                         }
  52                         if (stderr != null) {
  53                                 stderr.Close ();
  54                                 stderr = null;
  55                         }
  56                 }
  57         }
  58
  59         class TestInfo {
  60                 public string test, opt_set;
  61         }
  62
  63         public static int Main (String[] args) {
  64                 // Defaults
  65                 int concurrency = 1;
  66                 int timeout = 2 * 60; // in seconds
  67                 int expectedExitCode = 0;
  68
  69                 DateTime test_start_time = DateTime.UtcNow;
  70
  71                 // FIXME: Add support for runtime arguments + env variables
  72
  73                 string disabled_tests = null;
  74                 string runtime = "mono";
  75                 var opt_sets = new List<string> ();
  76
  77                 // Process options
  78                 int i = 0;
  79                 while (i < args.Length) {
  80                         if (args [i].StartsWith ("-")) {
  81                                 if (args [i] == "-j") {
  82                                         if (i + i >= args.Length) {
  83                                                 Console.WriteLine ("Missing argument to -j command line option.");
  84                                                 return 1;
  85                                         }
  86                                         if (args [i + 1] == "a")
  87                                                 concurrency = Environment.ProcessorCount;
  88                                         else
  89                                                 concurrency = Int32.Parse (args [i + 1]);
  90                                         i += 2;
  91                                 } else if (args [i] == "--timeout") {
  92                                         if (i + i >= args.Length) {
  93                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  94                                                 return 1;
  95                                         }
  96                                         timeout = Int32.Parse (args [i + 1]);
  97                                         i += 2;
  98                                 } else if (args [i] == "--disabled") {
  99                                         if (i + i >= args.Length) {
 100                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
 101                                                 return 1;
 102                                         }
 103                                         disabled_tests = args [i + 1];
 104                                         i += 2;
 105                                 } else if (args [i] == "--runtime") {
 106                                         if (i + i >= args.Length) {
 107                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
 108                                                 return 1;
 109                                         }
 110                                         runtime = args [i + 1];
 111                                         i += 2;
 112                                 } else if (args [i] == "--opt-sets") {
 113                                         if (i + i >= args.Length) {
 114                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 115                                                 return 1;
 116                                         }
 117                                         foreach (var s in args [i + 1].Split ())
 118                                                 opt_sets.Add (s);
 119                                         i += 2;
 120                                 } else if (args [i] == "--expected-exit-code") {
 121                                         if (i + i >= args.Length) {
 122                                                 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
 123                                                 return 1;
 124                                         }
 125                                         expectedExitCode = Int32.Parse (args [i + 1]);
 126                                         i += 2;
 127                                 } else {
 128                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 129                                         return 1;
 130                                 }
 131                         } else {
 132                                 break;
 133                         }
 134                 }
 135
 136                 var disabled = new Dictionary <string, string> ();
 137
 138                 if (disabled_tests != null) {
 139                         foreach (string test in disabled_tests.Split ())
 140                                 disabled [test] = test;
 141                 }
 142
 143                 // The remaining arguments are the tests
 144                 var tests = new List<string> ();
 145                 for (int j = i; j < args.Length; ++j)
 146                         if (!disabled.ContainsKey (args [j]))
 147                                 tests.Add (args [j]);
 148
 149                 int npassed = 0;
 150                 int nfailed = 0;
 151
 152                 var processes = new List<Process> ();
 153                 var passed = new List<ProcessData> ();
 154                 var failed = new List<ProcessData> ();
 155                 var process_data = new Dictionary<Process, ProcessData> ();
 156
 157                 object monitor = new object ();
 158
 159                 var terminated = new List<Process> ();
 160
 161                 if (concurrency != 1)
 162                         Console.WriteLine ("Running tests: ");
 163
 164                 var test_info = new List<TestInfo> ();
 165                 if (opt_sets.Count == 0) {
 166                         foreach (string s in tests)
 167                                 test_info.Add (new TestInfo { test = s });
 168                 } else {
 169                         foreach (string opt in opt_sets) {
 170                                 foreach (string s in tests)
 171                                         test_info.Add (new TestInfo { test = s, opt_set = opt });
 172                         }
 173                 }
 174
 175                 foreach (TestInfo ti in test_info) {
 176                         lock (monitor) {
 177                                 while (processes.Count == concurrency) {
 178                                         /* Wait for one process to terminate */
 179                                         Monitor.Wait (monitor);
 180                                 }
 181
 182                                 /* Cleaup terminated processes */
 183                                 foreach (Process dead in terminated) {
 184                                         if (process_data [dead].stdout != null)
 185                                                 process_data [dead].stdout.Close ();
 186                                         if (process_data [dead].stderr != null)
 187                                                 process_data [dead].stderr.Close ();
 188                                         // This is needed to avoid CreateProcess failed errors :(
 189                                         dead.Close ();
 190                                 }
 191                                 terminated.Clear ();
 192                         }
 193
 194                         string test = ti.test;
 195                         string opt_set = ti.opt_set;
 196
 197                         if (concurrency == 1)
 198                                 Console.Write ("Testing " + test + "... ");
 199
 200                         /* Spawn a new process */
 201                         string process_args;
 202                         if (opt_set == null)
 203                                 process_args = test;
 204                         else
 205                                 process_args = "-O=" + opt_set + " " + test;
 206                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 207                         info.UseShellExecute = false;
 208                         info.RedirectStandardOutput = true;
 209                         info.RedirectStandardError = true;
 210                         Process p = new Process ();
 211                         p.StartInfo = info;
 212
 213                         ProcessData data = new ProcessData ();
 214                         data.test = test;
 215
 216                         string log_prefix = "";
 217                         if (opt_set != null)
 218                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 219
 220                         data.stdoutFile = test + log_prefix + ".stdout";
 221                         data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create));
 222
 223                         data.stderrFile = test + log_prefix + ".stderr";
 224                         data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create));
 225
 226                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 227                                 Process p2 = (Process)sender;
 228
 229                                 StreamWriter fs;
 230
 231                                 lock (monitor) {
 232                                         fs = process_data [p2].stdout;
 233
 234                                         if (e.Data == null)
 235                                                 process_data [p2].stdout = null;
 236                                 }
 237
 238                                 if (e.Data == null) {
 239                                         fs.Close ();
 240                                 } else {
 241                                         fs.WriteLine (e.Data);
 242                                         fs.Flush ();
 243                                 }
 244                         };
 245
 246                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 247                                 Process p2 = (Process)sender;
 248
 249                                 StreamWriter fs;
 250
 251                                 lock (monitor) {
 252                                         fs = process_data [p2].stderr;
 253
 254                                         if (e.Data == null)
 255                                                 process_data [p2].stderr = null;
 256
 257                                 }
 258
 259                                 if (e.Data == null) {
 260                                         fs.Close ();
 261
 262                                         lock (monitor) {
 263                                                 process_data [p2].stderr = null;
 264                                         }
 265                                 } else {
 266                                         fs.WriteLine (e.Data);
 267                                         fs.Flush ();
 268                                 }
 269                         };
 270
 271                         lock (monitor) {
 272                                 processes.Add (p);
 273                                 process_data [p] = data;
 274                         }
 275                         p.Start ();
 276
 277                         p.BeginOutputReadLine ();
 278                         p.BeginErrorReadLine ();
 279
 280                         ThreadPool.QueueUserWorkItem (o => {
 281                                 Process process = (Process) o;
 282
 283                                 process.WaitForExit ();
 284
 285                                 lock (monitor) {
 286                                         if (process.ExitCode == expectedExitCode) {
 287                                                 if (concurrency == 1)
 288                                                         Console.WriteLine ("passed.");
 289                                                 else
 290                                                         Console.Write (".");
 291                                                 passed.Add(process_data [process]);
 292                                                 npassed ++;
 293                                         } else {
 294                                                 if (concurrency == 1)
 295                                                         Console.WriteLine ("failed.");
 296                                                 else
 297                                                         Console.Write ("F");
 298                                                 failed.Add (process_data [process]);
 299                                                 nfailed ++;
 300                                         }
 301                                         processes.Remove (process);
 302                                         terminated.Add (process);
 303                                         Monitor.Pulse (monitor);
 304                                 }
 305                         }, p);
 306                 }
 307
 308                 bool timed_out = false;
 309
 310                 /* Wait for all processes to terminate */
 311                 while (true) {
 312                         lock (monitor) {
 313                                 int nprocesses = processes.Count;
 314
 315                                 if (nprocesses == 0)
 316                                         break;
 317
 318                                 bool res = Monitor.Wait (monitor, 1000 * timeout);
 319                                 if (!res) {
 320                                         timed_out = true;
 321                                         break;
 322                                 }
 323                         }
 324                 }
 325
 326                 TimeSpan test_time = DateTime.UtcNow - test_start_time;
 327                 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
 328                 xmlWriterSettings.NewLineOnAttributes = true;
 329                 xmlWriterSettings.Indent = true;
 330                 using (XmlWriter writer = XmlWriter.Create ("TestResults_runtime.xml", xmlWriterSettings)) {
 331                         // <?xml version="1.0" encoding="utf-8" standalone="no"?>
 332                         writer.WriteStartDocument ();
 333                         // <!--This file represents the results of running a test suite-->
 334                         writer.WriteComment ("This file represents the results of running a test suite");
 335                         // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
 336                         writer.WriteStartElement ("test-results");
 337                         writer.WriteAttributeString ("name", "runtime-tests.dummy");
 338                         writer.WriteAttributeString ("total", (npassed + nfailed).ToString());
 339                         writer.WriteAttributeString ("failures", nfailed.ToString());
 340                         writer.WriteAttributeString ("not-run", "0");
 341                         writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 342                         writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 343                         //   <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
 344                         writer.WriteStartElement ("environment");
 345                         writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
 346                         writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
 347                         writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
 348                         writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
 349                         writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
 350                         writer.WriteAttributeString ("machine-name", Environment.MachineName );
 351                         writer.WriteAttributeString ("user", Environment.UserName );
 352                         writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
 353                         writer.WriteEndElement ();
 354                         //   <culture-info current-culture="en-GB" current-uiculture="en-GB" />
 355                         writer.WriteStartElement ("culture-info");
 356                         writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
 357                         writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
 358                         writer.WriteEndElement ();
 359                         //   <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
 360                         writer.WriteStartElement ("test-suite");
 361                         writer.WriteAttributeString ("name","runtime-tests.dummy");
 362                         writer.WriteAttributeString ("success", (nfailed == 0).ToString());
 363                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 364                         writer.WriteAttributeString ("asserts", nfailed.ToString());
 365                         //     <results>
 366                         writer.WriteStartElement ("results");
 367                         //       <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 368                         writer.WriteStartElement ("test-suite");
 369                         writer.WriteAttributeString ("name","MonoTests");
 370                         writer.WriteAttributeString ("success", (nfailed == 0).ToString());
 371                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 372                         writer.WriteAttributeString ("asserts", nfailed.ToString());
 373                         //         <results>
 374                         writer.WriteStartElement ("results");
 375                         //           <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 376                         writer.WriteStartElement ("test-suite");
 377                         writer.WriteAttributeString ("name","runtime");
 378                         writer.WriteAttributeString ("success", (nfailed == 0).ToString());
 379                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 380                         writer.WriteAttributeString ("asserts", nfailed.ToString());
 381                         //             <results>
 382                         writer.WriteStartElement ("results");
 383                         // Dump all passing tests first
 384                         foreach (ProcessData pd in passed) {
 385                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 386                                 writer.WriteStartElement ("test-case");
 387                                 writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test);
 388                                 writer.WriteAttributeString ("executed", "True");
 389                                 writer.WriteAttributeString ("success", "True");
 390                                 writer.WriteAttributeString ("time", "0");
 391                                 writer.WriteAttributeString ("asserts", "0");
 392                                 writer.WriteEndElement ();
 393                         }
 394                         // Now dump all failing tests
 395                         foreach (ProcessData pd in failed) {
 396                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 397                                 writer.WriteStartElement ("test-case");
 398                                 writer.WriteAttributeString ("name", "MonoTests.runtime." + pd.test);
 399                                 writer.WriteAttributeString ("executed", "True");
 400                                 writer.WriteAttributeString ("success", "False");
 401                                 writer.WriteAttributeString ("time", "0");
 402                                 writer.WriteAttributeString ("asserts", "1");
 403                                 writer.WriteStartElement ("failure");
 404                                 writer.WriteStartElement ("message");
 405                                 writer.WriteCData (DumpPseudoTrace (pd.stdoutFile));
 406                                 writer.WriteEndElement ();
 407                                 writer.WriteStartElement ("stack-trace");
 408                                 writer.WriteCData (DumpPseudoTrace (pd.stderrFile));
 409                                 writer.WriteEndElement ();
 410                                 writer.WriteEndElement ();
 411                                 writer.WriteEndElement ();
 412                         }
 413                         //             </results>
 414                         writer.WriteEndElement ();
 415                         //           </test-suite>
 416                         writer.WriteEndElement ();
 417                         //         </results>
 418                         writer.WriteEndElement ();
 419                         //       </test-suite>
 420                         writer.WriteEndElement ();
 421                         //     </results>
 422                         writer.WriteEndElement ();
 423                         //   </test-suite>
 424                         writer.WriteEndElement ();
 425                         // </test-results>
 426                         writer.WriteEndElement ();
 427                         writer.WriteEndDocument ();
 428                 }
 429
 430                 Console.WriteLine ();
 431
 432                 if (timed_out) {
 433                         Console.WriteLine ("\nrunning tests timed out:\n");
 434                         Console.WriteLine (npassed + nfailed);
 435                         lock (monitor) {
 436                                 foreach (Process p in processes) {
 437                                         ProcessData pd = process_data [p];
 438                                         pd.CloseStreams ();
 439                                         Console.WriteLine (pd.test);
 440                                         p.Kill ();
 441                                         DumpFile (pd.stdoutFile);
 442                                         DumpFile (pd.stderrFile);
 443                                 }
 444                         }
 445                         return 1;
 446                 }
 447
 448                 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
 449                 if (nfailed > 0) {
 450                         Console.WriteLine ("\nFailed tests:\n");
 451                         foreach (ProcessData pd in failed) {
 452                                 Console.WriteLine (pd.test);
 453                                 DumpFile (pd.stdoutFile);
 454                                 DumpFile (pd.stderrFile);
 455                         }
 456                         return 1;
 457                 } else {
 458                         return 0;
 459                 }
 460         }
 461
 462         static void DumpFile (string filename) {
 463                 if (File.Exists (filename)) {
 464                         Console.WriteLine ("=============== {0} ===============", filename);
 465                         Console.WriteLine (File.ReadAllText (filename));
 466                         Console.WriteLine ("=============== EOF ===============");
 467                 }
 468         }
 469
 470         static string DumpPseudoTrace (string filename) {
 471                 if (File.Exists (filename))
 472                         return File.ReadAllText (filename);
 473                 else
 474                         return string.Empty;
 475         }
 476 }