mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Threading;
  31 using System.Diagnostics;
  32 using System.Collections.Generic;
  33 using System.Globalization;
  34 using System.Xml;
  35
  36 //
  37 // This is a simple test runner with support for parallel execution
  38 //
  39
  40 public class TestRunner
  41 {
  42         class ProcessData {
  43                 public string test;
  44                 public StreamWriter stdout, stderr;
  45                 public string stdoutFile, stderrFile;
  46         }
  47
  48         class TestInfo {
  49                 public string test, opt_set;
  50         }
  51
  52         public static int Main (String[] args) {
  53                 // Defaults
  54                 int concurrency = 1;
  55                 int timeout = 2 * 60; // in seconds
  56                 int expectedExitCode = 0;
  57                 string testsuiteName = "runtime";
  58
  59                 DateTime test_start_time = DateTime.UtcNow;
  60
  61                 // FIXME: Add support for runtime arguments + env variables
  62
  63                 string disabled_tests = null;
  64                 string runtime = "mono";
  65                 var opt_sets = new List<string> ();
  66
  67                 // Process options
  68                 int i = 0;
  69                 while (i < args.Length) {
  70                         if (args [i].StartsWith ("-")) {
  71                                 if (args [i] == "-j") {
  72                                         if (i + i >= args.Length) {
  73                                                 Console.WriteLine ("Missing argument to -j command line option.");
  74                                                 return 1;
  75                                         }
  76                                         if (args [i + 1] == "a")
  77                                                 concurrency = Environment.ProcessorCount;
  78                                         else
  79                                                 concurrency = Int32.Parse (args [i + 1]);
  80                                         i += 2;
  81                                 } else if (args [i] == "--timeout") {
  82                                         if (i + i >= args.Length) {
  83                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  84                                                 return 1;
  85                                         }
  86                                         timeout = Int32.Parse (args [i + 1]);
  87                                         i += 2;
  88                                 } else if (args [i] == "--disabled") {
  89                                         if (i + i >= args.Length) {
  90                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  91                                                 return 1;
  92                                         }
  93                                         disabled_tests = args [i + 1];
  94                                         i += 2;
  95                                 } else if (args [i] == "--runtime") {
  96                                         if (i + i >= args.Length) {
  97                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  98                                                 return 1;
  99                                         }
 100                                         runtime = args [i + 1];
 101                                         i += 2;
 102                                 } else if (args [i] == "--opt-sets") {
 103                                         if (i + i >= args.Length) {
 104                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 105                                                 return 1;
 106                                         }
 107                                         foreach (var s in args [i + 1].Split ())
 108                                                 opt_sets.Add (s);
 109                                         i += 2;
 110                                 } else if (args [i] == "--expected-exit-code") {
 111                                         if (i + i >= args.Length) {
 112                                                 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
 113                                                 return 1;
 114                                         }
 115                                         expectedExitCode = Int32.Parse (args [i + 1]);
 116                                         i += 2;
 117                                 } else if (args [i] == "--testsuite-name") {
 118                                         if (i + i >= args.Length) {
 119                                                 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
 120                                                 return 1;
 121                                         }
 122                                         testsuiteName = args [i + 1];
 123                                         i += 2;
 124                                 } else {
 125                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 126                                         return 1;
 127                                 }
 128                         } else {
 129                                 break;
 130                         }
 131                 }
 132
 133                 var disabled = new Dictionary <string, string> ();
 134
 135                 if (disabled_tests != null) {
 136                         foreach (string test in disabled_tests.Split ())
 137                                 disabled [test] = test;
 138                 }
 139
 140                 // The remaining arguments are the tests
 141                 var tests = new List<string> ();
 142                 for (int j = i; j < args.Length; ++j)
 143                         if (!disabled.ContainsKey (args [j]))
 144                                 tests.Add (args [j]);
 145
 146                 var passed = new List<ProcessData> ();
 147                 var failed = new List<ProcessData> ();
 148                 var timedout = new List<ProcessData> ();
 149
 150                 object monitor = new object ();
 151
 152                 if (concurrency != 1)
 153                         Console.WriteLine ("Running tests: ");
 154
 155                 var test_info = new Queue<TestInfo> ();
 156                 if (opt_sets.Count == 0) {
 157                         foreach (string s in tests)
 158                                 test_info.Enqueue (new TestInfo { test = s });
 159                 } else {
 160                         foreach (string opt in opt_sets) {
 161                                 foreach (string s in tests)
 162                                         test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
 163                         }
 164                 }
 165
 166                 List<Thread> threads = new List<Thread> (concurrency);
 167
 168                 for (int j = 0; j < concurrency; ++j) {
 169                         Thread thread = new Thread (() => {
 170                                 while (true) {
 171                                         TestInfo ti;
 172
 173                                         lock (monitor) {
 174                                                 if (test_info.Count == 0)
 175                                                         break;
 176                                                 ti = test_info.Dequeue ();
 177                                         }
 178
 179                                         string test = ti.test;
 180                                         string opt_set = ti.opt_set;
 181
 182                                         if (concurrency == 1)
 183                                                 Console.Write ("Testing " + test + "... ");
 184
 185                                         /* Spawn a new process */
 186                                         string process_args;
 187                                         if (opt_set == null)
 188                                                 process_args = test;
 189                                         else
 190                                                 process_args = "-O=" + opt_set + " " + test;
 191                                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 192                                         info.UseShellExecute = false;
 193                                         info.RedirectStandardOutput = true;
 194                                         info.RedirectStandardError = true;
 195                                         Process p = new Process ();
 196                                         p.StartInfo = info;
 197
 198                                         ProcessData data = new ProcessData ();
 199                                         data.test = test;
 200
 201                                         string log_prefix = "";
 202                                         if (opt_set != null)
 203                                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 204
 205                                         data.stdoutFile = test + log_prefix + ".stdout";
 206                                         data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create));
 207
 208                                         data.stderrFile = test + log_prefix + ".stderr";
 209                                         data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create));
 210
 211                                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 212                                                 if (e.Data != null) {
 213                                                         data.stdout.WriteLine (e.Data);
 214                                                 } else {
 215                                                         data.stdout.Flush ();
 216                                                         data.stdout.Close ();
 217                                                 }
 218                                         };
 219
 220                                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 221                                                 if (e.Data != null) {
 222                                                         data.stderr.WriteLine (e.Data);
 223                                                 } else {
 224                                                         data.stderr.Flush ();
 225                                                         data.stderr.Close ();
 226                                                 }
 227                                         };
 228
 229                                         p.Start ();
 230
 231                                         p.BeginOutputReadLine ();
 232                                         p.BeginErrorReadLine ();
 233
 234                                         if (!p.WaitForExit (timeout * 1000)) {
 235                                                 lock (monitor) {
 236                                                         timedout.Add (data);
 237                                                 }
 238
 239                                                 if (concurrency == 1)
 240                                                         Console.WriteLine ("timed out.");
 241                                                 else
 242                                                         Console.Write (".");
 243
 244                                                 p.Kill ();
 245                                         } else if (p.ExitCode != expectedExitCode) {
 246                                                 lock (monitor) {
 247                                                         failed.Add (data);
 248                                                 }
 249
 250                                                 if (concurrency == 1)
 251                                                         Console.WriteLine ("failed.");
 252                                                 else
 253                                                         Console.Write (".");
 254                                         } else {
 255                                                 lock (monitor) {
 256                                                         passed.Add (data);
 257                                                 }
 258
 259                                                 if (concurrency == 1)
 260                                                         Console.WriteLine ("passed.");
 261                                                 else
 262                                                         Console.Write (".");
 263                                         }
 264                                 }
 265                         });
 266
 267                         thread.Start ();
 268
 269                         threads.Add (thread);
 270                 }
 271
 272                 for (int j = 0; j < threads.Count; ++j)
 273                         threads [j].Join ();
 274
 275                 int npassed = passed.Count;
 276                 int nfailed = failed.Count;
 277                 int ntimedout = timedout.Count;
 278
 279                 TimeSpan test_time = DateTime.UtcNow - test_start_time;
 280                 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
 281                 xmlWriterSettings.NewLineOnAttributes = true;
 282                 xmlWriterSettings.Indent = true;
 283                 using (XmlWriter writer = XmlWriter.Create (String.Format ("TestResults_{0}.xml", testsuiteName), xmlWriterSettings)) {
 284                         // <?xml version="1.0" encoding="utf-8" standalone="no"?>
 285                         writer.WriteStartDocument ();
 286                         // <!--This file represents the results of running a test suite-->
 287                         writer.WriteComment ("This file represents the results of running a test suite");
 288                         // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
 289                         writer.WriteStartElement ("test-results");
 290                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 291                         writer.WriteAttributeString ("total", (npassed + nfailed + ntimedout).ToString());
 292                         writer.WriteAttributeString ("failures", (nfailed + ntimedout).ToString());
 293                         writer.WriteAttributeString ("not-run", "0");
 294                         writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 295                         writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 296                         //   <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
 297                         writer.WriteStartElement ("environment");
 298                         writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
 299                         writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
 300                         writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
 301                         writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
 302                         writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
 303                         writer.WriteAttributeString ("machine-name", Environment.MachineName );
 304                         writer.WriteAttributeString ("user", Environment.UserName );
 305                         writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
 306                         writer.WriteEndElement ();
 307                         //   <culture-info current-culture="en-GB" current-uiculture="en-GB" />
 308                         writer.WriteStartElement ("culture-info");
 309                         writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
 310                         writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
 311                         writer.WriteEndElement ();
 312                         //   <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
 313                         writer.WriteStartElement ("test-suite");
 314                         writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
 315                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 316                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 317                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 318                         //     <results>
 319                         writer.WriteStartElement ("results");
 320                         //       <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 321                         writer.WriteStartElement ("test-suite");
 322                         writer.WriteAttributeString ("name","MonoTests");
 323                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 324                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 325                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 326                         //         <results>
 327                         writer.WriteStartElement ("results");
 328                         //           <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
 329                         writer.WriteStartElement ("test-suite");
 330                         writer.WriteAttributeString ("name", testsuiteName);
 331                         writer.WriteAttributeString ("success", (nfailed + ntimedout == 0).ToString());
 332                         writer.WriteAttributeString ("time", test_time.Seconds.ToString());
 333                         writer.WriteAttributeString ("asserts", (nfailed + ntimedout).ToString());
 334                         //             <results>
 335                         writer.WriteStartElement ("results");
 336                         // Dump all passing tests first
 337                         foreach (ProcessData pd in passed) {
 338                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 339                                 writer.WriteStartElement ("test-case");
 340                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 341                                 writer.WriteAttributeString ("executed", "True");
 342                                 writer.WriteAttributeString ("success", "True");
 343                                 writer.WriteAttributeString ("time", "0");
 344                                 writer.WriteAttributeString ("asserts", "0");
 345                                 writer.WriteEndElement ();
 346                         }
 347                         // Now dump all failing tests
 348                         foreach (ProcessData pd in failed) {
 349                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 350                                 writer.WriteStartElement ("test-case");
 351                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
 352                                 writer.WriteAttributeString ("executed", "True");
 353                                 writer.WriteAttributeString ("success", "False");
 354                                 writer.WriteAttributeString ("time", "0");
 355                                 writer.WriteAttributeString ("asserts", "1");
 356                                 writer.WriteStartElement ("failure");
 357                                 writer.WriteStartElement ("message");
 358                                 writer.WriteCData (DumpPseudoTrace (pd.stdoutFile));
 359                                 writer.WriteEndElement ();
 360                                 writer.WriteStartElement ("stack-trace");
 361                                 writer.WriteCData (DumpPseudoTrace (pd.stderrFile));
 362                                 writer.WriteEndElement ();
 363                                 writer.WriteEndElement ();
 364                                 writer.WriteEndElement ();
 365                         }
 366                         // Then dump all timing out tests
 367                         foreach (ProcessData pd in timedout) {
 368                                 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
 369                                 writer.WriteStartElement ("test-case");
 370                                 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
 371                                 writer.WriteAttributeString ("executed", "True");
 372                                 writer.WriteAttributeString ("success", "False");
 373                                 writer.WriteAttributeString ("time", "0");
 374                                 writer.WriteAttributeString ("asserts", "1");
 375                                 writer.WriteStartElement ("failure");
 376                                 writer.WriteStartElement ("message");
 377                                 writer.WriteCData (DumpPseudoTrace (pd.stdoutFile));
 378                                 writer.WriteEndElement ();
 379                                 writer.WriteStartElement ("stack-trace");
 380                                 writer.WriteCData (DumpPseudoTrace (pd.stderrFile));
 381                                 writer.WriteEndElement ();
 382                                 writer.WriteEndElement ();
 383                                 writer.WriteEndElement ();
 384                         }
 385                         //             </results>
 386                         writer.WriteEndElement ();
 387                         //           </test-suite>
 388                         writer.WriteEndElement ();
 389                         //         </results>
 390                         writer.WriteEndElement ();
 391                         //       </test-suite>
 392                         writer.WriteEndElement ();
 393                         //     </results>
 394                         writer.WriteEndElement ();
 395                         //   </test-suite>
 396                         writer.WriteEndElement ();
 397                         // </test-results>
 398                         writer.WriteEndElement ();
 399                         writer.WriteEndDocument ();
 400                 }
 401
 402                 Console.WriteLine ();
 403                 Console.WriteLine ("{0,4} test(s) passed", npassed);
 404                 Console.WriteLine ("{0,4} test(s) failed", nfailed);
 405                 Console.WriteLine ("{0,4} test(s) timed out", ntimedout);
 406
 407                 if (nfailed > 0) {
 408                         Console.WriteLine ();
 409                         Console.WriteLine ("Failed test(s):");
 410                         foreach (ProcessData pd in failed) {
 411                                 Console.WriteLine ();
 412                                 Console.WriteLine (pd.test);
 413                                 DumpFile (pd.stdoutFile);
 414                                 DumpFile (pd.stderrFile);
 415                         }
 416                 }
 417
 418                 if (ntimedout > 0) {
 419                         Console.WriteLine ();
 420                         Console.WriteLine ("Timed out test(s):");
 421                         foreach (ProcessData pd in timedout) {
 422                                 Console.WriteLine ();
 423                                 Console.WriteLine (pd.test);
 424                                 DumpFile (pd.stdoutFile);
 425                                 DumpFile (pd.stderrFile);
 426                         }
 427                 }
 428
 429                 return (ntimedout == 0 && nfailed == 0) ? 0 : 1;
 430         }
 431
 432         static void DumpFile (string filename) {
 433                 if (File.Exists (filename)) {
 434                         Console.WriteLine ("=============== {0} ===============", filename);
 435                         Console.WriteLine (File.ReadAllText (filename));
 436                         Console.WriteLine ("=============== EOF ===============");
 437                 }
 438         }
 439
 440         static string DumpPseudoTrace (string filename) {
 441                 if (File.Exists (filename))
 442                         return File.ReadAllText (filename);
 443                 else
 444                         return string.Empty;
 445         }
 446 }