mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Threading;
  31 using System.Diagnostics;
  32 using System.Collections.Generic;
  33
  34 //
  35 // This is a simple test runner with support for parallel execution
  36 //
  37
  38 public class TestRunner
  39 {
  40         class ProcessData {
  41                 public string test;
  42                 public StreamWriter stdout, stderr;
  43                 public string stdoutFile, stderrFile;
  44
  45                 public void CloseStreams () {
  46                         if (stdout != null) {
  47                                 stdout.Close ();
  48                                 stdout = null;
  49                         }
  50                         if (stderr != null) {
  51                                 stderr.Close ();
  52                                 stderr = null;
  53                         }
  54                 }
  55         }
  56
  57         class TestInfo {
  58                 public string test, opt_set;
  59         }
  60
  61         public static int Main (String[] args) {
  62                 // Defaults
  63                 int concurrency = 1;
  64                 int timeout = 2 * 60; // in seconds
  65
  66                 // FIXME: Add support for runtime arguments + env variables
  67
  68                 string disabled_tests = null;
  69                 string runtime = "mono";
  70                 var opt_sets = new List<string> ();
  71
  72                 // Process options
  73                 int i = 0;
  74                 while (i < args.Length) {
  75                         if (args [i].StartsWith ("-")) {
  76                                 if (args [i] == "-j") {
  77                                         if (i + i >= args.Length) {
  78                                                 Console.WriteLine ("Missing argument to -j command line option.");
  79                                                 return 1;
  80                                         }
  81                                         if (args [i + 1] == "a")
  82                                                 concurrency = Environment.ProcessorCount;
  83                                         else
  84                                                 concurrency = Int32.Parse (args [i + 1]);
  85                                         i += 2;
  86                                 } else if (args [i] == "--timeout") {
  87                                         if (i + i >= args.Length) {
  88                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  89                                                 return 1;
  90                                         }
  91                                         timeout = Int32.Parse (args [i + 1]);
  92                                         i += 2;
  93                                 } else if (args [i] == "--disabled") {
  94                                         if (i + i >= args.Length) {
  95                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  96                                                 return 1;
  97                                         }
  98                                         disabled_tests = args [i + 1];
  99                                         i += 2;
 100                                 } else if (args [i] == "--runtime") {
 101                                         if (i + i >= args.Length) {
 102                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
 103                                                 return 1;
 104                                         }
 105                                         runtime = args [i + 1];
 106                                         i += 2;
 107                                 } else if (args [i] == "--opt-sets") {
 108                                         if (i + i >= args.Length) {
 109                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
 110                                                 return 1;
 111                                         }
 112                                         foreach (var s in args [i + 1].Split ())
 113                                                 opt_sets.Add (s);
 114                                         i += 2;
 115                                 } else {
 116                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 117                                         return 1;
 118                                 }
 119                         } else {
 120                                 break;
 121                         }
 122                 }
 123
 124                 var disabled = new Dictionary <string, string> ();
 125
 126                 if (disabled_tests != null) {
 127                         foreach (string test in disabled_tests.Split ())
 128                                 disabled [test] = test;
 129                 }
 130
 131                 // The remaining arguments are the tests
 132                 var tests = new List<string> ();
 133                 for (int j = i; j < args.Length; ++j)
 134                         if (!disabled.ContainsKey (args [j]))
 135                                 tests.Add (args [j]);
 136
 137                 int npassed = 0;
 138                 int nfailed = 0;
 139
 140                 var processes = new List<Process> ();
 141                 var failed = new List<ProcessData> ();
 142                 var process_data = new Dictionary<Process, ProcessData> ();
 143
 144                 object monitor = new object ();
 145
 146                 var terminated = new List<Process> ();
 147
 148                 if (concurrency != 1)
 149                         Console.WriteLine ("Running tests: ");
 150
 151                 var test_info = new List<TestInfo> ();
 152                 if (opt_sets.Count == 0) {
 153                         foreach (string s in tests)
 154                                 test_info.Add (new TestInfo { test = s });
 155                 } else {
 156                         foreach (string opt in opt_sets) {
 157                                 foreach (string s in tests)
 158                                         test_info.Add (new TestInfo { test = s, opt_set = opt });
 159                         }
 160                 }
 161
 162                 foreach (TestInfo ti in test_info) {
 163                         lock (monitor) {
 164                                 while (processes.Count == concurrency) {
 165                                         /* Wait for one process to terminate */
 166                                         Monitor.Wait (monitor);
 167                                 }
 168
 169                                 /* Cleaup terminated processes */
 170                                 foreach (Process dead in terminated) {
 171                                         if (process_data [dead].stdout != null)
 172                                                 process_data [dead].stdout.Close ();
 173                                         if (process_data [dead].stderr != null)
 174                                                 process_data [dead].stderr.Close ();
 175                                         // This is needed to avoid CreateProcess failed errors :(
 176                                         dead.Close ();
 177                                 }
 178                                 terminated.Clear ();
 179                         }
 180
 181                         string test = ti.test;
 182                         string opt_set = ti.opt_set;
 183
 184                         if (concurrency == 1)
 185                                 Console.Write ("Testing " + test + "... ");
 186
 187                         /* Spawn a new process */
 188                         string process_args;
 189                         if (opt_set == null)
 190                                 process_args = test;
 191                         else
 192                                 process_args = "-O=" + opt_set + " " + test;
 193                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 194                         info.UseShellExecute = false;
 195                         info.RedirectStandardOutput = true;
 196                         info.RedirectStandardError = true;
 197                         Process p = new Process ();
 198                         p.StartInfo = info;
 199                         p.EnableRaisingEvents = true;
 200
 201                         ProcessData data = new ProcessData ();
 202                         data.test = test;
 203
 204                         p.Exited += delegate (object sender, EventArgs e) {
 205                                 // Anon methods share some of their state, so we can't use
 206                                 // variables which change during the loop (test, p)
 207                                 Process dead = (Process)sender;
 208
 209                                 lock (monitor) {
 210                                         if (dead.ExitCode == 0) {
 211                                                 if (concurrency == 1)
 212                                                         Console.WriteLine ("passed.");
 213                                                 else
 214                                                         Console.Write (".");
 215                                                 npassed ++;
 216                                         } else {
 217                                                 if (concurrency == 1)
 218                                                         Console.WriteLine ("failed.");
 219                                                 else
 220                                                         Console.Write ("F");
 221                                                 failed.Add (process_data [dead]);
 222                                                 nfailed ++;
 223                                         }
 224                                         processes.Remove (dead);
 225                                         terminated.Add (dead);
 226                                         Monitor.Pulse (monitor);
 227                                 }
 228                         };
 229
 230                         string log_prefix = "";
 231                         if (opt_set != null)
 232                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 233
 234                         data.stdoutFile = test + log_prefix + ".stdout";
 235                         data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create));
 236
 237                         data.stderrFile = test + log_prefix + ".stderr";
 238                         data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create));
 239
 240                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 241                                 Process p2 = (Process)sender;
 242
 243                                 StreamWriter fs;
 244
 245                                 lock (monitor) {
 246                                         fs = process_data [p2].stdout;
 247
 248                                         if (String.IsNullOrEmpty (e.Data))
 249                                                 process_data [p2].stdout = null;
 250                                 }
 251
 252                                 if (String.IsNullOrEmpty (e.Data)) {
 253                                         fs.Close ();
 254                                 } else {
 255                                         fs.WriteLine (e.Data);
 256                                         fs.Flush ();
 257                                 }
 258                         };
 259
 260                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 261                                 Process p2 = (Process)sender;
 262
 263                                 StreamWriter fs;
 264
 265                                 lock (monitor) {
 266                                         fs = process_data [p2].stderr;
 267
 268                                         if (String.IsNullOrEmpty (e.Data))
 269                                                 process_data [p2].stderr = null;
 270
 271                                 }
 272
 273                                 if (String.IsNullOrEmpty (e.Data)) {
 274                                         fs.Close ();
 275
 276                                         lock (monitor) {
 277                                                 process_data [p2].stderr = null;
 278                                         }
 279                                 } else {
 280                                         fs.WriteLine (e.Data);
 281                                         fs.Flush ();
 282                                 }
 283                         };
 284
 285                         lock (monitor) {
 286                                 processes.Add (p);
 287                                 process_data [p] = data;
 288                         }
 289                         p.Start ();
 290
 291                         p.BeginOutputReadLine ();
 292                         p.BeginErrorReadLine ();
 293                 }
 294
 295                 bool timed_out = false;
 296
 297                 /* Wait for all processes to terminate */
 298                 while (true) {
 299                         lock (monitor) {
 300                                 int nprocesses = processes.Count;
 301
 302                                 if (nprocesses == 0)
 303                                         break;
 304
 305                                 bool res = Monitor.Wait (monitor, 1000 * timeout);
 306                                 if (!res) {
 307                                         timed_out = true;
 308                                         break;
 309                                 }
 310                         }
 311                 }
 312
 313                 Console.WriteLine ();
 314
 315                 if (timed_out) {
 316                         Console.WriteLine ("\nrunning tests timed out:\n");
 317                         Console.WriteLine (npassed + nfailed);
 318                         lock (monitor) {
 319                                 foreach (Process p in processes) {
 320                                         ProcessData pd = process_data [p];
 321                                         pd.CloseStreams ();
 322                                         Console.WriteLine (pd.test);
 323                                         p.Kill ();
 324                                         DumpFile (pd.stdoutFile);
 325                                         DumpFile (pd.stderrFile);
 326                                 }
 327                         }
 328                         return 1;
 329                 }
 330
 331                 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
 332                 if (nfailed > 0) {
 333                         Console.WriteLine ("\nFailed tests:\n");
 334                         foreach (ProcessData pd in failed) {
 335                                 Console.WriteLine (pd.test);
 336                                 DumpFile (pd.stdoutFile);
 337                                 DumpFile (pd.stderrFile);
 338                         }
 339                         return 1;
 340                 } else {
 341                         return 0;
 342                 }
 343         }
 344
 345         static void DumpFile (string filename) {
 346                 if (File.Exists (filename)) {
 347                         Console.WriteLine ("=============== {0} ===============", filename);
 348                         Console.WriteLine (File.ReadAllText (filename));
 349                         Console.WriteLine ("=============== EOF ===============");
 350                 }
 351         }
 352 }