mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Threading;
  31 using System.Diagnostics;
  32 using System.Collections.Generic;
  33
  34 //
  35 // This is a simple test runner with support for parallel execution
  36 //
  37
  38 public class TestRunner
  39 {
  40         class ProcessData {
  41                 public string test;
  42                 public StreamWriter stdout, stderr;
  43         }
  44
  45         class TestInfo {
  46                 public string test, opt_set;
  47         }
  48
  49         public static int Main (String[] args) {
  50                 // Defaults
  51                 int concurrency = 1;
  52                 int timeout = 2 * 60; // in seconds
  53
  54                 // FIXME: Add support for runtime arguments + env variables
  55
  56                 string disabled_tests = null;
  57                 string runtime = "mono";
  58                 var opt_sets = new List<string> ();
  59
  60                 // Process options
  61                 int i = 0;
  62                 while (i < args.Length) {
  63                         if (args [i].StartsWith ("-")) {
  64                                 if (args [i] == "-j") {
  65                                         if (i + i >= args.Length) {
  66                                                 Console.WriteLine ("Missing argument to -j command line option.");
  67                                                 return 1;
  68                                         }
  69                                         if (args [i + 1] == "a")
  70                                                 concurrency = Environment.ProcessorCount;
  71                                         else
  72                                                 concurrency = Int32.Parse (args [i + 1]);
  73                                         i += 2;
  74                                 } else if (args [i] == "--timeout") {
  75                                         if (i + i >= args.Length) {
  76                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  77                                                 return 1;
  78                                         }
  79                                         timeout = Int32.Parse (args [i + 1]);
  80                                         i += 2;
  81                                 } else if (args [i] == "--disabled") {
  82                                         if (i + i >= args.Length) {
  83                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  84                                                 return 1;
  85                                         }
  86                                         disabled_tests = args [i + 1];
  87                                         i += 2;
  88                                 } else if (args [i] == "--runtime") {
  89                                         if (i + i >= args.Length) {
  90                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  91                                                 return 1;
  92                                         }
  93                                         runtime = args [i + 1];
  94                                         i += 2;
  95                                 } else if (args [i] == "--opt-sets") {
  96                                         if (i + i >= args.Length) {
  97                                                 Console.WriteLine ("Missing argument to --opt-sets command line option.");
  98                                                 return 1;
  99                                         }
 100                                         foreach (var s in args [i + 1].Split ())
 101                                                 opt_sets.Add (s);
 102                                         i += 2;
 103                                 } else {
 104                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
 105                                         return 1;
 106                                 }
 107                         } else {
 108                                 break;
 109                         }
 110                 }
 111
 112                 var disabled = new Dictionary <string, string> ();
 113
 114                 if (disabled_tests != null) {
 115                         foreach (string test in disabled_tests.Split ())
 116                                 disabled [test] = test;
 117                 }
 118
 119                 // The remaining arguments are the tests
 120                 var tests = new List<string> ();
 121                 for (int j = i; j < args.Length; ++j)
 122                         if (!disabled.ContainsKey (args [j]))
 123                                 tests.Add (args [j]);
 124
 125                 int npassed = 0;
 126                 int nfailed = 0;
 127
 128                 var processes = new List<Process> ();
 129                 var failed = new List<string> ();
 130                 var process_data = new Dictionary<Process, ProcessData> ();
 131
 132                 object monitor = new object ();
 133
 134                 var terminated = new List<Process> ();
 135
 136                 if (concurrency != 1)
 137                         Console.WriteLine ("Running tests: ");
 138
 139                 var test_info = new List<TestInfo> ();
 140                 if (opt_sets.Count == 0) {
 141                         foreach (string s in tests)
 142                                 test_info.Add (new TestInfo { test = s });
 143                 } else {
 144                         foreach (string opt in opt_sets) {
 145                                 foreach (string s in tests)
 146                                         test_info.Add (new TestInfo { test = s, opt_set = opt });
 147                         }
 148                 }
 149
 150                 foreach (TestInfo ti in test_info) {
 151                         lock (monitor) {
 152                                 while (processes.Count == concurrency) {
 153                                         /* Wait for one process to terminate */
 154                                         Monitor.Wait (monitor);
 155                                 }
 156
 157                                 /* Cleaup terminated processes */
 158                                 foreach (Process dead in terminated) {
 159                                         if (process_data [dead].stdout != null)
 160                                                 process_data [dead].stdout.Close ();
 161                                         if (process_data [dead].stderr != null)
 162                                                 process_data [dead].stderr.Close ();
 163                                         // This is needed to avoid CreateProcess failed errors :(
 164                                         dead.Close ();
 165                                 }
 166                                 terminated.Clear ();
 167                         }
 168
 169                         string test = ti.test;
 170                         string opt_set = ti.opt_set;
 171
 172                         if (concurrency == 1)
 173                                 Console.Write ("Testing " + test + "... ");
 174
 175                         /* Spawn a new process */
 176                         string process_args;
 177                         if (opt_set == null)
 178                                 process_args = test;
 179                         else
 180                                 process_args = "-O=" + opt_set + " " + test;
 181                         ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
 182                         info.UseShellExecute = false;
 183                         info.RedirectStandardOutput = true;
 184                         info.RedirectStandardError = true;
 185                         Process p = new Process ();
 186                         p.StartInfo = info;
 187                         p.EnableRaisingEvents = true;
 188
 189                         ProcessData data = new ProcessData ();
 190                         data.test = test;
 191
 192                         p.Exited += delegate (object sender, EventArgs e) {
 193                                 // Anon methods share some of their state, so we can't use
 194                                 // variables which change during the loop (test, p)
 195                                 Process dead = (Process)sender;
 196
 197                                 lock (monitor) {
 198                                         if (dead.ExitCode == 0) {
 199                                                 if (concurrency == 1)
 200                                                         Console.WriteLine ("passed.");
 201                                                 else
 202                                                         Console.Write (".");
 203                                                 npassed ++;
 204                                         } else {
 205                                                 if (concurrency == 1)
 206                                                         Console.WriteLine ("failed.");
 207                                                 else
 208                                                         Console.Write ("F");
 209                                                 failed.Add (process_data [dead].test);
 210                                                 nfailed ++;
 211                                         }
 212                                         processes.Remove (dead);
 213                                         terminated.Add (dead);
 214                                         Monitor.Pulse (monitor);
 215                                 }
 216                         };
 217
 218                         string log_prefix = "";
 219                         if (opt_set != null)
 220                                 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
 221
 222                         data.stdout = new StreamWriter (new FileStream (test + log_prefix + ".stdout", FileMode.Create));
 223
 224                         data.stderr = new StreamWriter (new FileStream (test + log_prefix + ".stderr", FileMode.Create));
 225
 226                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 227                                 Process p2 = (Process)sender;
 228
 229                                 StreamWriter fs;
 230
 231                                 lock (monitor) {
 232                                         fs = process_data [p2].stdout;
 233
 234                                         if (String.IsNullOrEmpty (e.Data))
 235                                                 process_data [p2].stdout = null;
 236                                 }
 237
 238                                 if (String.IsNullOrEmpty (e.Data))
 239                                         fs.Close ();
 240                                 else
 241                                         fs.WriteLine (e.Data);
 242                         };
 243
 244                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 245                                 Process p2 = (Process)sender;
 246
 247                                 StreamWriter fs;
 248
 249                                 lock (monitor) {
 250                                         fs = process_data [p2].stderr;
 251
 252                                         if (String.IsNullOrEmpty (e.Data))
 253                                                 process_data [p2].stderr = null;
 254
 255                                 }
 256
 257                                 if (String.IsNullOrEmpty (e.Data)) {
 258                                         fs.Close ();
 259
 260                                         lock (monitor) {
 261                                                 process_data [p2].stderr = null;
 262                                         }
 263                                 }
 264                                 else
 265                                         fs.WriteLine (e.Data);
 266                         };
 267
 268                         lock (monitor) {
 269                                 processes.Add (p);
 270                                 process_data [p] = data;
 271                         }
 272                         p.Start ();
 273
 274                         p.BeginOutputReadLine ();
 275                         p.BeginErrorReadLine ();
 276                 }
 277
 278                 bool timed_out = false;
 279
 280                 /* Wait for all processes to terminate */
 281                 while (true) {
 282                         lock (monitor) {
 283                                 int nprocesses = processes.Count;
 284
 285                                 if (nprocesses == 0)
 286                                         break;
 287
 288                                 bool res = Monitor.Wait (monitor, 1000 * timeout);
 289                                 if (!res) {
 290                                         timed_out = true;
 291                                         break;
 292                                 }
 293                         }
 294                 }
 295
 296                 Console.WriteLine ();
 297
 298                 if (timed_out) {
 299                         Console.WriteLine ("\nrunning tests timed out:\n");
 300                         Console.WriteLine (npassed + nfailed);
 301                         lock (monitor) {
 302                                 foreach (Process p in processes) {
 303                                         Console.WriteLine (process_data [p].test);
 304                                 }
 305                         }
 306                         return 1;
 307                 }
 308
 309                 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
 310                 if (nfailed > 0) {
 311                         Console.WriteLine ("\nFailed tests:\n");
 312                         foreach (string s in failed)
 313                                 Console.WriteLine (s);
 314                         return 1;
 315                 } else {
 316                         return 0;
 317                 }
 318         }
 319 }