mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Threading;
  31 using System.Diagnostics;
  32 using System.Collections.Generic;
  33
  34 //
  35 // This is a simple test runner with support for parallel execution
  36 //
  37
  38 public class TestRunner
  39 {
  40         class ProcessData {
  41                 public string test;
  42                 public StreamWriter stdout, stderr;
  43         }
  44
  45         public static int Main (String[] args) {
  46                 // Defaults
  47                 int concurrency = 1;
  48                 int timeout = 2 * 60; // in seconds
  49
  50                 // FIXME: Add support for runtime arguments + env variables
  51
  52                 string disabled_tests = null;
  53                 string runtime = "mono";
  54
  55                 // Process options
  56                 int i = 0;
  57                 while (i < args.Length) {
  58                         if (args [i].StartsWith ("-")) {
  59                                 if (args [i] == "-j") {
  60                                         if (i + i >= args.Length) {
  61                                                 Console.WriteLine ("Missing argument to -j command line option.");
  62                                                 return 1;
  63                                         }
  64                                         if (args [i + 1] == "a")
  65                                                 concurrency = Environment.ProcessorCount;
  66                                         else
  67                                                 concurrency = Int32.Parse (args [i + 1]);
  68                                         i += 2;
  69                                 } else if (args [i] == "--timeout") {
  70                                         if (i + i >= args.Length) {
  71                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  72                                                 return 1;
  73                                         }
  74                                         timeout = Int32.Parse (args [i + 1]);
  75                                         i += 2;
  76                                 } else if (args [i] == "--disabled") {
  77                                         if (i + i >= args.Length) {
  78                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  79                                                 return 1;
  80                                         }
  81                                         disabled_tests = args [i + 1];
  82                                         i += 2;
  83                                 } else if (args [i] == "--runtime") {
  84                                         if (i + i >= args.Length) {
  85                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  86                                                 return 1;
  87                                         }
  88                                         runtime = args [i + 1];
  89                                         i += 2;
  90                                 } else {
  91                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
  92                                         return 1;
  93                                 }
  94                         } else {
  95                                 break;
  96                         }
  97                 }
  98
  99                 var disabled = new Dictionary <string, string> ();
 100
 101                 if (disabled_tests != null) {
 102                         foreach (string test in disabled_tests.Split ())
 103                                 disabled [test] = test;
 104                 }
 105
 106                 // The remaining arguments are the tests
 107                 var tests = new List<string> ();
 108                 for (int j = i; j < args.Length; ++j)
 109                         if (!disabled.ContainsKey (args [j]))
 110                                 tests.Add (args [j]);
 111
 112                 int npassed = 0;
 113                 int nfailed = 0;
 114
 115                 var processes = new List<Process> ();
 116                 var failed = new List<string> ();
 117                 var process_data = new Dictionary<Process, ProcessData> ();
 118
 119                 object monitor = new object ();
 120
 121                 var terminated = new List<Process> ();
 122
 123                 if (concurrency != 1)
 124                         Console.WriteLine ("Running tests: ");
 125
 126                 foreach (string test in tests) {
 127                         lock (monitor) {
 128                                 while (processes.Count == concurrency) {
 129                                         /* Wait for one process to terminate */
 130                                         Monitor.Wait (monitor);
 131                                 }
 132
 133                                 /* Cleaup terminated processes */
 134                                 foreach (Process dead in terminated) {
 135                                         if (process_data [dead].stdout != null)
 136                                                 process_data [dead].stdout.Close ();
 137                                         if (process_data [dead].stderr != null)
 138                                                 process_data [dead].stderr.Close ();
 139                                         // This is needed to avoid CreateProcess failed errors :(
 140                                         dead.Close ();
 141                                 }
 142                                 terminated.Clear ();
 143                         }
 144
 145                         if (concurrency == 1)
 146                                 Console.Write ("Testing " + test + "... ");
 147
 148                         /* Spawn a new process */
 149                         ProcessStartInfo info = new ProcessStartInfo (runtime, test);
 150                         info.UseShellExecute = false;
 151                         info.RedirectStandardOutput = true;
 152                         info.RedirectStandardError = true;
 153                         Process p = new Process ();
 154                         p.StartInfo = info;
 155                         p.EnableRaisingEvents = true;
 156
 157                         ProcessData data = new ProcessData ();
 158                         data.test = test;
 159
 160                         p.Exited += delegate (object sender, EventArgs e) {
 161                                 // Anon methods share some of their state, so we can't use
 162                                 // variables which change during the loop (test, p)
 163                                 Process dead = (Process)sender;
 164
 165                                 lock (monitor) {
 166                                         if (dead.ExitCode == 0) {
 167                                                 if (concurrency == 1)
 168                                                         Console.WriteLine ("passed.");
 169                                                 else
 170                                                         Console.Write (".");
 171                                                 npassed ++;
 172                                         } else {
 173                                                 if (concurrency == 1)
 174                                                         Console.WriteLine ("failed.");
 175                                                 else
 176                                                         Console.Write ("F");
 177                                                 failed.Add (process_data [dead].test);
 178                                                 nfailed ++;
 179                                         }
 180                                         processes.Remove (dead);
 181                                         terminated.Add (dead);
 182                                         Monitor.Pulse (monitor);
 183                                 }
 184                         };
 185
 186                         data.stdout = new StreamWriter (new FileStream (test + ".stdout", FileMode.Create));
 187
 188                         data.stderr = new StreamWriter (new FileStream (test + ".stderr", FileMode.Create));
 189
 190                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 191                                 Process p2 = (Process)sender;
 192
 193                                 StreamWriter fs;
 194
 195                                 lock (monitor) {
 196                                         fs = process_data [p2].stdout;
 197
 198                                         if (String.IsNullOrEmpty (e.Data))
 199                                                 process_data [p2].stdout = null;
 200                                 }
 201
 202                                 if (String.IsNullOrEmpty (e.Data))
 203                                         fs.Close ();
 204                                 else
 205                                         fs.WriteLine (e.Data);
 206                         };
 207
 208                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 209                                 Process p2 = (Process)sender;
 210
 211                                 StreamWriter fs;
 212
 213                                 lock (monitor) {
 214                                         fs = process_data [p2].stderr;
 215
 216                                         if (String.IsNullOrEmpty (e.Data))
 217                                                 process_data [p2].stderr = null;
 218
 219                                 }
 220
 221                                 if (String.IsNullOrEmpty (e.Data)) {
 222                                         fs.Close ();
 223
 224                                         lock (monitor) {
 225                                                 process_data [p2].stderr = null;
 226                                         }
 227                                 }
 228                                 else
 229                                         fs.WriteLine (e.Data);
 230                         };
 231
 232                         lock (monitor) {
 233                                 processes.Add (p);
 234                                 process_data [p] = data;
 235                         }
 236                         p.Start ();
 237
 238                         p.BeginOutputReadLine ();
 239                         p.BeginErrorReadLine ();
 240                 }
 241
 242                 bool timed_out = false;
 243
 244                 /* Wait for all processes to terminate */
 245                 while (true) {
 246                         lock (monitor) {
 247                                 int nprocesses = processes.Count;
 248
 249                                 if (nprocesses == 0)
 250                                         break;
 251
 252                                 bool res = Monitor.Wait (monitor, 1000 * timeout);
 253                                 if (!res) {
 254                                         timed_out = true;
 255                                         break;
 256                                 }
 257                         }
 258                 }
 259
 260                 Console.WriteLine ();
 261
 262                 if (timed_out) {
 263                         Console.WriteLine ("\nrunning tests timed out:\n");
 264                         Console.WriteLine (npassed + nfailed);
 265                         lock (monitor) {
 266                                 foreach (Process p in processes) {
 267                                         Console.WriteLine (process_data [p].test);
 268                                 }
 269                         }
 270                         return 1;
 271                 }
 272
 273                 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
 274                 if (nfailed > 0) {
 275                         Console.WriteLine ("\nFailed tests:\n");
 276                         foreach (string s in failed)
 277                                 Console.WriteLine (s);
 278                         return 1;
 279                 } else {
 280                         return 0;
 281                 }
 282         }
 283 }