mono/tests/test-runner.cs

   1 //
   2 // test-runner.cs
   3 //
   4 // Author:
   5 //   Zoltan Varga (vargaz@gmail.com)
   6 //
   7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Threading;
  31 using System.Diagnostics;
  32 using System.Collections.Generic;
  33
  34 //
  35 // This is a simple test runner with support for parallel execution
  36 //
  37
  38 public class TestRunner
  39 {
  40         class ProcessData {
  41                 public string test;
  42                 public StreamWriter stdout, stderr;
  43         }
  44
  45         public static int Main (String[] args) {
  46                 // Defaults
  47                 int concurrency = 1;
  48                 int timeout = 2 * 60; // in seconds
  49
  50                 // FIXME: Add support for runtime arguments + env variables
  51
  52                 string disabled_tests = null;
  53                 string runtime = "mono";
  54
  55                 // Process options
  56                 int i = 0;
  57                 while (i < args.Length) {
  58                         if (args [i].StartsWith ("-")) {
  59                                 if (args [i] == "-j") {
  60                                         if (i + i >= args.Length) {
  61                                                 Console.WriteLine ("Missing argument to -j command line option.");
  62                                                 return 1;
  63                                         }
  64                                         if (args [i + 1] == "a")
  65                                                 concurrency = Environment.ProcessorCount;
  66                                         else
  67                                                 concurrency = Int32.Parse (args [i + 1]);
  68                                         i += 2;
  69                                 } else if (args [i] == "--timeout") {
  70                                         if (i + i >= args.Length) {
  71                                                 Console.WriteLine ("Missing argument to --timeout command line option.");
  72                                                 return 1;
  73                                         }
  74                                         timeout = Int32.Parse (args [i + 1]);
  75                                         i += 2;
  76                                 } else if (args [i] == "--disabled") {
  77                                         if (i + i >= args.Length) {
  78                                                 Console.WriteLine ("Missing argument to --disabled command line option.");
  79                                                 return 1;
  80                                         }
  81                                         disabled_tests = args [i + 1];
  82                                         i += 2;
  83                                 } else if (args [i] == "--runtime") {
  84                                         if (i + i >= args.Length) {
  85                                                 Console.WriteLine ("Missing argument to --runtime command line option.");
  86                                                 return 1;
  87                                         }
  88                                         runtime = args [i + 1];
  89                                         i += 2;
  90                                 } else {
  91                                         Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
  92                                         return 1;
  93                                 }
  94                         } else {
  95                                 break;
  96                         }
  97                 }
  98
  99                 var disabled = new Dictionary <string, string> ();
 100
 101                 if (disabled_tests != null) {
 102                         foreach (string test in disabled_tests.Split ())
 103                                 disabled [test] = test;
 104                 }
 105
 106                 // The remaining arguments are the tests
 107                 var tests = new List<string> ();
 108                 for (int j = i; j < args.Length; ++j)
 109                         if (!disabled.ContainsKey (args [j]))
 110                                 tests.Add (args [j]);
 111
 112                 int npassed = 0;
 113                 int nfailed = 0;
 114
 115                 var processes = new List<Process> ();
 116                 var failed = new List<string> ();
 117                 var process_data = new Dictionary<Process, ProcessData> ();
 118
 119                 object monitor = new object ();
 120
 121                 if (concurrency != 1)
 122                         Console.WriteLine ("Running tests: ");
 123
 124                 foreach (string test in tests) {
 125                         lock (monitor) {
 126                                 while (processes.Count == concurrency) {
 127                                         /* Wait for one process to terminate */
 128                                         Monitor.Wait (monitor);
 129                                 }
 130                         }
 131
 132                         if (concurrency == 1)
 133                                 Console.Write ("Testing " + test + "... ");
 134
 135                         /* Spawn a new process */
 136                         ProcessStartInfo info = new ProcessStartInfo (runtime, test);
 137                         info.UseShellExecute = false;
 138                         info.RedirectStandardOutput = true;
 139                         info.RedirectStandardError = true;
 140                         Process p = new Process ();
 141                         p.StartInfo = info;
 142                         p.EnableRaisingEvents = true;
 143
 144                         ProcessData data = new ProcessData ();
 145                         data.test = test;
 146
 147                         p.Exited += delegate (object sender, EventArgs e) {
 148                                 // Anon methods share some of their state, so we can't use
 149                                 // variables which change during the loop (test, p)
 150                                 Process dead = (Process)sender;
 151
 152                                 lock (monitor) {
 153                                         if (dead.ExitCode == 0) {
 154                                                 if (concurrency == 1)
 155                                                         Console.WriteLine ("passed.");
 156                                                 else
 157                                                         Console.Write (".");
 158                                                 npassed ++;
 159                                         } else {
 160                                                 if (concurrency == 1)
 161                                                         Console.WriteLine ("failed.");
 162                                                 else
 163                                                         Console.Write ("F");
 164                                                 failed.Add (process_data [dead].test);
 165                                                 nfailed ++;
 166                                         }
 167                                         processes.Remove (dead);
 168                                         process_data [dead].stdout.Close ();
 169                                         process_data [dead].stderr.Close ();
 170                                         // This is needed to avoid CreateProcess failed errors :(
 171                                         dead.Close ();
 172                                         Monitor.Pulse (monitor);
 173                                 }
 174                         };
 175
 176                         data.stdout = new StreamWriter (new FileStream (test + ".stdout", FileMode.Create));
 177
 178                         data.stderr = new StreamWriter (new FileStream (test + ".stderr", FileMode.Create));
 179
 180                         p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 181                                 Process p2 = (Process)sender;
 182
 183                                 StreamWriter fs;
 184
 185                                 lock (monitor) {
 186                                         fs = process_data [p2].stdout;
 187                                 }
 188
 189                                 if (String.IsNullOrEmpty (e.Data))
 190                                         fs.Close ();
 191                                 else
 192                                         fs.WriteLine (e.Data);
 193                         };
 194
 195                         p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
 196                                 Process p2 = (Process)sender;
 197
 198                                 StreamWriter fs;
 199
 200                                 lock (monitor) {
 201                                         fs = process_data [p2].stderr;
 202                                 }
 203
 204                                 if (String.IsNullOrEmpty (e.Data))
 205                                         fs.Close ();
 206                                 else
 207                                         fs.WriteLine (e.Data);
 208                         };
 209
 210                         lock (monitor) {
 211                                 processes.Add (p);
 212                                 process_data [p] = data;
 213                         }
 214                         p.Start ();
 215
 216                         p.BeginOutputReadLine ();
 217                         p.BeginErrorReadLine ();
 218                 }
 219
 220                 bool timed_out = false;
 221
 222                 /* Wait for all processes to terminate */
 223                 while (true) {
 224                         lock (monitor) {
 225                                 int nprocesses = processes.Count;
 226
 227                                 if (nprocesses == 0)
 228                                         break;
 229
 230                                 bool res = Monitor.Wait (monitor, 1000 * timeout);
 231                                 if (!res) {
 232                                         timed_out = true;
 233                                         break;
 234                                 }
 235                         }
 236                 }
 237
 238                 Console.WriteLine ();
 239
 240                 if (timed_out) {
 241                         Console.WriteLine ("\nrunning tests timed out:\n");
 242                         Console.WriteLine (npassed + nfailed);
 243                         lock (monitor) {
 244                                 foreach (Process p in processes) {
 245                                         Console.WriteLine (process_data [p].test);
 246                                 }
 247                         }
 248                         return 1;
 249                 }
 250
 251                 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
 252                 if (nfailed > 0) {
 253                         Console.WriteLine ("\nFailed tests:\n");
 254                         foreach (string s in failed)
 255                                 Console.WriteLine (s);
 256                         return 1;
 257                 } else {
 258                         return 0;
 259                 }
 260         }
 261 }