5 // Zoltan Varga (vargaz@gmail.com)
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
9 // Permission is hereby granted, free of charge, to any person obtaining
10 // a copy of this software and associated documentation files (the
11 // "Software"), to deal in the Software without restriction, including
12 // without limitation the rights to use, copy, modify, merge, publish,
13 // distribute, sublicense, and/or sell copies of the Software, and to
14 // permit persons to whom the Software is furnished to do so, subject to
15 // the following conditions:
17 // The above copyright notice and this permission notice shall be
18 // included in all copies or substantial portions of the Software.
20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 using System.Threading;
31 using System.Diagnostics;
32 using System.Collections.Generic;
35 // This is a simple test runner with support for parallel execution
38 public class TestRunner
42 public StreamWriter stdout, stderr;
46 public string test, opt_set;
49 public static int Main (String[] args) {
52 int timeout = 2 * 60; // in seconds
54 // FIXME: Add support for runtime arguments + env variables
56 string disabled_tests = null;
57 string runtime = "mono";
58 var opt_sets = new List<string> ();
62 while (i < args.Length) {
63 if (args [i].StartsWith ("-")) {
64 if (args [i] == "-j") {
65 if (i + i >= args.Length) {
66 Console.WriteLine ("Missing argument to -j command line option.");
69 if (args [i + 1] == "a")
70 concurrency = Environment.ProcessorCount;
72 concurrency = Int32.Parse (args [i + 1]);
74 } else if (args [i] == "--timeout") {
75 if (i + i >= args.Length) {
76 Console.WriteLine ("Missing argument to --timeout command line option.");
79 timeout = Int32.Parse (args [i + 1]);
81 } else if (args [i] == "--disabled") {
82 if (i + i >= args.Length) {
83 Console.WriteLine ("Missing argument to --disabled command line option.");
86 disabled_tests = args [i + 1];
88 } else if (args [i] == "--runtime") {
89 if (i + i >= args.Length) {
90 Console.WriteLine ("Missing argument to --runtime command line option.");
93 runtime = args [i + 1];
95 } else if (args [i] == "--opt-sets") {
96 if (i + i >= args.Length) {
97 Console.WriteLine ("Missing argument to --opt-sets command line option.");
100 foreach (var s in args [i + 1].Split ())
104 Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
112 var disabled = new Dictionary <string, string> ();
114 if (disabled_tests != null) {
115 foreach (string test in disabled_tests.Split ())
116 disabled [test] = test;
119 // The remaining arguments are the tests
120 var tests = new List<string> ();
121 for (int j = i; j < args.Length; ++j)
122 if (!disabled.ContainsKey (args [j]))
123 tests.Add (args [j]);
128 var processes = new List<Process> ();
129 var failed = new List<string> ();
130 var process_data = new Dictionary<Process, ProcessData> ();
132 object monitor = new object ();
134 var terminated = new List<Process> ();
136 if (concurrency != 1)
137 Console.WriteLine ("Running tests: ");
139 var test_info = new List<TestInfo> ();
140 if (opt_sets.Count == 0) {
141 foreach (string s in tests)
142 test_info.Add (new TestInfo { test = s });
144 foreach (string opt in opt_sets) {
145 foreach (string s in tests)
146 test_info.Add (new TestInfo { test = s, opt_set = opt });
150 foreach (TestInfo ti in test_info) {
152 while (processes.Count == concurrency) {
153 /* Wait for one process to terminate */
154 Monitor.Wait (monitor);
157 /* Cleaup terminated processes */
158 foreach (Process dead in terminated) {
159 if (process_data [dead].stdout != null)
160 process_data [dead].stdout.Close ();
161 if (process_data [dead].stderr != null)
162 process_data [dead].stderr.Close ();
163 // This is needed to avoid CreateProcess failed errors :(
169 string test = ti.test;
170 string opt_set = ti.opt_set;
172 if (concurrency == 1)
173 Console.Write ("Testing " + test + "... ");
175 /* Spawn a new process */
180 process_args = "-O=" + opt_set + " " + test;
181 ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
182 info.UseShellExecute = false;
183 info.RedirectStandardOutput = true;
184 info.RedirectStandardError = true;
185 Process p = new Process ();
187 p.EnableRaisingEvents = true;
189 ProcessData data = new ProcessData ();
192 p.Exited += delegate (object sender, EventArgs e) {
193 // Anon methods share some of their state, so we can't use
194 // variables which change during the loop (test, p)
195 Process dead = (Process)sender;
198 if (dead.ExitCode == 0) {
199 if (concurrency == 1)
200 Console.WriteLine ("passed.");
205 if (concurrency == 1)
206 Console.WriteLine ("failed.");
209 failed.Add (process_data [dead].test);
212 processes.Remove (dead);
213 terminated.Add (dead);
214 Monitor.Pulse (monitor);
218 string log_prefix = "";
220 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
222 data.stdout = new StreamWriter (new FileStream (test + log_prefix + ".stdout", FileMode.Create));
224 data.stderr = new StreamWriter (new FileStream (test + log_prefix + ".stderr", FileMode.Create));
226 p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
227 Process p2 = (Process)sender;
232 fs = process_data [p2].stdout;
234 if (String.IsNullOrEmpty (e.Data))
235 process_data [p2].stdout = null;
238 if (String.IsNullOrEmpty (e.Data))
241 fs.WriteLine (e.Data);
244 p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
245 Process p2 = (Process)sender;
250 fs = process_data [p2].stderr;
252 if (String.IsNullOrEmpty (e.Data))
253 process_data [p2].stderr = null;
257 if (String.IsNullOrEmpty (e.Data)) {
261 process_data [p2].stderr = null;
265 fs.WriteLine (e.Data);
270 process_data [p] = data;
274 p.BeginOutputReadLine ();
275 p.BeginErrorReadLine ();
278 bool timed_out = false;
280 /* Wait for all processes to terminate */
283 int nprocesses = processes.Count;
288 bool res = Monitor.Wait (monitor, 1000 * timeout);
296 Console.WriteLine ();
299 Console.WriteLine ("\nrunning tests timed out:\n");
300 Console.WriteLine (npassed + nfailed);
302 foreach (Process p in processes) {
303 Console.WriteLine (process_data [p].test);
310 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
312 Console.WriteLine ("\nFailed tests:\n");
313 foreach (string s in failed)
314 Console.WriteLine (s);