5 // Zoltan Varga (vargaz@gmail.com)
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
9 // Permission is hereby granted, free of charge, to any person obtaining
10 // a copy of this software and associated documentation files (the
11 // "Software"), to deal in the Software without restriction, including
12 // without limitation the rights to use, copy, modify, merge, publish,
13 // distribute, sublicense, and/or sell copies of the Software, and to
14 // permit persons to whom the Software is furnished to do so, subject to
15 // the following conditions:
17 // The above copyright notice and this permission notice shall be
18 // included in all copies or substantial portions of the Software.
20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 using System.Threading;
31 using System.Diagnostics;
32 using System.Collections.Generic;
35 // This is a simple test runner with support for parallel execution
38 public class TestRunner
42 public StreamWriter stdout, stderr;
43 public string stdoutFile, stderrFile;
45 public void CloseStreams () {
58 public string test, opt_set;
61 public static int Main (String[] args) {
64 int timeout = 2 * 60; // in seconds
66 // FIXME: Add support for runtime arguments + env variables
68 string disabled_tests = null;
69 string runtime = "mono";
70 var opt_sets = new List<string> ();
74 while (i < args.Length) {
75 if (args [i].StartsWith ("-")) {
76 if (args [i] == "-j") {
77 if (i + i >= args.Length) {
78 Console.WriteLine ("Missing argument to -j command line option.");
81 if (args [i + 1] == "a")
82 concurrency = Environment.ProcessorCount;
84 concurrency = Int32.Parse (args [i + 1]);
86 } else if (args [i] == "--timeout") {
87 if (i + i >= args.Length) {
88 Console.WriteLine ("Missing argument to --timeout command line option.");
91 timeout = Int32.Parse (args [i + 1]);
93 } else if (args [i] == "--disabled") {
94 if (i + i >= args.Length) {
95 Console.WriteLine ("Missing argument to --disabled command line option.");
98 disabled_tests = args [i + 1];
100 } else if (args [i] == "--runtime") {
101 if (i + i >= args.Length) {
102 Console.WriteLine ("Missing argument to --runtime command line option.");
105 runtime = args [i + 1];
107 } else if (args [i] == "--opt-sets") {
108 if (i + i >= args.Length) {
109 Console.WriteLine ("Missing argument to --opt-sets command line option.");
112 foreach (var s in args [i + 1].Split ())
116 Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
124 var disabled = new Dictionary <string, string> ();
126 if (disabled_tests != null) {
127 foreach (string test in disabled_tests.Split ())
128 disabled [test] = test;
131 // The remaining arguments are the tests
132 var tests = new List<string> ();
133 for (int j = i; j < args.Length; ++j)
134 if (!disabled.ContainsKey (args [j]))
135 tests.Add (args [j]);
140 var processes = new List<Process> ();
141 var failed = new List<ProcessData> ();
142 var process_data = new Dictionary<Process, ProcessData> ();
144 object monitor = new object ();
146 var terminated = new List<Process> ();
148 if (concurrency != 1)
149 Console.WriteLine ("Running tests: ");
151 var test_info = new List<TestInfo> ();
152 if (opt_sets.Count == 0) {
153 foreach (string s in tests)
154 test_info.Add (new TestInfo { test = s });
156 foreach (string opt in opt_sets) {
157 foreach (string s in tests)
158 test_info.Add (new TestInfo { test = s, opt_set = opt });
162 foreach (TestInfo ti in test_info) {
164 while (processes.Count == concurrency) {
165 /* Wait for one process to terminate */
166 Monitor.Wait (monitor);
169 /* Cleaup terminated processes */
170 foreach (Process dead in terminated) {
171 if (process_data [dead].stdout != null)
172 process_data [dead].stdout.Close ();
173 if (process_data [dead].stderr != null)
174 process_data [dead].stderr.Close ();
175 // This is needed to avoid CreateProcess failed errors :(
181 string test = ti.test;
182 string opt_set = ti.opt_set;
184 if (concurrency == 1)
185 Console.Write ("Testing " + test + "... ");
187 /* Spawn a new process */
192 process_args = "-O=" + opt_set + " " + test;
193 ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
194 info.UseShellExecute = false;
195 info.RedirectStandardOutput = true;
196 info.RedirectStandardError = true;
197 Process p = new Process ();
199 p.EnableRaisingEvents = true;
201 ProcessData data = new ProcessData ();
204 p.Exited += delegate (object sender, EventArgs e) {
205 // Anon methods share some of their state, so we can't use
206 // variables which change during the loop (test, p)
207 Process dead = (Process)sender;
210 if (dead.ExitCode == 0) {
211 if (concurrency == 1)
212 Console.WriteLine ("passed.");
217 if (concurrency == 1)
218 Console.WriteLine ("failed.");
221 failed.Add (process_data [dead]);
224 processes.Remove (dead);
225 terminated.Add (dead);
226 Monitor.Pulse (monitor);
230 string log_prefix = "";
232 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
234 data.stdoutFile = test + log_prefix + ".stdout";
235 data.stdout = new StreamWriter (new FileStream (data.stdoutFile, FileMode.Create));
237 data.stderrFile = test + log_prefix + ".stderr";
238 data.stderr = new StreamWriter (new FileStream (data.stderrFile, FileMode.Create));
240 p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
241 Process p2 = (Process)sender;
246 fs = process_data [p2].stdout;
248 if (String.IsNullOrEmpty (e.Data))
249 process_data [p2].stdout = null;
252 if (String.IsNullOrEmpty (e.Data)) {
255 fs.WriteLine (e.Data);
260 p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
261 Process p2 = (Process)sender;
266 fs = process_data [p2].stderr;
268 if (String.IsNullOrEmpty (e.Data))
269 process_data [p2].stderr = null;
273 if (String.IsNullOrEmpty (e.Data)) {
277 process_data [p2].stderr = null;
280 fs.WriteLine (e.Data);
287 process_data [p] = data;
291 p.BeginOutputReadLine ();
292 p.BeginErrorReadLine ();
295 bool timed_out = false;
297 /* Wait for all processes to terminate */
300 int nprocesses = processes.Count;
305 bool res = Monitor.Wait (monitor, 1000 * timeout);
313 Console.WriteLine ();
316 Console.WriteLine ("\nrunning tests timed out:\n");
317 Console.WriteLine (npassed + nfailed);
319 foreach (Process p in processes) {
320 ProcessData pd = process_data [p];
322 Console.WriteLine (pd.test);
324 DumpFile (pd.stdoutFile);
325 DumpFile (pd.stderrFile);
331 Console.WriteLine ("" + npassed + " test(s) passed. " + nfailed + " test(s) did not pass.");
333 Console.WriteLine ("\nFailed tests:\n");
334 foreach (ProcessData pd in failed) {
335 Console.WriteLine (pd.test);
336 DumpFile (pd.stdoutFile);
337 DumpFile (pd.stderrFile);
345 static void DumpFile (string filename) {
346 if (File.Exists (filename)) {
347 Console.WriteLine ("=============== {0} ===============", filename);
348 Console.WriteLine (File.ReadAllText (filename));
349 Console.WriteLine ("=============== EOF ===============");