3 using System.Diagnostics;
4 using System.Reflection;
6 using System.Collections;
12 string Output { get; }
13 bool Invoke (string[] args);
16 class ReflectionTester: ITester {
21 public ReflectionTester (Assembly a)
23 ep = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint").GetMethod ("InvokeCompiler",
24 BindingFlags.Static | BindingFlags.Public);
26 throw new MissingMethodException ("static InvokeCompiler");
27 method_arg = new object [2];
30 public string Output {
32 return output.GetStringBuilder ().ToString ();
36 public bool Invoke(string[] args)
38 output = new StringWriter ();
39 method_arg [0] = args;
40 method_arg [1] = output;
41 return (bool)ep.Invoke (null, method_arg);
45 class ProcessTester: ITester
50 public ProcessTester (string p_path)
52 pi = new ProcessStartInfo ();
54 pi.CreateNoWindow = true;
55 pi.WindowStyle = ProcessWindowStyle.Hidden;
56 pi.RedirectStandardOutput = true;
57 pi.RedirectStandardError = true;
58 pi.UseShellExecute = false;
61 public string Output {
67 public bool Invoke(string[] args)
69 StringBuilder sb = new StringBuilder ();
70 foreach (string s in args) {
74 pi.Arguments = sb.ToString ();
75 Process p = Process.Start (pi);
76 output = p.StandardError.ReadToEnd ();
77 if (output.Length == 0)
78 output = p.StandardOutput.ReadToEnd ();
80 return p.ExitCode == 0;
93 static ArrayList know_issues = new ArrayList ();
94 static ArrayList ignore_list = new ArrayList ();
95 static ArrayList no_error_list = new ArrayList ();
96 static ArrayList regression = new ArrayList ();
98 static StreamWriter log_file;
100 static void Log (string msg, params object [] rest)
102 Console.Write (msg, rest);
103 log_file.Write (msg, rest);
106 static void LogLine ()
108 Console.WriteLine ();
109 log_file.WriteLine ();
112 static void LogLine (string msg, params object [] rest)
114 Console.WriteLine (msg, rest);
115 log_file.WriteLine (msg, rest);
118 static int Main(string[] args) {
119 if (args.Length != 4) {
120 Console.Error.WriteLine ("Usage: TestRunner (test-pattern|compiler-name) compiler know-issues log-file");
124 string test_pattern = args [0];
125 string mcs = args [1];
126 string issue_file = args [2];
127 string log_fname = args [3];
129 log_file = new StreamWriter (log_fname, false);
131 // THIS IS BUG #73763 workaround
132 if (test_pattern == "gmcs")
133 test_pattern = "*cs*.cs";
135 test_pattern = "cs*.cs";
137 string wrong_errors_file = issue_file;
138 string[] files = Directory.GetFiles (".", test_pattern);
140 ReadWrongErrors (wrong_errors_file);
143 Console.WriteLine ("Loading: " + mcs);
144 tester = new ReflectionTester (Assembly.LoadFile (mcs));
147 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
148 if (!File.Exists (mcs)) {
149 Console.Error.WriteLine ("ERROR: Tested compiler was not found");
152 tester = new ProcessTester (mcs);
157 int total = files.Length;
158 foreach (string s in files) {
159 string filename = Path.GetFileName (s);
160 if (filename.StartsWith ("CS")) { // Windows hack
167 string[] extra = GetExtraOptions (s);
169 test_args = new string [1 + extra.Length];
170 extra.CopyTo (test_args, 0);
172 test_args = new string [1];
174 test_args [test_args.Length - 1] = s;
178 if (ignore_list.Contains (filename)) {
179 LogLine ("NOT TESTED");
186 while (Char.IsLetter (filename, start_char))
189 int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
190 string expected = filename.Substring (start_char, end_char - start_char);
192 bool result = tester.Invoke (test_args);
194 HandleFailure (filename, CompilerError.Missing);
198 CompilerError result_code = GetCompilerError (expected, tester.Output);
199 if (HandleFailure (filename, result_code)) {
202 LogLine (tester.Output);
205 catch (Exception e) {
206 HandleFailure (filename, CompilerError.Missing);
211 LogLine ("Done" + Environment.NewLine);
212 LogLine ("{0} correctly detected error cases ({1:.##%})", success, (float) (success) / (float)total);
214 know_issues.AddRange (no_error_list);
215 if (know_issues.Count > 0) {
217 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
218 foreach (string s in know_issues)
221 if (regression.Count > 0) {
223 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
224 foreach (string s in regression)
230 return regression.Count == 0 ? 0 : 1;
233 static void ReadWrongErrors (string file)
235 const string ignored = "IGNORE";
236 const string no_error = "NO ERROR";
238 using (StreamReader sr = new StreamReader (file)) {
240 while ((line = sr.ReadLine()) != null) {
241 if (line.StartsWith ("#"))
244 ArrayList active_cont = know_issues;
246 if (line.IndexOf (ignored) > 0)
247 active_cont = ignore_list;
248 else if (line.IndexOf (no_error) > 0)
249 active_cont = no_error_list;
251 string file_name = line.Split (' ')[0];
252 if (file_name.Length == 0)
255 active_cont.Add (file_name);
260 static bool HandleFailure (string file, CompilerError status)
263 case CompilerError.Expected:
264 if (know_issues.Contains (file) || no_error_list.Contains (file)) {
265 LogLine ("FIXED ISSUE");
271 case CompilerError.Wrong:
272 if (know_issues.Contains (file)) {
273 LogLine ("KNOWN ISSUE (Wrong error reported)");
274 know_issues.Remove (file);
277 if (no_error_list.Contains (file)) {
278 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR)");
279 no_error_list.Remove (file);
282 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR)");
287 case CompilerError.Missing:
288 if (no_error_list.Contains (file)) {
289 LogLine ("KNOWN ISSUE (No error reported)");
290 no_error_list.Remove (file);
294 if (know_issues.Contains (file)) {
295 LogLine ("REGRESSION (WRONG ERROR -> NO ERROR)");
296 know_issues.Remove (file);
299 LogLine ("REGRESSION (CORRECT ERROR -> NO ERROR)");
305 regression.Add (file);
309 static CompilerError GetCompilerError (string expected, string buffer)
311 const string error_prefix = "CS";
312 const string ignored_error = "error CS5001";
313 string tested_text = "error " + error_prefix + expected;
314 StringReader sr = new StringReader (buffer);
315 string line = sr.ReadLine ();
316 bool any_error = false;
317 while (line != null) {
319 if (line.IndexOf (tested_text) != -1)
320 return CompilerError.Expected;
322 if (line.IndexOf (error_prefix) != -1 &&
323 line.IndexOf (ignored_error) == -1)
326 line = sr.ReadLine ();
329 return any_error ? CompilerError.Wrong : CompilerError.Missing;
332 static string[] GetExtraOptions (string file)
334 const string options = "// Compiler options:";
337 using (StreamReader sr = new StreamReader (file)) {
339 while (row++ < 3 && (line = sr.ReadLine()) != null) {
340 int index = line.IndexOf (options);
342 string[] o = line.Substring (index + options.Length).Split (' ');
343 for (int i = 0; i < o.Length; i++)
344 o [i] = o[i].TrimStart ();