mcs/errors/TestRunner.cs

   1 using System;
   2 using System.IO;
   3 using System.Diagnostics;
   4 using System.Reflection;
   5 using System.Text;
   6 using System.Collections;
   7
   8 namespace TestRunner {
   9
  10         interface ITester
  11         {
  12                 string Output { get; }
  13                 bool Invoke (string[] args);
  14         }
  15
  16         class ReflectionTester: ITester {
  17                 MethodInfo ep;
  18                 object[] method_arg;
  19                 StringWriter output;
  20
  21                 public ReflectionTester (Assembly a)
  22                 {
  23                         ep = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint").GetMethod ("InvokeCompiler",
  24                                 BindingFlags.Static | BindingFlags.Public);
  25                         if (ep == null)
  26                                 throw new MissingMethodException ("static InvokeCompiler");
  27                         method_arg = new object [2];
  28                 }
  29
  30                 public string Output {
  31                         get {
  32                                 return output.GetStringBuilder ().ToString ();
  33                         }
  34                 }
  35
  36                 public bool Invoke(string[] args)
  37                 {
  38                         output = new StringWriter ();
  39                         method_arg [0] = args;
  40                         method_arg [1] = output;
  41                         return (bool)ep.Invoke (null, method_arg);
  42                 }
  43         }
  44
  45         class ProcessTester: ITester
  46         {
  47                 ProcessStartInfo pi;
  48                 string output;
  49
  50                 public ProcessTester (string p_path)
  51                 {
  52                         pi = new ProcessStartInfo ();
  53                         pi.FileName = p_path;
  54                         pi.CreateNoWindow = true;
  55                         pi.WindowStyle = ProcessWindowStyle.Hidden;
  56                         pi.RedirectStandardOutput = true;
  57                         pi.RedirectStandardError = true;
  58                         pi.UseShellExecute = false;
  59                 }
  60
  61                 public string Output {
  62                         get {
  63                                 return output;
  64                         }
  65                 }
  66
  67                 public bool Invoke(string[] args)
  68                 {
  69                         StringBuilder sb = new StringBuilder ();
  70                         foreach (string s in args) {
  71                                 sb.Append (s);
  72                                 sb.Append (" ");
  73                         }
  74                         pi.Arguments = sb.ToString ();
  75                         Process p = Process.Start (pi);
  76                         output = p.StandardError.ReadToEnd ();
  77                         if (output.Length == 0)
  78                             output = p.StandardOutput.ReadToEnd ();
  79                         p.WaitForExit ();
  80                         return p.ExitCode == 0;
  81                 }
  82         }
  83
  84         class Tester {
  85
  86                 enum CompilerError
  87                 {
  88                         Expected,
  89                         Wrong,
  90                         Missing
  91                 }
  92
  93                 static ArrayList know_issues = new ArrayList ();
  94                 static ArrayList ignore_list = new ArrayList ();
  95                 static ArrayList no_error_list = new ArrayList ();
  96                 static ArrayList regression = new ArrayList ();
  97
  98                 static StreamWriter log_file;
  99
 100                 static void Log (string msg, params object [] rest)
 101                 {
 102                         Console.Write (msg, rest);
 103                         log_file.Write (msg, rest);
 104                 }
 105
 106                 static void LogLine ()
 107                 {
 108                         Console.WriteLine ();
 109                         log_file.WriteLine ();
 110                 }
 111
 112                 static void LogLine (string msg, params object [] rest)
 113                 {
 114                         Console.WriteLine (msg, rest);
 115                         log_file.WriteLine (msg, rest);
 116                 }
 117
 118                 static int Main(string[] args) {
 119                         if (args.Length != 4) {
 120                                 Console.Error.WriteLine ("Usage: TestRunner (test-pattern|compiler-name) compiler know-issues log-file");
 121                                 return 1;
 122                         }
 123
 124                         string test_pattern = args [0];
 125                         string mcs = args [1];
 126                         string issue_file = args [2];
 127                         string log_fname = args [3];
 128
 129                         log_file = new StreamWriter (log_fname, false);
 130
 131                         // THIS IS BUG #73763 workaround
 132                         if (test_pattern == "gmcs")
 133                                 test_pattern = "*cs*.cs";
 134                         else
 135                                 test_pattern = "cs*.cs";
 136
 137                         string wrong_errors_file = issue_file;
 138                         string[] files = Directory.GetFiles (".", test_pattern);
 139
 140                         ReadWrongErrors (wrong_errors_file);
 141                         ITester tester;
 142                         try {
 143                                 Console.WriteLine ("Loading: " + mcs);
 144                                 tester = new ReflectionTester (Assembly.LoadFile (mcs));
 145                         }
 146                         catch (Exception) {
 147                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
 148                                 if (!File.Exists (mcs)) {
 149                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
 150                                         return 1;
 151                                 }
 152                                 tester = new ProcessTester (mcs);
 153                         }
 154
 155                         string[] test_args;
 156                         int success = 0;
 157                         int total = files.Length;
 158                         foreach (string s in files) {
 159                                 string filename = Path.GetFileName (s);
 160                                 if (filename.StartsWith ("CS")) { // Windows hack
 161                                         total--;
 162                                         continue;
 163                                 }
 164
 165                                 Log (filename);
 166
 167                                 string[] extra = GetExtraOptions (s);
 168                                 if (extra != null) {
 169                                         test_args = new string [1 + extra.Length];
 170                                         extra.CopyTo (test_args, 0);
 171                                 } else {
 172                                         test_args = new string [1];
 173                                 }
 174                                 test_args [test_args.Length - 1] = s;
 175
 176                                 Log ("...\t");
 177
 178                                 if (ignore_list.Contains (filename)) {
 179                                         LogLine ("NOT TESTED");
 180                                         total--;
 181                                         continue;
 182                                 }
 183
 184                                 try {
 185                                         int start_char = 0;
 186                                         while (Char.IsLetter (filename, start_char))
 187                                                 ++start_char;
 188
 189                                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
 190                                         string expected = filename.Substring (start_char, end_char - start_char);
 191
 192                                         bool result = tester.Invoke (test_args);
 193                                         if (result) {
 194                                                 HandleFailure (filename, CompilerError.Missing);
 195                                                 continue;
 196                                         }
 197
 198                                         CompilerError result_code = GetCompilerError (expected, tester.Output);
 199                                         if (HandleFailure (filename, result_code)) {
 200                                                 success++;
 201                                         } else {
 202                                                 LogLine (tester.Output);
 203                                         }
 204                                 }
 205                                 catch (Exception e) {
 206                                         HandleFailure (filename, CompilerError.Missing);
 207                                         Log (e.ToString ());
 208                                 }
 209                         }
 210
 211                         LogLine ("Done" + Environment.NewLine);
 212                         LogLine ("{0} correctly detected error cases ({1:.##%})", success, (float) (success) / (float)total);
 213
 214                         know_issues.AddRange (no_error_list);
 215                         if (know_issues.Count > 0) {
 216                                 LogLine ();
 217                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 218                                 foreach (string s in know_issues)
 219                                         LogLine (s);
 220                         }
 221                         if (regression.Count > 0) {
 222                                 LogLine ();
 223                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 224                                 foreach (string s in regression)
 225                                         LogLine (s);
 226                         }
 227
 228                         log_file.Close ();
 229
 230                         return regression.Count == 0 ? 0 : 1;
 231                 }
 232
 233                 static void ReadWrongErrors (string file)
 234                 {
 235                         const string ignored = "IGNORE";
 236                         const string no_error = "NO ERROR";
 237
 238                         using (StreamReader sr = new StreamReader (file)) {
 239                                 string line;
 240                                 while ((line = sr.ReadLine()) != null) {
 241                                         if (line.StartsWith ("#"))
 242                                                 continue;
 243
 244                                         ArrayList active_cont = know_issues;
 245
 246                                         if (line.IndexOf (ignored) > 0)
 247                                                 active_cont = ignore_list;
 248                                         else if (line.IndexOf (no_error) > 0)
 249                                                 active_cont = no_error_list;
 250
 251                                         string file_name = line.Split (' ')[0];
 252                                         if (file_name.Length == 0)
 253                                                 continue;
 254
 255                                         active_cont.Add (file_name);
 256                                 }
 257                         }
 258                 }
 259
 260                 static bool HandleFailure (string file, CompilerError status)
 261                 {
 262                         switch (status) {
 263                                 case CompilerError.Expected:
 264                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
 265                                                 LogLine ("FIXED ISSUE");
 266                                                 return true;
 267                                         }
 268                                         LogLine ("OK");
 269                                         return true;
 270
 271                                 case CompilerError.Wrong:
 272                                         if (know_issues.Contains (file)) {
 273                                                 LogLine ("KNOWN ISSUE (Wrong error reported)");
 274                                                 know_issues.Remove (file);
 275                                                 return false;
 276                                         }
 277                                         if (no_error_list.Contains (file)) {
 278                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR)");
 279                                                 no_error_list.Remove (file);
 280                                         }
 281                                         else {
 282                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR)");
 283                                         }
 284
 285                                         break;
 286
 287                                 case CompilerError.Missing:
 288                                         if (no_error_list.Contains (file)) {
 289                                                 LogLine ("KNOWN ISSUE (No error reported)");
 290                                                 no_error_list.Remove (file);
 291                                                 return false;
 292                                         }
 293
 294                                         if (know_issues.Contains (file)) {
 295                                                 LogLine ("REGRESSION (WRONG ERROR -> NO ERROR)");
 296                                                 know_issues.Remove (file);
 297                                         }
 298                                         else {
 299                                                 LogLine ("REGRESSION (CORRECT ERROR -> NO ERROR)");
 300                                         }
 301
 302                                         break;
 303                         }
 304
 305                         regression.Add (file);
 306                         return false;
 307                 }
 308
 309                 static CompilerError GetCompilerError (string expected, string buffer)
 310                 {
 311                         const string error_prefix = "CS";
 312                         const string ignored_error = "error CS5001";
 313                         string tested_text = "error " + error_prefix + expected;
 314                         StringReader sr = new StringReader (buffer);
 315                         string line = sr.ReadLine ();
 316                         bool any_error = false;
 317                         while (line != null) {
 318
 319                                 if (line.IndexOf (tested_text) != -1)
 320                                         return CompilerError.Expected;
 321
 322                                 if (line.IndexOf (error_prefix) != -1 &&
 323                                         line.IndexOf (ignored_error) == -1)
 324                                         any_error = true;
 325
 326                                 line = sr.ReadLine ();
 327                         }
 328
 329                         return any_error ? CompilerError.Wrong : CompilerError.Missing;
 330                 }
 331
 332                 static string[] GetExtraOptions (string file)
 333                 {
 334                         const string options = "// Compiler options:";
 335
 336                         int row = 0;
 337                         using (StreamReader sr = new StreamReader (file)) {
 338                                 String line;
 339                                 while (row++ < 3 && (line = sr.ReadLine()) != null) {
 340                                         int index = line.IndexOf (options);
 341                                         if (index != -1) {
 342                                                 string[] o = line.Substring (index + options.Length).Split (' ');
 343                                                 for (int i = 0; i < o.Length; i++)
 344                                                         o [i] = o[i].TrimStart ();
 345                                                 return o;
 346                                         }
 347                                 }
 348                         }
 349                         return null;
 350                 }
 351         }
 352 }