mcs/tools/compiler-tester/compiler-tester.cs

   1 using System;
   2 using System.IO;
   3 using System.Diagnostics;
   4 using System.Reflection;
   5 using System.Text;
   6 using System.Collections;
   7
   8 namespace TestRunner {
   9
  10         interface ITester
  11         {
  12                 string Output { get; }
  13                 bool Invoke (string[] args);
  14                 bool IsWarning (int warningNumber);
  15         }
  16
  17         class ReflectionTester: ITester {
  18                 MethodInfo ep;
  19                 object[] method_arg;
  20                 StringWriter output;
  21                 int[] all_warnings;
  22
  23                 public ReflectionTester (Assembly a)
  24                 {
  25                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  26
  27                         if (t == null)
  28                                 Console.Error.WriteLine ("null, huh?");
  29
  30                         ep = t.GetMethod ("InvokeCompiler",
  31                                 BindingFlags.Static | BindingFlags.Public);
  32                         if (ep == null)
  33                                 throw new MissingMethodException ("static InvokeCompiler");
  34                         method_arg = new object [2];
  35
  36                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  37                         all_warnings = (int[])pi.GetValue (null, null);
  38                         Array.Sort (all_warnings);
  39                 }
  40
  41                 public string Output {
  42                         get {
  43                                 return output.GetStringBuilder ().ToString ();
  44                         }
  45                 }
  46
  47                 public bool Invoke(string[] args)
  48                 {
  49                         output = new StringWriter ();
  50                         method_arg [0] = args;
  51                         method_arg [1] = output;
  52                         return (bool)ep.Invoke (null, method_arg);
  53                 }
  54
  55                 public bool IsWarning (int warningNumber)
  56                 {
  57                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  58                 }
  59         }
  60
  61 #if !NET_2_1
  62         class ProcessTester: ITester
  63         {
  64                 ProcessStartInfo pi;
  65                 string output;
  66
  67                 public ProcessTester (string p_path)
  68                 {
  69                         pi = new ProcessStartInfo ();
  70                         pi.FileName = p_path;
  71                         pi.CreateNoWindow = true;
  72                         pi.WindowStyle = ProcessWindowStyle.Hidden;
  73                         pi.RedirectStandardOutput = true;
  74                         pi.RedirectStandardError = true;
  75                         pi.UseShellExecute = false;
  76                 }
  77
  78                 public string Output {
  79                         get {
  80                                 return output;
  81                         }
  82                 }
  83
  84                 public bool Invoke(string[] args)
  85                 {
  86                         StringBuilder sb = new StringBuilder ("/nologo ");
  87                         foreach (string s in args) {
  88                                 sb.Append (s);
  89                                 sb.Append (" ");
  90                         }
  91                         pi.Arguments = sb.ToString ();
  92                         Process p = Process.Start (pi);
  93                         output = p.StandardError.ReadToEnd ();
  94                         if (output.Length == 0)
  95                             output = p.StandardOutput.ReadToEnd ();
  96                         p.WaitForExit ();
  97                         return p.ExitCode == 0;
  98                 }
  99
 100                 public bool IsWarning (int warningNumber)
 101                 {
 102                         throw new NotImplementedException ();
 103                 }
 104         }
 105 #endif
 106
 107         class TestCase
 108         {
 109                 public readonly string FileName;
 110                 public readonly string[] CompilerOptions;
 111                 public readonly string[] Dependencies;
 112
 113                 public TestCase (string filename, string[] options, string[] deps)
 114                 {
 115                         this.FileName = filename;
 116                         this.CompilerOptions = options;
 117                         this.Dependencies = deps;
 118                 }
 119         }
 120
 121         class Checker: IDisposable
 122         {
 123                 protected ITester tester;
 124                 protected int success;
 125                 protected int total;
 126                 protected int ignored;
 127                 protected int syntax_errors;
 128                 string issue_file;
 129                 StreamWriter log_file;
 130                 // protected string[] compiler_options;
 131                 // protected string[] dependencies;
 132
 133                 protected ArrayList tests = new ArrayList ();
 134                 protected Hashtable test_hash = new Hashtable ();
 135                 protected ArrayList regression = new ArrayList ();
 136                 protected ArrayList know_issues = new ArrayList ();
 137                 protected ArrayList ignore_list = new ArrayList ();
 138                 protected ArrayList no_error_list = new ArrayList ();
 139
 140                 protected bool verbose; // = true;
 141
 142                 int total_known_issues;
 143
 144                 protected Checker (ITester tester, string log_file, string issue_file)
 145                 {
 146                         this.tester = tester;
 147                         this.issue_file = issue_file;
 148                         ReadWrongErrors (issue_file);
 149                         this.log_file = new StreamWriter (log_file, false);
 150                 }
 151
 152                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 153                                                         out string[] dependencies)
 154                 {
 155                         int row = 0;
 156                         compiler_options = null;
 157                         dependencies = null;
 158                         try {
 159                                 using (StreamReader sr = new StreamReader (file)) {
 160                                         String line;
 161                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 162                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 163                                                                       ref dependencies))
 164                                                         return false;
 165                                         }
 166                                 }
 167                         } catch {
 168                                 return false;
 169                         }
 170                         return true;
 171                 }
 172
 173                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 174                                                         ref string[] compiler_options,
 175                                                         ref string[] dependencies)
 176                 {
 177                         const string options = "// Compiler options:";
 178                         const string depends = "// Dependencies:";
 179
 180                         if (row == 1) {
 181                                 compiler_options = null;
 182                                 dependencies = null;
 183                         }
 184
 185                         int index = line.IndexOf (options);
 186                         if (index != -1) {
 187                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 188                                 for (int i = 0; i < compiler_options.Length; i++)
 189                                         compiler_options[i] = compiler_options[i].TrimStart ();
 190                         }
 191                         index = line.IndexOf (depends);
 192                         if (index != -1) {
 193                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 194                                 for (int i = 0; i < dependencies.Length; i++)
 195                                         dependencies[i] = dependencies[i].TrimStart ();
 196                         }
 197
 198                         return true;
 199                 }
 200
 201                 public bool Do (string filename)
 202                 {
 203                         if (test_hash.Contains (filename))
 204                                 return true;
 205
 206                         if (ignore_list.Contains (filename)) {
 207                                 ++ignored;
 208                                 LogFileLine (filename, "NOT TESTED");
 209                                 return false;
 210                         }
 211
 212                         string[] compiler_options, dependencies;
 213                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 214                                 LogFileLine (filename, "ERROR");
 215                                 return false;
 216                         }
 217
 218                         TestCase test = new TestCase (filename, compiler_options, dependencies);
 219                         test_hash.Add (filename, test);
 220
 221                         ++total;
 222                         if (dependencies != null) {
 223                                 foreach (string dependency in dependencies) {
 224                                         if (!Do (dependency)) {
 225                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 226                                                 return false;
 227                                         }
 228                                 }
 229                         }
 230
 231                         tests.Add (test);
 232
 233                         return Check (test);
 234                 }
 235
 236                 protected virtual bool Check (TestCase test)
 237                 {
 238                         string[] test_args;
 239
 240                         if (test.CompilerOptions != null) {
 241                                 test_args = new string [1 + test.CompilerOptions.Length];
 242                                 test.CompilerOptions.CopyTo (test_args, 0);
 243                         } else {
 244                                 test_args = new string [1];
 245                         }
 246                         test_args [test_args.Length - 1] = test.FileName;
 247
 248                         return tester.Invoke (test_args);
 249                 }
 250
 251
 252                 void ReadWrongErrors (string file)
 253                 {
 254                         const string ignored = "IGNORE";
 255                         const string no_error = "NO ERROR";
 256
 257                         using (StreamReader sr = new StreamReader (file)) {
 258                                 string line;
 259                                 while ((line = sr.ReadLine()) != null) {
 260                                         if (line.StartsWith ("#"))
 261                                                 continue;
 262
 263                                         ArrayList active_cont = know_issues;
 264
 265                                         if (line.IndexOf (ignored) > 0)
 266                                                 active_cont = ignore_list;
 267                                         else if (line.IndexOf (no_error) > 0)
 268                                                 active_cont = no_error_list;
 269
 270                                         string file_name = line.Split (' ')[0];
 271                                         if (file_name.Length == 0)
 272                                                 continue;
 273
 274                                         active_cont.Add (file_name);
 275                                 }
 276                         }
 277                         total_known_issues = know_issues.Count;
 278                 }
 279
 280                 public virtual void PrintSummary ()
 281                 {
 282                         LogLine ("Done" + Environment.NewLine);
 283                         LogLine ("{0} test cases passed ({1:0.##%})", success, (float) (success) / (float)total);
 284
 285                         if (syntax_errors > 0)
 286                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 287
 288                         if (ignored > 0)
 289                                 LogLine ("{0} test(s) ignored", ignored);
 290
 291                         if (total_known_issues - know_issues.Count > 0)
 292                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 293
 294                         know_issues.AddRange (no_error_list);
 295                         if (know_issues.Count > 0) {
 296                                 LogLine ("");
 297                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 298                                 foreach (string s in know_issues)
 299                                         LogLine (s);
 300                         }
 301                         if (regression.Count > 0) {
 302                                 LogLine ("");
 303                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 304                                 foreach (string s in regression)
 305                                         LogLine (s);
 306                         }
 307                 }
 308
 309                 public int ResultCode
 310                 {
 311                         get {
 312                                 return regression.Count == 0 ? 0 : 1;
 313                         }
 314                 }
 315
 316                 protected void Log (string msg, params object [] rest)
 317                 {
 318                         Console.Write (msg, rest);
 319                         log_file.Write (msg, rest);
 320                 }
 321
 322                 protected void LogLine (string msg, params object [] rest)
 323                 {
 324                         Console.WriteLine (msg, rest);
 325                         log_file.WriteLine (msg, rest);
 326                 }
 327
 328                 protected void LogFileLine (string file, string msg, params object [] args)
 329                 {
 330                         string s = file + "...\t" + string.Format (msg, args);
 331                         Console.WriteLine (s);
 332                         log_file.WriteLine (s);
 333                 }
 334
 335                 #region IDisposable Members
 336
 337                 public void Dispose()
 338                 {
 339                         log_file.Close ();
 340                 }
 341
 342                 #endregion
 343         }
 344
 345         class PositiveChecker: Checker
 346         {
 347                 readonly string files_folder;
 348                 readonly static object[] default_args = new object[1] { new string[] {} };
 349                 string doc_output;
 350
 351 #if !NET_2_1
 352                 ProcessStartInfo pi;
 353 #endif
 354                 readonly string mono;
 355
 356                 protected enum TestResult {
 357                         CompileError,
 358                         ExecError,
 359                         LoadError,
 360                         XmlError,
 361                         Success
 362                 }
 363
 364                 public PositiveChecker (ITester tester, string log_file, string issue_file):
 365                         base (tester, log_file, issue_file)
 366                 {
 367                         files_folder = Directory.GetCurrentDirectory ();
 368
 369 #if !NET_2_1
 370                         pi = new ProcessStartInfo ();
 371                         pi.CreateNoWindow = true;
 372                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 373                         pi.RedirectStandardOutput = true;
 374                         pi.RedirectStandardError = true;
 375                         pi.UseShellExecute = false;
 376
 377                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 378                         if (mono != null) {
 379                                 pi.FileName = mono;
 380                         }
 381 #endif
 382                 }
 383
 384                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 385                                                         out string[] dependencies) {
 386                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 387                                 return false;
 388
 389                         doc_output = null;
 390                         if (compiler_options == null)
 391                                 return true;
 392
 393                         foreach (string one_opt in compiler_options) {
 394                                 if (one_opt.StartsWith ("-doc:")) {
 395                                         doc_output = one_opt.Split (':', '/')[1];
 396                                 }
 397                         }
 398                         return true;
 399                 }
 400
 401                 protected override bool Check(TestCase test)
 402                 {
 403                         string filename = test.FileName;
 404                         try {
 405                                 if (!base.Check (test)) {
 406                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 407                                         return false;
 408                                 }
 409                         }
 410                         catch (Exception e) {
 411                                 if (e.InnerException != null)
 412                                         e = e.InnerException;
 413
 414                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 415                                 return false;
 416                         }
 417
 418                         // Test setup
 419                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 420                                 if (verbose)
 421                                         LogFileLine (filename, "OK");
 422                                 --total;
 423                                 return true;
 424                         }
 425
 426                         MethodInfo mi = null;
 427                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 428
 429                         // Enable .dll only tests (no execution required)
 430                         if (!File.Exists(file)) {
 431                                 HandleFailure (filename, TestResult.Success, null);
 432                                 return true;
 433                         }
 434
 435                         try {
 436                                 mi = Assembly.LoadFile (file).EntryPoint;
 437                         }
 438                         catch (FileNotFoundException) {
 439                                 if (File.Exists (file)) {
 440                                         Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 441                                 }
 442                         }
 443                         catch (Exception e) {
 444                                 HandleFailure (filename, TestResult.LoadError, e.ToString ());
 445                                 return false;
 446                         }
 447
 448                         if (!ExecuteFile (mi, file, filename))
 449                                 return false;
 450
 451                         if (doc_output != null) {
 452                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 453                                 try {
 454 #if !NET_2_1
 455                                         XmlComparer.Compare (ref_file, doc_output);
 456 #endif
 457                                 }
 458                                 catch (Exception e) {
 459                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 460                                         return false;
 461                                 }
 462                         }
 463
 464                         HandleFailure (filename, TestResult.Success, null);
 465                         return true;
 466                 }
 467
 468 #if !NET_2_1
 469                 int ExecFile (string exe_name, string filename)
 470                 {
 471                         if (mono == null)
 472                                 pi.FileName = exe_name;
 473                         else
 474                                 pi.Arguments = exe_name;
 475
 476                         Process p = Process.Start (pi);
 477                         p.WaitForExit ();
 478                         return p.ExitCode;
 479                 }
 480 #endif
 481
 482                 bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
 483                 {
 484                         TextWriter stdout = Console.Out;
 485                         TextWriter stderr = Console.Error;
 486                         Console.SetOut (TextWriter.Null);
 487                         Console.SetError (TextWriter.Null);
 488                         ParameterInfo[] pi = entry_point.GetParameters ();
 489                         object[] args = pi.Length == 0 ? null : default_args;
 490
 491                         object result = null;
 492                         try {
 493                                 try {
 494                                         result = entry_point.Invoke (null, args);
 495                                 } finally {
 496                                         Console.SetOut (stdout);
 497                                         Console.SetError (stderr);
 498                                 }
 499                         }
 500                         catch (Exception e) {
 501 #if !NET_2_1
 502                                 int exit_code = ExecFile (exe_name, filename);
 503                                 if (exit_code == 0) {
 504                                         LogLine ("(appdomain method failed, external executable succeeded)");
 505                                         LogLine (e.ToString ());
 506                                         return true;
 507                                 }
 508 #endif
 509                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 510                                 return false;
 511                         }
 512
 513                         if (result is int && (int)result != 0) {
 514                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 515                                 return false;
 516                         }
 517                         return true;
 518                 }
 519
 520                 void HandleFailure (string file, TestResult status, string extra)
 521                 {
 522                         switch (status) {
 523                                 case TestResult.Success:
 524                                         success++;
 525                                         if (know_issues.Contains (file)) {
 526                                                 LogFileLine (file, "FIXED ISSUE");
 527                                                 return;
 528                                         }
 529                                         if (verbose)
 530                                                 LogFileLine (file, "OK");
 531                                         return;
 532
 533                                 case TestResult.CompileError:
 534                                         if (know_issues.Contains (file)) {
 535                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 536                                                 know_issues.Remove (file);
 537                                                 return;
 538                                         }
 539                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 540                                         break;
 541
 542                                 case TestResult.ExecError:
 543                                         if (know_issues.Contains (file)) {
 544                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 545                                                 know_issues.Remove (file);
 546                                                 return;
 547                                         }
 548                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 549                                         break;
 550
 551                                 case TestResult.XmlError:
 552                                         if (know_issues.Contains (file)) {
 553                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
 554                                                 know_issues.Remove (file);
 555                                                 return;
 556                                         }
 557                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 558                                         break;
 559
 560                                 case TestResult.LoadError:
 561                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
 562                                         break;
 563                         }
 564
 565                         if (extra != null)
 566                                 LogLine ("{0}", extra);
 567
 568                         regression.Add (file);
 569                 }
 570         }
 571
 572         class NegativeChecker: Checker
 573         {
 574                 string expected_message;
 575                 string error_message;
 576                 bool check_msg;
 577                 bool check_error_line;
 578                 bool is_warning;
 579                 IDictionary wrong_warning;
 580
 581                 protected enum CompilerError {
 582                         Expected,
 583                         Wrong,
 584                         Missing,
 585                         WrongMessage,
 586                         MissingLocation,
 587                         Duplicate
 588                 }
 589
 590                 public NegativeChecker (ITester tester, string log_file, string issue_file, bool check_msg):
 591                         base (tester, log_file, issue_file)
 592                 {
 593                         this.check_msg = check_msg;
 594                         wrong_warning = new Hashtable ();
 595                 }
 596
 597                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
 598                                                         ref string[] compiler_options,
 599                                                         ref string[] dependencies)
 600                 {
 601                         if (row == 1) {
 602                                 expected_message = null;
 603
 604                                 int index = line.IndexOf (':');
 605                                 if (index == -1 || index > 15) {
 606                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
 607                                         ++syntax_errors;
 608                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
 609                                                               ref dependencies);
 610                                         return false;
 611                                 }
 612
 613                                 expected_message = line.Substring (index + 1).Trim ();
 614                         }
 615
 616                         if (row == 2) {
 617                                 string filtered = line.Replace(" ", "");
 618
 619                                 // Some error tests require to have different error text for different runtimes.
 620                                 if (filtered.StartsWith ("//GMCS")) {
 621                                         row = 1;
 622 #if !NET_2_0
 623                                         return true;
 624 #else
 625                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
 626 #endif
 627                                 }
 628
 629                                 check_error_line = !filtered.StartsWith ("//Line:0");
 630
 631                                 if (!filtered.StartsWith ("//Line:")) {
 632                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
 633                                         ++syntax_errors;
 634                                         return false;
 635                                 }
 636                         }
 637
 638                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
 639                                 return false;
 640
 641                         is_warning = false;
 642                         if (compiler_options != null) {
 643                                 foreach (string s in compiler_options) {
 644                                         if (s.EndsWith ("warnaserror"))
 645                                                 is_warning = true;
 646                                 }
 647                         }
 648                         return true;
 649                 }
 650
 651
 652                 protected override bool Check (TestCase test)
 653                 {
 654                         string filename = test.FileName;
 655
 656                         int start_char = 0;
 657                         while (Char.IsLetter (filename, start_char))
 658                                 ++start_char;
 659
 660                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
 661                         string expected = filename.Substring (start_char, end_char - start_char);
 662
 663                         try {
 664                                 if (base.Check (test)) {
 665                                         HandleFailure (filename, CompilerError.Missing);
 666                                         return false;
 667                                 }
 668                         }
 669                         catch (Exception e) {
 670                                 HandleFailure (filename, CompilerError.Missing);
 671                                 if (e.InnerException != null)
 672                                         e = e.InnerException;
 673
 674                                 Log (e.ToString ());
 675                                 return false;
 676                         }
 677
 678                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
 679                         if (tester.IsWarning (err_id)) {
 680                                 if (!is_warning)
 681                                         wrong_warning [err_id] = true;
 682                         } else {
 683                                 if (is_warning)
 684                                         wrong_warning [err_id] = false;
 685                         }
 686
 687                         CompilerError result_code = GetCompilerError (expected, tester.Output);
 688                         if (HandleFailure (filename, result_code)) {
 689                                 success++;
 690                                 return true;
 691                         }
 692
 693                         if (result_code == CompilerError.Wrong)
 694                                 LogLine (tester.Output);
 695
 696                         return false;
 697                 }
 698
 699                 CompilerError GetCompilerError (string expected, string buffer)
 700                 {
 701                         const string error_prefix = "CS";
 702                         const string ignored_error = "error CS5001";
 703                         string tested_text = "error " + error_prefix + expected;
 704                         StringReader sr = new StringReader (buffer);
 705                         string line = sr.ReadLine ();
 706                         ArrayList ld = new ArrayList ();
 707                         CompilerError result = CompilerError.Missing;
 708                         while (line != null) {
 709                                 if (ld.Contains (line)) {
 710                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
 711                                                 return CompilerError.Duplicate;
 712                                 }
 713                                 ld.Add (line);
 714
 715                                 if (result != CompilerError.Expected) {
 716                                         if (line.IndexOf (tested_text) != -1) {
 717                                                 if (check_msg) {
 718                                                         int first = line.IndexOf (':');
 719                                                         int second = line.IndexOf (':', first + 1);
 720                                                         if (second == -1 || !check_error_line)
 721                                                                 second = first;
 722
 723                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
 724                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
 725                                                                 error_message = msg;
 726                                                                 return CompilerError.WrongMessage;
 727                                                         }
 728
 729                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
 730                                                                 return CompilerError.MissingLocation;
 731                                                 }
 732                                                 result = CompilerError.Expected;
 733                                         } else if (line.IndexOf (error_prefix) != -1 &&
 734                                                 line.IndexOf (ignored_error) == -1)
 735                                                 result = CompilerError.Wrong;
 736                                 }
 737
 738                                 line = sr.ReadLine ();
 739                         }
 740
 741                         return result;
 742                 }
 743
 744                 bool HandleFailure (string file, CompilerError status)
 745                 {
 746                         switch (status) {
 747                                 case CompilerError.Expected:
 748                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
 749                                                 LogFileLine (file, "FIXED ISSUE");
 750                                                 return true;
 751                                         }
 752
 753                                         if (verbose)
 754                                                 LogFileLine (file, "OK");
 755                                         return true;
 756
 757                                 case CompilerError.Wrong:
 758                                         if (know_issues.Contains (file)) {
 759                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
 760                                                 know_issues.Remove (file);
 761                                                 return false;
 762                                         }
 763                                         if (no_error_list.Contains (file)) {
 764                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
 765                                                 no_error_list.Remove (file);
 766                                         }
 767                                         else {
 768                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
 769                                         }
 770                                         break;
 771
 772                                 case CompilerError.WrongMessage:
 773                                         if (know_issues.Contains (file)) {
 774                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
 775                                                 know_issues.Remove (file);
 776                                                 return false;
 777                                         }
 778                                         if (no_error_list.Contains (file)) {
 779                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
 780                                                 no_error_list.Remove (file);
 781                                         }
 782                                         else {
 783                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
 784                                                 LogLine ("Exp: {0}", expected_message);
 785                                                 LogLine ("Was: {0}", error_message);
 786                                         }
 787                                         break;
 788
 789                                 case CompilerError.Missing:
 790                                         if (no_error_list.Contains (file)) {
 791                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
 792                                                 no_error_list.Remove (file);
 793                                                 return false;
 794                                         }
 795
 796                                         if (know_issues.Contains (file)) {
 797                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
 798                                                 know_issues.Remove (file);
 799                                         }
 800                                         else {
 801                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
 802                                         }
 803
 804                                         break;
 805
 806                                 case CompilerError.MissingLocation:
 807                                         if (know_issues.Contains (file)) {
 808                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
 809                                                 know_issues.Remove (file);
 810                                                 return false;
 811                                         }
 812                                         if (no_error_list.Contains (file)) {
 813                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
 814                                                 no_error_list.Remove (file);
 815                                         }
 816                                         else {
 817                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
 818                                         }
 819                                         break;
 820
 821                                 case CompilerError.Duplicate:
 822                                         // Will become an error soon
 823                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
 824                                         return true;
 825                         }
 826
 827                         regression.Add (file);
 828                         return false;
 829                 }
 830
 831                 public override void PrintSummary()
 832                 {
 833                         base.PrintSummary ();
 834
 835                         if (wrong_warning.Count > 0) {
 836                                 LogLine ("");
 837                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
 838                                 LogLine ("");
 839                                 foreach (DictionaryEntry de in wrong_warning)
 840                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
 841                         }
 842                 }
 843
 844         }
 845
 846         class Tester {
 847
 848                 static int Main(string[] args) {
 849                         if (args.Length != 5) {
 850                                 Console.Error.WriteLine ("Usage: TestRunner [negative|positive] test-pattern compiler know-issues log-file");
 851                                 return 1;
 852                         }
 853
 854                         string mode = args[0].ToLower ();
 855 #if NET_2_0
 856                         string test_pattern = args [1] == "0" ? "*cs*.cs" : "*test-*.cs"; //args [1];
 857 #else
 858                         string test_pattern = args [1] == "0" ? "cs*.cs" : "test-*.cs"; //args [1];
 859 #endif
 860                         string mcs = args [2];
 861                         string issue_file = args [3];
 862                         string log_fname = args [4];
 863
 864                         string[] files = Directory.GetFiles (".", test_pattern);
 865
 866                         ITester tester;
 867                         try {
 868                                 Console.WriteLine ("Testing: " + mcs);
 869                                 tester = new ReflectionTester (Assembly.LoadFile (mcs));
 870                         }
 871                         catch (Exception) {
 872 #if NET_2_1
 873                                 throw;
 874 #else
 875                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
 876                                 if (!File.Exists (mcs)) {
 877                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
 878                                         return 1;
 879                                 }
 880                                 tester = new ProcessTester (mcs);
 881 #endif
 882                         }
 883
 884                         Checker checker;
 885                         switch (mode) {
 886                                 case "negative":
 887                                         checker = new NegativeChecker (tester, log_fname, issue_file, true);
 888                                         break;
 889                                 case "positive":
 890                                         checker = new PositiveChecker (tester, log_fname, issue_file);
 891                                         break;
 892                                 default:
 893                                         Console.Error.WriteLine ("You must specify testing mode (positive or negative)");
 894                                         return 1;
 895                         }
 896
 897                         foreach (string s in files) {
 898                                 string filename = Path.GetFileName (s);
 899                                 if (Char.IsUpper (filename, 0)) { // Windows hack
 900                                         continue;
 901                                 }
 902
 903                                 if (filename.EndsWith ("-p2.cs"))
 904                                         continue;
 905
 906                                 checker.Do (filename);
 907                         }
 908
 909                         checker.PrintSummary ();
 910
 911                         checker.Dispose ();
 912
 913                         return checker.ResultCode;
 914                 }
 915         }
 916 }