mcs/tools/compiler-tester/compiler-tester.cs

   1 using System;
   2 using System.IO;
   3 using System.Diagnostics;
   4 using System.Reflection;
   5 using System.Text;
   6 using System.Collections;
   7
   8 namespace TestRunner {
   9
  10         interface ITester
  11         {
  12                 string Output { get; }
  13                 bool Invoke (string[] args);
  14                 bool IsWarning (int warningNumber);
  15         }
  16
  17         class ReflectionTester: ITester {
  18                 MethodInfo ep;
  19                 object[] method_arg;
  20                 StringWriter output;
  21                 int[] all_warnings;
  22
  23                 public ReflectionTester (Assembly a)
  24                 {
  25                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  26                         ep = t.GetMethod ("InvokeCompiler",
  27                                 BindingFlags.Static | BindingFlags.Public);
  28                         if (ep == null)
  29                                 throw new MissingMethodException ("static InvokeCompiler");
  30                         method_arg = new object [2];
  31
  32                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  33                         all_warnings = (int[])pi.GetValue (null, null);
  34                         Array.Sort (all_warnings);
  35                 }
  36
  37                 public string Output {
  38                         get {
  39                                 return output.GetStringBuilder ().ToString ();
  40                         }
  41                 }
  42
  43                 public bool Invoke(string[] args)
  44                 {
  45                         output = new StringWriter ();
  46                         method_arg [0] = args;
  47                         method_arg [1] = output;
  48                         return (bool)ep.Invoke (null, method_arg);
  49                 }
  50
  51                 public bool IsWarning (int warningNumber)
  52                 {
  53                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  54                 }
  55         }
  56
  57         class ProcessTester: ITester
  58         {
  59                 ProcessStartInfo pi;
  60                 string output;
  61
  62                 public ProcessTester (string p_path)
  63                 {
  64                         pi = new ProcessStartInfo ();
  65                         pi.FileName = p_path;
  66                         pi.CreateNoWindow = true;
  67                         pi.WindowStyle = ProcessWindowStyle.Hidden;
  68                         pi.RedirectStandardOutput = true;
  69                         pi.RedirectStandardError = true;
  70                         pi.UseShellExecute = false;
  71                 }
  72
  73                 public string Output {
  74                         get {
  75                                 return output;
  76                         }
  77                 }
  78
  79                 public bool Invoke(string[] args)
  80                 {
  81                         StringBuilder sb = new StringBuilder ("/nologo ");
  82                         foreach (string s in args) {
  83                                 sb.Append (s);
  84                                 sb.Append (" ");
  85                         }
  86                         pi.Arguments = sb.ToString ();
  87                         Process p = Process.Start (pi);
  88                         output = p.StandardError.ReadToEnd ();
  89                         if (output.Length == 0)
  90                             output = p.StandardOutput.ReadToEnd ();
  91                         p.WaitForExit ();
  92                         return p.ExitCode == 0;
  93                 }
  94
  95                 public bool IsWarning (int warningNumber)
  96                 {
  97                         throw new NotImplementedException ();
  98                 }
  99         }
 100
 101         class TestCase
 102         {
 103                 public readonly string FileName;
 104                 public readonly string[] CompilerOptions;
 105                 public readonly string[] Dependencies;
 106
 107                 public TestCase (string filename, string[] options, string[] deps)
 108                 {
 109                         this.FileName = filename;
 110                         this.CompilerOptions = options;
 111                         this.Dependencies = deps;
 112                 }
 113         }
 114
 115         class Checker: IDisposable
 116         {
 117                 protected ITester tester;
 118                 protected int success;
 119                 protected int total;
 120                 protected int ignored;
 121                 string issue_file;
 122                 StreamWriter log_file;
 123                 // protected string[] compiler_options;
 124                 // protected string[] dependencies;
 125
 126                 protected ArrayList tests = new ArrayList ();
 127                 protected Hashtable test_hash = new Hashtable ();
 128                 protected ArrayList regression = new ArrayList ();
 129                 protected ArrayList know_issues = new ArrayList ();
 130                 protected ArrayList ignore_list = new ArrayList ();
 131                 protected ArrayList no_error_list = new ArrayList ();
 132
 133                 int total_known_issues;
 134
 135                 protected Checker (ITester tester, string log_file, string issue_file)
 136                 {
 137                         this.tester = tester;
 138                         this.issue_file = issue_file;
 139                         ReadWrongErrors (issue_file);
 140                         this.log_file = new StreamWriter (log_file, false);
 141                 }
 142
 143                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 144                                                         out string[] dependencies)
 145                 {
 146                         int row = 0;
 147                         compiler_options = null;
 148                         dependencies = null;
 149                         try {
 150                                 using (StreamReader sr = new StreamReader (file)) {
 151                                         String line;
 152                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 153                                                 if (!AnalyzeTestFile (ref row, line, ref compiler_options,
 154                                                                       ref dependencies))
 155                                                         return false;
 156                                         }
 157                                 }
 158                         } catch {
 159                                 return false;
 160                         }
 161                         return true;
 162                 }
 163
 164                 protected virtual bool AnalyzeTestFile (ref int row, string line,
 165                                                         ref string[] compiler_options,
 166                                                         ref string[] dependencies)
 167                 {
 168                         const string options = "// Compiler options:";
 169                         const string depends = "// Dependencies:";
 170
 171                         if (row == 1) {
 172                                 compiler_options = null;
 173                                 dependencies = null;
 174                         }
 175
 176                         int index = line.IndexOf (options);
 177                         if (index != -1) {
 178                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 179                                 for (int i = 0; i < compiler_options.Length; i++)
 180                                         compiler_options[i] = compiler_options[i].TrimStart ();
 181                         }
 182                         index = line.IndexOf (depends);
 183                         if (index != -1) {
 184                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 185                                 for (int i = 0; i < dependencies.Length; i++)
 186                                         dependencies[i] = dependencies[i].TrimStart ();
 187                         }
 188
 189                         return true;
 190                 }
 191
 192                 public bool Do (string filename)
 193                 {
 194                         if (test_hash.Contains (filename))
 195                                 return true;
 196
 197                         if (ignore_list.Contains (filename)) {
 198                                 ++ignored;
 199                                 LogLine (filename + "...\tNOT TESTED");
 200                                 return false;
 201                         }
 202
 203                         string[] compiler_options, dependencies;
 204                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 205                                 LogLine (filename + "...\tERROR");
 206                                 return false;
 207                         }
 208
 209                         TestCase test = new TestCase (filename, compiler_options, dependencies);
 210                         test_hash.Add (filename, test);
 211
 212                         ++total;
 213                         if (dependencies != null) {
 214                                 foreach (string dependency in dependencies) {
 215                                         if (!Do (dependency)) {
 216                                                 LogLine (filename + "...\tDEPENDENCY FAILED");
 217                                                 return false;
 218                                         }
 219                                 }
 220                         }
 221
 222                         tests.Add (test);
 223
 224                         Log (filename);
 225                         Log ("...\t");
 226
 227                         return Check (test);
 228                 }
 229
 230                 protected virtual bool Check (TestCase test)
 231                 {
 232                         string[] test_args;
 233
 234                         if (test.CompilerOptions != null) {
 235                                 test_args = new string [1 + test.CompilerOptions.Length];
 236                                 test.CompilerOptions.CopyTo (test_args, 0);
 237                         } else {
 238                                 test_args = new string [1];
 239                         }
 240                         test_args [test_args.Length - 1] = test.FileName;
 241
 242                         return tester.Invoke (test_args);
 243                 }
 244
 245
 246                 void ReadWrongErrors (string file)
 247                 {
 248                         const string ignored = "IGNORE";
 249                         const string no_error = "NO ERROR";
 250
 251                         using (StreamReader sr = new StreamReader (file)) {
 252                                 string line;
 253                                 while ((line = sr.ReadLine()) != null) {
 254                                         if (line.StartsWith ("#"))
 255                                                 continue;
 256
 257                                         ArrayList active_cont = know_issues;
 258
 259                                         if (line.IndexOf (ignored) > 0)
 260                                                 active_cont = ignore_list;
 261                                         else if (line.IndexOf (no_error) > 0)
 262                                                 active_cont = no_error_list;
 263
 264                                         string file_name = line.Split (' ')[0];
 265                                         if (file_name.Length == 0)
 266                                                 continue;
 267
 268                                         active_cont.Add (file_name);
 269                                 }
 270                         }
 271                         total_known_issues = know_issues.Count;
 272                 }
 273
 274                 public virtual void PrintSummary ()
 275                 {
 276                         LogLine ("Done" + Environment.NewLine);
 277                         LogLine ("{0} test cases passed ({1:.##%})", success, (float) (success) / (float)total);
 278
 279                         if (ignored > 0)
 280                                 LogLine ("{0} test cases ignored", ignored);
 281
 282                         if (total_known_issues - know_issues.Count > 0)
 283                                 LogLine ("{0} known issues", total_known_issues - know_issues.Count);
 284
 285                         know_issues.AddRange (no_error_list);
 286                         if (know_issues.Count > 0) {
 287                                 LogLine ("");
 288                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 289                                 foreach (string s in know_issues)
 290                                         LogLine (s);
 291                         }
 292                         if (regression.Count > 0) {
 293                                 LogLine ("");
 294                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 295                                 foreach (string s in regression)
 296                                         LogLine (s);
 297                         }
 298                 }
 299
 300                 public int ResultCode
 301                 {
 302                         get {
 303                                 return regression.Count == 0 ? 0 : 1;
 304                         }
 305                 }
 306
 307                 protected void Log (string msg, params object [] rest)
 308                 {
 309                         Console.Write (msg, rest);
 310                         log_file.Write (msg, rest);
 311                 }
 312
 313                 protected void LogLine (string msg, params object [] rest)
 314                 {
 315                         Console.WriteLine (msg, rest);
 316                         log_file.WriteLine (msg, rest);
 317                 }
 318
 319                 #region IDisposable Members
 320
 321                 public void Dispose()
 322                 {
 323                         log_file.Close ();
 324                 }
 325
 326                 #endregion
 327         }
 328
 329         class PositiveChecker: Checker {
 330                 readonly string files_folder;
 331                 readonly static object[] default_args = new object[1] { new string[] {} };
 332                 string doc_output;
 333
 334                 // When we cannot load assembly to domain we are automaticaly switching to process calling
 335                 bool appdomain_limit_reached;
 336
 337                 ProcessStartInfo pi;
 338                 readonly string mono;
 339
 340                 // This is really pain
 341                 // This number is highly experimental on my box I am not able to load more than 1000 files to domain
 342                 // Every files is there twice and we need some space for tests which reference assemblies
 343                 const int MAX_TESTS_IN_DOMAIN = 420;
 344                 int test_counter;
 345
 346                 protected enum TestResult {
 347                         CompileError,
 348                         ExecError,
 349                         LoadError,
 350                         XmlError,
 351                         Success
 352                 }
 353
 354                 public PositiveChecker (ITester tester, string log_file, string issue_file):
 355                         base (tester, log_file, issue_file)
 356                 {
 357                         files_folder = Directory.GetCurrentDirectory ();
 358
 359                         pi = new ProcessStartInfo ();
 360                         pi.CreateNoWindow = true;
 361                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 362                         pi.RedirectStandardOutput = true;
 363                         pi.RedirectStandardError = true;
 364                         pi.UseShellExecute = false;
 365
 366                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 367                         if (mono != null) {
 368                                 pi.FileName = mono;
 369                         }
 370                 }
 371
 372                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 373                                                         out string[] dependencies) {
 374                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 375                                 return false;
 376
 377                         doc_output = null;
 378                         if (compiler_options == null)
 379                                 return true;
 380
 381                         foreach (string one_opt in compiler_options) {
 382                                 if (one_opt.StartsWith ("-doc:")) {
 383                                         doc_output = one_opt.Split (':')[1];
 384                                 }
 385                         }
 386                         return true;
 387                 }
 388
 389                 protected override bool Check(TestCase test)
 390                 {
 391                         string filename = test.FileName;
 392                         try {
 393                                 if (!base.Check (test)) {
 394                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 395                                         return false;
 396                                 }
 397                         }
 398                         catch (Exception e) {
 399                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 400                                 return false;
 401                         }
 402
 403                         // Test setup
 404                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 405                                 LogLine ("OK");
 406                                 --total;
 407                                 return true;
 408                         }
 409
 410                         MethodInfo mi = null;
 411                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 412
 413                         // Enable .dll only tests (no execution required)
 414                         if (!File.Exists(file)) {
 415                                 HandleFailure (filename, TestResult.Success, null);
 416                                 return true;
 417                         }
 418
 419                         if (!appdomain_limit_reached) {
 420                                 try {
 421                                         mi = Assembly.LoadFile (file).EntryPoint;
 422                                         if (test_counter++ > MAX_TESTS_IN_DOMAIN)
 423                                                 appdomain_limit_reached = true;
 424                                 }
 425                                 catch (FileNotFoundException) {
 426                                         if (File.Exists (file)) {
 427                                                 Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 428                                                 appdomain_limit_reached = true;
 429                                         }
 430                                 }
 431                                 catch (Exception e) {
 432                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 433                                         return false;
 434                                 }
 435                         }
 436
 437                         if (appdomain_limit_reached) {
 438                                 if (!ExecuteFile (file, filename))
 439                                         return false;
 440                         } else {
 441                                 if (!ExecuteFile (mi, file, filename))
 442                                         return false;
 443                         }
 444
 445                         if (doc_output != null) {
 446                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 447                                 try {
 448                                         XmlComparer.Compare (ref_file, doc_output);
 449                                 }
 450                                 catch (Exception e) {
 451                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 452                                         return false;
 453                                 }
 454                         }
 455
 456                         HandleFailure (filename, TestResult.Success, null);
 457                         return true;
 458                 }
 459
 460                 int ExecFile (string exe_name, string filename)
 461                 {
 462                         if (mono == null)
 463                                 pi.FileName = exe_name;
 464                         else
 465                                 pi.Arguments = exe_name;
 466
 467                         Process p = Process.Start (pi);
 468                         p.WaitForExit ();
 469                         return p.ExitCode;
 470                 }
 471
 472                 bool ExecuteFile (string exe_name, string filename)
 473                 {
 474                         int exit_code = ExecFile (exe_name, filename);
 475                         if (exit_code == 0)
 476                                 return true;
 477
 478                         HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + exit_code);
 479                         return false;
 480                 }
 481
 482                 bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
 483                 {
 484                         TextWriter stdout = Console.Out;
 485                         TextWriter stderr = Console.Error;
 486                         Console.SetOut (TextWriter.Null);
 487                         Console.SetError (TextWriter.Null);
 488                         ParameterInfo[] pi = entry_point.GetParameters ();
 489                         object[] args = pi.Length == 0 ? null : default_args;
 490
 491                         object result = null;
 492                         try {
 493                                 try {
 494                                         result = entry_point.Invoke (null, args);
 495                                 } finally {
 496                                         Console.SetOut (stdout);
 497                                         Console.SetError (stderr);
 498                                 }
 499                         }
 500                         catch (Exception e) {
 501                                 int exit_code = ExecFile (exe_name, filename);
 502                                 if (exit_code == 0) {
 503                                         LogLine ("(appdomain method failed, external executable succeeded)");
 504                                         LogLine (e.ToString ());
 505                                         return true;
 506                                 }
 507                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 508                                 return false;
 509                         }
 510
 511                         if (result is int && (int)result != 0) {
 512                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 513                                 return false;
 514                         }
 515                         return true;
 516                 }
 517
 518                 void HandleFailure (string file, TestResult status, string extra)
 519                 {
 520                         switch (status) {
 521                                 case TestResult.Success:
 522                                         success++;
 523                                         if (know_issues.Contains (file)) {
 524                                                 LogLine ("FIXED ISSUE");
 525                                                 return;
 526                                         }
 527                                         LogLine ("OK");
 528                                         return;
 529
 530                                 case TestResult.CompileError:
 531                                         if (know_issues.Contains (file)) {
 532                                                 LogLine ("KNOWN ISSUE (Compilation error)");
 533                                                 know_issues.Remove (file);
 534                                                 return;
 535                                         }
 536                                         LogLine ("REGRESSION (SUCCESS -> COMPILATION ERROR)");
 537                                         break;
 538
 539                                 case TestResult.ExecError:
 540                                         if (know_issues.Contains (file)) {
 541                                                 LogLine ("KNOWN ISSUE (Execution error)");
 542                                                 know_issues.Remove (file);
 543                                                 return;
 544                                         }
 545                                         LogLine ("REGRESSION (SUCCESS -> EXECUTION ERROR)");
 546                                         break;
 547
 548                                 case TestResult.XmlError:
 549                                         if (know_issues.Contains (file)) {
 550                                                 LogLine ("KNOWN ISSUE (Xml comparision error)");
 551                                                 know_issues.Remove (file);
 552                                                 return;
 553                                         }
 554                                         LogLine ("REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 555                                         break;
 556
 557                                 case TestResult.LoadError:
 558                                         LogLine ("REGRESSION (SUCCESS -> LOAD ERROR)");
 559                                         break;
 560                         }
 561
 562                         if (extra != null)
 563                                 LogLine ("{0}", extra);
 564
 565                         regression.Add (file);
 566                 }
 567         }
 568
 569         class NegativeChecker: Checker
 570         {
 571                 string expected_message;
 572                 string error_message;
 573                 bool check_msg;
 574                 bool check_error_line;
 575                 bool is_warning;
 576                 IDictionary wrong_warning;
 577
 578                 protected enum CompilerError {
 579                         Expected,
 580                         Wrong,
 581                         Missing,
 582                         WrongMessage,
 583                         MissingLocation,
 584                         Duplicate
 585                 }
 586
 587                 public NegativeChecker (ITester tester, string log_file, string issue_file, bool check_msg):
 588                         base (tester, log_file, issue_file)
 589                 {
 590                         this.check_msg = check_msg;
 591                         wrong_warning = new Hashtable ();
 592                 }
 593
 594                 protected override bool AnalyzeTestFile (ref int row, string line,
 595                                                         ref string[] compiler_options,
 596                                                         ref string[] dependencies)
 597                 {
 598                         if (row == 1) {
 599                                 expected_message = null;
 600
 601                                 int index = line.IndexOf (':');
 602                                 if (index == -1 || index > 15) {
 603                                         LogLine ("IGNORING: Wrong test file syntax (missing error mesage text)");
 604                                         ++ignored;
 605                                         base.AnalyzeTestFile (ref row, line, ref compiler_options,
 606                                                               ref dependencies);
 607                                         return false;
 608                                 }
 609
 610                                 expected_message = line.Substring (index + 1).Trim ();
 611                         }
 612
 613                         if (row == 2) {
 614                                 string filtered = line.Replace(" ", "");
 615
 616                                 // Some error tests require to have different error text for different runtimes.
 617                                 if (filtered.StartsWith ("//GMCS")) {
 618                                         row = 1;
 619 #if !NET_2_0
 620                                         return true;
 621 #else
 622                                         return AnalyzeTestFile(ref row, line, ref compiler_options, ref dependencies);
 623 #endif
 624                                 }
 625
 626                                 check_error_line = !filtered.StartsWith ("//Line:0");
 627
 628                                 if (!filtered.StartsWith ("//Line:")) {
 629                                         LogLine ("IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
 630                                         ++ignored;
 631                                         return false;
 632                                 }
 633                         }
 634
 635                         if (!base.AnalyzeTestFile (ref row, line, ref compiler_options, ref dependencies))
 636                                 return false;
 637
 638                         is_warning = false;
 639                         if (compiler_options != null) {
 640                                 foreach (string s in compiler_options) {
 641                                         if (s.EndsWith ("warnaserror"))
 642                                                 is_warning = true;
 643                                 }
 644                         }
 645                         return true;
 646                 }
 647
 648
 649                 protected override bool Check (TestCase test)
 650                 {
 651                         string filename = test.FileName;
 652
 653                         int start_char = 0;
 654                         while (Char.IsLetter (filename, start_char))
 655                                 ++start_char;
 656
 657                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
 658                         string expected = filename.Substring (start_char, end_char - start_char);
 659
 660                         try {
 661                                 if (base.Check (test)) {
 662                                         HandleFailure (filename, CompilerError.Missing);
 663                                         return false;
 664                                 }
 665                         }
 666                         catch (Exception e) {
 667                                 HandleFailure (filename, CompilerError.Missing);
 668                                 Log (e.ToString ());
 669                                 return false;
 670                         }
 671
 672                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
 673                         if (tester.IsWarning (err_id)) {
 674                                 if (!is_warning)
 675                                         wrong_warning [err_id] = true;
 676                         } else {
 677                                 if (is_warning)
 678                                         wrong_warning [err_id] = false;
 679                         }
 680
 681                         CompilerError result_code = GetCompilerError (expected, tester.Output);
 682                         if (HandleFailure (filename, result_code)) {
 683                                 success++;
 684                                 return true;
 685                         }
 686
 687                         if (result_code == CompilerError.Wrong)
 688                                 LogLine (tester.Output);
 689
 690                         return false;
 691                 }
 692
 693                 CompilerError GetCompilerError (string expected, string buffer)
 694                 {
 695                         const string error_prefix = "CS";
 696                         const string ignored_error = "error CS5001";
 697                         string tested_text = "error " + error_prefix + expected;
 698                         StringReader sr = new StringReader (buffer);
 699                         string line = sr.ReadLine ();
 700                         ArrayList ld = new ArrayList ();
 701                         CompilerError result = CompilerError.Missing;
 702                         while (line != null) {
 703                                 if (ld.Contains (line)) {
 704                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
 705                                                 return CompilerError.Duplicate;
 706                                 }
 707                                 ld.Add (line);
 708
 709                                 if (result != CompilerError.Expected) {
 710                                         if (line.IndexOf (tested_text) != -1) {
 711                                                 if (check_msg) {
 712                                                         int first = line.IndexOf (':');
 713                                                         int second = line.IndexOf (':', first + 1);
 714                                                         if (second == -1 || !check_error_line)
 715                                                                 second = first;
 716
 717                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
 718                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
 719                                                                 error_message = msg;
 720                                                                 return CompilerError.WrongMessage;
 721                                                         }
 722
 723                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
 724                                                                 return CompilerError.MissingLocation;
 725                                                 }
 726                                                 result = CompilerError.Expected;
 727                                         } else if (line.IndexOf (error_prefix) != -1 &&
 728                                                 line.IndexOf (ignored_error) == -1)
 729                                                 result = CompilerError.Wrong;
 730                                 }
 731
 732                                 line = sr.ReadLine ();
 733                         }
 734
 735                         return result;
 736                 }
 737
 738                 bool HandleFailure (string file, CompilerError status)
 739                 {
 740                         switch (status) {
 741                                 case CompilerError.Expected:
 742                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
 743                                                 LogLine ("FIXED ISSUE");
 744                                                 return true;
 745                                         }
 746                                         LogLine ("OK");
 747                                         return true;
 748
 749                                 case CompilerError.Wrong:
 750                                         if (know_issues.Contains (file)) {
 751                                                 LogLine ("KNOWN ISSUE (Wrong error reported)");
 752                                                 know_issues.Remove (file);
 753                                                 return false;
 754                                         }
 755                                         if (no_error_list.Contains (file)) {
 756                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR CODE)");
 757                                                 no_error_list.Remove (file);
 758                                         }
 759                                         else {
 760                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
 761                                         }
 762                                         break;
 763
 764                                 case CompilerError.WrongMessage:
 765                                         if (know_issues.Contains (file)) {
 766                                                 LogLine ("KNOWN ISSUE (Wrong error message reported)");
 767                                                 know_issues.Remove (file);
 768                                                 return false;
 769                                         }
 770                                         if (no_error_list.Contains (file)) {
 771                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
 772                                                 no_error_list.Remove (file);
 773                                         }
 774                                         else {
 775                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
 776                                                 LogLine ("Exp: {0}", expected_message);
 777                                                 LogLine ("Was: {0}", error_message);
 778                                         }
 779                                         break;
 780
 781                                 case CompilerError.Missing:
 782                                         if (no_error_list.Contains (file)) {
 783                                                 LogLine ("KNOWN ISSUE (No error reported)");
 784                                                 no_error_list.Remove (file);
 785                                                 return false;
 786                                         }
 787
 788                                         if (know_issues.Contains (file)) {
 789                                                 LogLine ("REGRESSION (WRONG ERROR -> NO ERROR)");
 790                                                 know_issues.Remove (file);
 791                                         }
 792                                         else {
 793                                                 LogLine ("REGRESSION (CORRECT ERROR -> NO ERROR)");
 794                                         }
 795
 796                                         break;
 797
 798                                 case CompilerError.MissingLocation:
 799                                         if (know_issues.Contains (file)) {
 800                                                 LogLine ("KNOWN ISSUE (Missing error location)");
 801                                                 know_issues.Remove (file);
 802                                                 return false;
 803                                         }
 804                                         if (no_error_list.Contains (file)) {
 805                                                 LogLine ("REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
 806                                                 no_error_list.Remove (file);
 807                                         }
 808                                         else {
 809                                                 LogLine ("REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
 810                                         }
 811                                         break;
 812
 813                                 case CompilerError.Duplicate:
 814                                         // Will become an error soon
 815                                         LogLine("WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
 816                                         return true;
 817                         }
 818
 819                         regression.Add (file);
 820                         return false;
 821                 }
 822
 823                 public override void PrintSummary()
 824                 {
 825                         base.PrintSummary ();
 826
 827                         if (wrong_warning.Count > 0) {
 828                                 LogLine ("");
 829                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
 830                                 LogLine ("");
 831                                 foreach (DictionaryEntry de in wrong_warning)
 832                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
 833                         }
 834                 }
 835
 836         }
 837
 838         class Tester {
 839
 840                 static int Main(string[] args) {
 841                         if (args.Length != 5) {
 842                                 Console.Error.WriteLine ("Usage: TestRunner [negative|positive] test-pattern compiler know-issues log-file");
 843                                 return 1;
 844                         }
 845
 846                         string mode = args[0].ToLower ();
 847 #if NET_2_0
 848                         string test_pattern = args [1] == "0" ? "*cs*.cs" : "*test-*.cs"; //args [1];
 849 #else
 850                         string test_pattern = args [1] == "0" ? "cs*.cs" : "test-*.cs"; //args [1];
 851 #endif
 852                         string mcs = args [2];
 853                         string issue_file = args [3];
 854                         string log_fname = args [4];
 855
 856                         string[] files = Directory.GetFiles (".", test_pattern);
 857
 858                         ITester tester;
 859                         try {
 860                                 Console.WriteLine ("Loading: " + mcs);
 861                                 tester = new ReflectionTester (Assembly.LoadFile (mcs));
 862                         }
 863                         catch (Exception) {
 864                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
 865                                 if (!File.Exists (mcs)) {
 866                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
 867                                         return 1;
 868                                 }
 869                                 tester = new ProcessTester (mcs);
 870                         }
 871
 872                         Checker checker;
 873                         switch (mode) {
 874                                 case "negative":
 875                                         checker = new NegativeChecker (tester, log_fname, issue_file, true);
 876                                         break;
 877                                 case "positive":
 878                                         checker = new PositiveChecker (tester, log_fname, issue_file);
 879                                         break;
 880                                 default:
 881                                         Console.Error.WriteLine ("You must specify testing mode (positive or negative)");
 882                                         return 1;
 883                         }
 884
 885                         foreach (string s in files) {
 886                                 string filename = Path.GetFileName (s);
 887                                 if (Char.IsUpper (filename, 0)) { // Windows hack
 888                                         continue;
 889                                 }
 890
 891                                 if (filename.EndsWith ("-p2.cs"))
 892                                         continue;
 893
 894                                 checker.Do (filename);
 895                         }
 896
 897                         checker.PrintSummary ();
 898
 899                         checker.Dispose ();
 900
 901                         return checker.ResultCode;
 902                 }
 903         }
 904 }