mcs/tools/compiler-tester/compiler-tester.cs

   1 using System;
   2 using System.IO;
   3 using System.Diagnostics;
   4 using System.Reflection;
   5 using System.Text;
   6 using System.Collections;
   7
   8 namespace TestRunner {
   9
  10         interface ITester
  11         {
  12                 string Output { get; }
  13                 bool Invoke (string[] args);
  14                 bool IsWarning (int warningNumber);
  15         }
  16
  17         class ReflectionTester: ITester {
  18                 MethodInfo ep;
  19                 object[] method_arg;
  20                 StringWriter output;
  21                 int[] all_warnings;
  22
  23                 public ReflectionTester (Assembly a)
  24                 {
  25                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  26                         ep = t.GetMethod ("InvokeCompiler",
  27                                 BindingFlags.Static | BindingFlags.Public);
  28                         if (ep == null)
  29                                 throw new MissingMethodException ("static InvokeCompiler");
  30                         method_arg = new object [2];
  31
  32                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  33                         all_warnings = (int[])pi.GetValue (null, null);
  34                         Array.Sort (all_warnings);
  35                 }
  36
  37                 public string Output {
  38                         get {
  39                                 return output.GetStringBuilder ().ToString ();
  40                         }
  41                 }
  42
  43                 public bool Invoke(string[] args)
  44                 {
  45                         output = new StringWriter ();
  46                         method_arg [0] = args;
  47                         method_arg [1] = output;
  48                         return (bool)ep.Invoke (null, method_arg);
  49                 }
  50
  51                 public bool IsWarning (int warningNumber)
  52                 {
  53                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  54                 }
  55         }
  56
  57         class ProcessTester: ITester
  58         {
  59                 ProcessStartInfo pi;
  60                 string output;
  61
  62                 public ProcessTester (string p_path)
  63                 {
  64                         pi = new ProcessStartInfo ();
  65                         pi.FileName = p_path;
  66                         pi.CreateNoWindow = true;
  67                         pi.WindowStyle = ProcessWindowStyle.Hidden;
  68                         pi.RedirectStandardOutput = true;
  69                         pi.RedirectStandardError = true;
  70                         pi.UseShellExecute = false;
  71                 }
  72
  73                 public string Output {
  74                         get {
  75                                 return output;
  76                         }
  77                 }
  78
  79                 public bool Invoke(string[] args)
  80                 {
  81                         StringBuilder sb = new StringBuilder ("/nologo ");
  82                         foreach (string s in args) {
  83                                 sb.Append (s);
  84                                 sb.Append (" ");
  85                         }
  86                         pi.Arguments = sb.ToString ();
  87                         Process p = Process.Start (pi);
  88                         output = p.StandardError.ReadToEnd ();
  89                         if (output.Length == 0)
  90                             output = p.StandardOutput.ReadToEnd ();
  91                         p.WaitForExit ();
  92                         return p.ExitCode == 0;
  93                 }
  94
  95                 public bool IsWarning (int warningNumber)
  96                 {
  97                         throw new NotImplementedException ();
  98                 }
  99         }
 100
 101         class TestCase
 102         {
 103                 public readonly string FileName;
 104                 public readonly string[] CompilerOptions;
 105                 public readonly string[] Dependencies;
 106
 107                 public TestCase (string filename, string[] options, string[] deps)
 108                 {
 109                         this.FileName = filename;
 110                         this.CompilerOptions = options;
 111                         this.Dependencies = deps;
 112                 }
 113         }
 114
 115         class Checker: IDisposable
 116         {
 117                 protected ITester tester;
 118                 protected int success;
 119                 protected int total;
 120                 protected int ignored;
 121                 string issue_file;
 122                 StreamWriter log_file;
 123                 // protected string[] compiler_options;
 124                 // protected string[] dependencies;
 125
 126                 protected ArrayList tests = new ArrayList ();
 127                 protected Hashtable test_hash = new Hashtable ();
 128                 protected ArrayList regression = new ArrayList ();
 129                 protected ArrayList know_issues = new ArrayList ();
 130                 protected ArrayList ignore_list = new ArrayList ();
 131                 protected ArrayList no_error_list = new ArrayList ();
 132
 133                 int total_known_issues;
 134
 135                 protected Checker (ITester tester, string log_file, string issue_file)
 136                 {
 137                         this.tester = tester;
 138                         this.issue_file = issue_file;
 139                         ReadWrongErrors (issue_file);
 140                         this.log_file = new StreamWriter (log_file, false);
 141                 }
 142
 143                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 144                                                         out string[] dependencies)
 145                 {
 146                         int row = 0;
 147                         compiler_options = null;
 148                         dependencies = null;
 149                         try {
 150                                 using (StreamReader sr = new StreamReader (file)) {
 151                                         String line;
 152                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 153                                                 if (!AnalyzeTestFile (ref row, line, ref compiler_options,
 154                                                                       ref dependencies))
 155                                                         return false;
 156                                         }
 157                                 }
 158                         } catch {
 159                                 return false;
 160                         }
 161                         return true;
 162                 }
 163
 164                 protected virtual bool AnalyzeTestFile (ref int row, string line,
 165                                                         ref string[] compiler_options,
 166                                                         ref string[] dependencies)
 167                 {
 168                         const string options = "// Compiler options:";
 169                         const string depends = "// Dependencies:";
 170
 171                         if (row == 1) {
 172                                 compiler_options = null;
 173                                 dependencies = null;
 174                         }
 175
 176                         int index = line.IndexOf (options);
 177                         if (index != -1) {
 178                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 179                                 for (int i = 0; i < compiler_options.Length; i++)
 180                                         compiler_options[i] = compiler_options[i].TrimStart ();
 181                         }
 182                         index = line.IndexOf (depends);
 183                         if (index != -1) {
 184                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 185                                 for (int i = 0; i < dependencies.Length; i++)
 186                                         dependencies[i] = dependencies[i].TrimStart ();
 187                         }
 188
 189                         return true;
 190                 }
 191
 192                 public bool Do (string filename)
 193                 {
 194                         if (test_hash.Contains (filename))
 195                                 return true;
 196
 197                         if (ignore_list.Contains (filename)) {
 198                                 ++ignored;
 199                                 LogLine (filename + "...\tNOT TESTED");
 200                                 return false;
 201                         }
 202
 203                         string[] compiler_options, dependencies;
 204                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 205                                 LogLine (filename + "...\tERROR");
 206                                 return false;
 207                         }
 208
 209                         TestCase test = new TestCase (filename, compiler_options, dependencies);
 210                         test_hash.Add (filename, test);
 211
 212                         ++total;
 213                         if (dependencies != null) {
 214                                 foreach (string dependency in dependencies) {
 215                                         if (!Do (dependency)) {
 216                                                 LogLine (filename + "...\tDEPENDENCY FAILED");
 217                                                 return false;
 218                                         }
 219                                 }
 220                         }
 221
 222                         tests.Add (test);
 223
 224                         Log (filename);
 225                         Log ("...\t");
 226
 227                         return Check (test);
 228                 }
 229
 230                 protected virtual bool Check (TestCase test)
 231                 {
 232                         string[] test_args;
 233
 234                         if (test.CompilerOptions != null) {
 235                                 test_args = new string [1 + test.CompilerOptions.Length];
 236                                 test.CompilerOptions.CopyTo (test_args, 0);
 237                         } else {
 238                                 test_args = new string [1];
 239                         }
 240                         test_args [test_args.Length - 1] = test.FileName;
 241
 242                         return tester.Invoke (test_args);
 243                 }
 244
 245
 246                 void ReadWrongErrors (string file)
 247                 {
 248                         const string ignored = "IGNORE";
 249                         const string no_error = "NO ERROR";
 250
 251                         using (StreamReader sr = new StreamReader (file)) {
 252                                 string line;
 253                                 while ((line = sr.ReadLine()) != null) {
 254                                         if (line.StartsWith ("#"))
 255                                                 continue;
 256
 257                                         ArrayList active_cont = know_issues;
 258
 259                                         if (line.IndexOf (ignored) > 0)
 260                                                 active_cont = ignore_list;
 261                                         else if (line.IndexOf (no_error) > 0)
 262                                                 active_cont = no_error_list;
 263
 264                                         string file_name = line.Split (' ')[0];
 265                                         if (file_name.Length == 0)
 266                                                 continue;
 267
 268                                         active_cont.Add (file_name);
 269                                 }
 270                         }
 271                         total_known_issues = know_issues.Count;
 272                 }
 273
 274                 public virtual void PrintSummary ()
 275                 {
 276                         LogLine ("Done" + Environment.NewLine);
 277                         LogLine ("{0} test cases passed ({1:.##%})", success, (float) (success) / (float)total);
 278
 279                         if (ignored > 0)
 280                                 LogLine ("{0} test cases ignored", ignored);
 281
 282                         if (total_known_issues - know_issues.Count > 0)
 283                                 LogLine ("{0} known issues", total_known_issues - know_issues.Count);
 284
 285                         know_issues.AddRange (no_error_list);
 286                         if (know_issues.Count > 0) {
 287                                 LogLine ("");
 288                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 289                                 foreach (string s in know_issues)
 290                                         LogLine (s);
 291                         }
 292                         if (regression.Count > 0) {
 293                                 LogLine ("");
 294                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 295                                 foreach (string s in regression)
 296                                         LogLine (s);
 297                         }
 298                 }
 299
 300                 public int ResultCode
 301                 {
 302                         get {
 303                                 return regression.Count == 0 ? 0 : 1;
 304                         }
 305                 }
 306
 307                 protected void Log (string msg, params object [] rest)
 308                 {
 309                         Console.Write (msg, rest);
 310                         log_file.Write (msg, rest);
 311                 }
 312
 313                 protected void LogLine (string msg, params object [] rest)
 314                 {
 315                         Console.WriteLine (msg, rest);
 316                         log_file.WriteLine (msg, rest);
 317                 }
 318
 319                 #region IDisposable Members
 320
 321                 public void Dispose()
 322                 {
 323                         log_file.Close ();
 324                 }
 325
 326                 #endregion
 327         }
 328
 329         class PositiveChecker: Checker {
 330                 readonly string files_folder;
 331                 readonly static object[] default_args = new object[1] { new string[] {} };
 332                 string doc_output;
 333
 334                 // When we cannot load assembly to domain we are automaticaly switching to process calling
 335                 bool appdomain_limit_reached;
 336
 337                 ProcessStartInfo pi;
 338                 readonly string mono;
 339
 340                 // This is really pain
 341                 // This number is highly experimental on my box I am not able to load more than 1000 files to domain
 342                 // Every files is there twice and we need some space for tests which reference assemblies
 343                 const int MAX_TESTS_IN_DOMAIN = 420;
 344                 int test_counter;
 345
 346                 protected enum TestResult {
 347                         CompileError,
 348                         ExecError,
 349                         LoadError,
 350                         XmlError,
 351                         Success
 352                 }
 353
 354                 public PositiveChecker (ITester tester, string log_file, string issue_file):
 355                         base (tester, log_file, issue_file)
 356                 {
 357                         files_folder = Directory.GetCurrentDirectory ();
 358
 359                         pi = new ProcessStartInfo ();
 360                         pi.CreateNoWindow = true;
 361                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 362                         pi.RedirectStandardOutput = true;
 363                         pi.RedirectStandardError = true;
 364                         pi.UseShellExecute = false;
 365
 366                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 367                         if (mono != null) {
 368                                 pi.FileName = mono;
 369                         }
 370                 }
 371
 372                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 373                                                         out string[] dependencies) {
 374                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 375                                 return false;
 376
 377                         doc_output = null;
 378                         if (compiler_options == null)
 379                                 return true;
 380
 381                         foreach (string one_opt in compiler_options) {
 382                                 if (one_opt.StartsWith ("-doc:")) {
 383                                         doc_output = one_opt.Split (':')[1];
 384                                 }
 385                         }
 386                         return true;
 387                 }
 388
 389                 protected override bool Check(TestCase test)
 390                 {
 391                         string filename = test.FileName;
 392                         try {
 393                                 if (!base.Check (test)) {
 394                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 395                                         return false;
 396                                 }
 397                         }
 398                         catch (Exception e) {
 399                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 400                                 return false;
 401                         }
 402
 403                         // Test setup
 404                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 405                                 LogLine ("OK");
 406                                 --total;
 407                                 return true;
 408                         }
 409
 410                         MethodInfo mi = null;
 411                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 412                         if (!appdomain_limit_reached) {
 413                                 try {
 414                                         mi = Assembly.LoadFile (file).EntryPoint;
 415                                         if (test_counter++ > MAX_TESTS_IN_DOMAIN)
 416                                                 appdomain_limit_reached = true;
 417                                 }
 418                                 catch (FileNotFoundException) {
 419                                         if (File.Exists (file)) {
 420                                                 Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 421                                                 appdomain_limit_reached = true;
 422                                         }
 423                                 }
 424                                 catch (Exception e) {
 425                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 426                                         return false;
 427                                 }
 428                         }
 429
 430                         if (appdomain_limit_reached) {
 431                                 if (!ExecuteFile (file, filename))
 432                                         return false;
 433                         } else {
 434                                 if (!ExecuteFile (mi, file, filename))
 435                                         return false;
 436                         }
 437
 438                         if (doc_output != null) {
 439                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 440                                 try {
 441                                         XmlComparer.Compare (ref_file, doc_output);
 442                                 }
 443                                 catch (Exception e) {
 444                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 445                                         return false;
 446                                 }
 447                         }
 448
 449                         HandleFailure (filename, TestResult.Success, null);
 450                         return true;
 451                 }
 452
 453                 int ExecFile (string exe_name, string filename)
 454                 {
 455                         if (mono == null)
 456                                 pi.FileName = exe_name;
 457                         else
 458                                 pi.Arguments = exe_name;
 459
 460                         Process p = Process.Start (pi);
 461                         p.WaitForExit ();
 462                         return p.ExitCode;
 463                 }
 464
 465                 bool ExecuteFile (string exe_name, string filename)
 466                 {
 467                         int exit_code = ExecFile (exe_name, filename);
 468                         if (exit_code == 0)
 469                                 return true;
 470
 471                         HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + exit_code);
 472                         return false;
 473                 }
 474
 475                 bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
 476                 {
 477                         TextWriter stdout = Console.Out;
 478                         TextWriter stderr = Console.Error;
 479                         Console.SetOut (TextWriter.Null);
 480                         Console.SetError (TextWriter.Null);
 481                         ParameterInfo[] pi = entry_point.GetParameters ();
 482                         object[] args = pi.Length == 0 ? null : default_args;
 483
 484                         object result = null;
 485                         try {
 486                                 try {
 487                                         result = entry_point.Invoke (null, args);
 488                                 } finally {
 489                                         Console.SetOut (stdout);
 490                                         Console.SetError (stderr);
 491                                 }
 492                         }
 493                         catch (Exception e) {
 494                                 int exit_code = ExecFile (exe_name, filename);
 495                                 if (exit_code == 0) {
 496                                         LogLine ("(appdomain method failed, external executable succeeded)");
 497                                         LogLine (e.ToString ());
 498                                         return true;
 499                                 }
 500                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 501                                 return false;
 502                         }
 503
 504                         if (result is int && (int)result != 0) {
 505                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 506                                 return false;
 507                         }
 508                         return true;
 509                 }
 510
 511                 void HandleFailure (string file, TestResult status, string extra)
 512                 {
 513                         switch (status) {
 514                                 case TestResult.Success:
 515                                         success++;
 516                                         if (know_issues.Contains (file)) {
 517                                                 LogLine ("FIXED ISSUE");
 518                                                 return;
 519                                         }
 520                                         LogLine ("OK");
 521                                         return;
 522
 523                                 case TestResult.CompileError:
 524                                         if (know_issues.Contains (file)) {
 525                                                 LogLine ("KNOWN ISSUE (Compilation error)");
 526                                                 know_issues.Remove (file);
 527                                                 return;
 528                                         }
 529                                         LogLine ("REGRESSION (SUCCESS -> COMPILATION ERROR)");
 530                                         break;
 531
 532                                 case TestResult.ExecError:
 533                                         if (know_issues.Contains (file)) {
 534                                                 LogLine ("KNOWN ISSUE (Execution error)");
 535                                                 know_issues.Remove (file);
 536                                                 return;
 537                                         }
 538                                         LogLine ("REGRESSION (SUCCESS -> EXECUTION ERROR)");
 539                                         break;
 540
 541                                 case TestResult.XmlError:
 542                                         if (know_issues.Contains (file)) {
 543                                                 LogLine ("KNOWN ISSUE (Xml comparision error)");
 544                                                 know_issues.Remove (file);
 545                                                 return;
 546                                         }
 547                                         LogLine ("REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 548                                         break;
 549
 550                                 case TestResult.LoadError:
 551                                         LogLine ("REGRESSION (SUCCESS -> LOAD ERROR)");
 552                                         break;
 553                         }
 554
 555                         if (extra != null)
 556                                 LogLine (extra);
 557
 558                         regression.Add (file);
 559                 }
 560         }
 561
 562         class NegativeChecker: Checker
 563         {
 564                 string expected_message;
 565                 string error_message;
 566                 bool check_msg;
 567                 bool check_error_line;
 568                 bool is_warning;
 569                 IDictionary wrong_warning;
 570
 571                 protected enum CompilerError {
 572                         Expected,
 573                         Wrong,
 574                         Missing,
 575                         WrongMessage,
 576                         MissingLocation,
 577                         Duplicate
 578                 }
 579
 580                 public NegativeChecker (ITester tester, string log_file, string issue_file, bool check_msg):
 581                         base (tester, log_file, issue_file)
 582                 {
 583                         this.check_msg = check_msg;
 584                         wrong_warning = new Hashtable ();
 585                 }
 586
 587                 protected override bool AnalyzeTestFile (ref int row, string line,
 588                                                         ref string[] compiler_options,
 589                                                         ref string[] dependencies)
 590                 {
 591                         if (row == 1) {
 592                                 expected_message = null;
 593
 594                                 int index = line.IndexOf (':');
 595                                 if (index == -1 || index > 15) {
 596                                         LogLine ("IGNORING: Wrong test file syntax (missing error mesage text)");
 597                                         ++ignored;
 598                                         base.AnalyzeTestFile (ref row, line, ref compiler_options,
 599                                                               ref dependencies);
 600                                         return false;
 601                                 }
 602
 603                                 expected_message = line.Substring (index + 1).Trim ();
 604                         }
 605
 606                         if (row == 2) {
 607                                 string filtered = line.Replace(" ", "");
 608
 609                                 // Some error tests require to have different error text for different runtimes.
 610                                 if (filtered.StartsWith ("//GMCS")) {
 611                                         row = 1;
 612 #if !NET_2_0
 613                                         return true;
 614 #else
 615                                         return AnalyzeTestFile(ref row, line, ref compiler_options, ref dependencies);
 616 #endif
 617                                 }
 618
 619                                 check_error_line = !filtered.StartsWith ("//Line:0");
 620
 621                                 if (!filtered.StartsWith ("//Line:")) {
 622                                         LogLine ("IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
 623                                         ++ignored;
 624                                         return false;
 625                                 }
 626                         }
 627
 628                         if (!base.AnalyzeTestFile (ref row, line, ref compiler_options, ref dependencies))
 629                                 return false;
 630
 631                         is_warning = false;
 632                         if (compiler_options != null) {
 633                                 foreach (string s in compiler_options) {
 634                                         if (s.EndsWith ("warnaserror"))
 635                                                 is_warning = true;
 636                                 }
 637                         }
 638                         return true;
 639                 }
 640
 641
 642                 protected override bool Check (TestCase test)
 643                 {
 644                         string filename = test.FileName;
 645
 646                         int start_char = 0;
 647                         while (Char.IsLetter (filename, start_char))
 648                                 ++start_char;
 649
 650                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
 651                         string expected = filename.Substring (start_char, end_char - start_char);
 652
 653                         try {
 654                                 if (base.Check (test)) {
 655                                         HandleFailure (filename, CompilerError.Missing);
 656                                         return false;
 657                                 }
 658                         }
 659                         catch (Exception e) {
 660                                 HandleFailure (filename, CompilerError.Missing);
 661                                 Log (e.ToString ());
 662                                 return false;
 663                         }
 664
 665                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
 666                         if (tester.IsWarning (err_id)) {
 667                                 if (!is_warning)
 668                                         wrong_warning [err_id] = true;
 669                         } else {
 670                                 if (is_warning)
 671                                         wrong_warning [err_id] = false;
 672                         }
 673
 674                         CompilerError result_code = GetCompilerError (expected, tester.Output);
 675                         if (HandleFailure (filename, result_code)) {
 676                                 success++;
 677                                 return true;
 678                         }
 679
 680                         if (result_code == CompilerError.Wrong)
 681                                 LogLine (tester.Output);
 682
 683                         return false;
 684                 }
 685
 686                 CompilerError GetCompilerError (string expected, string buffer)
 687                 {
 688                         const string error_prefix = "CS";
 689                         const string ignored_error = "error CS5001";
 690                         string tested_text = "error " + error_prefix + expected;
 691                         StringReader sr = new StringReader (buffer);
 692                         string line = sr.ReadLine ();
 693                         ArrayList ld = new ArrayList ();
 694                         CompilerError result = CompilerError.Missing;
 695                         while (line != null) {
 696                                 if (ld.Contains (line)) {
 697                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
 698                                                 return CompilerError.Duplicate;
 699                                 }
 700                                 ld.Add (line);
 701
 702                                 if (result != CompilerError.Expected) {
 703                                         if (line.IndexOf (tested_text) != -1) {
 704                                                 if (check_msg) {
 705                                                         int first = line.IndexOf (':');
 706                                                         int second = line.IndexOf (':', first + 1);
 707                                                         if (second == -1 || !check_error_line)
 708                                                                 second = first;
 709
 710                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
 711                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
 712                                                                 error_message = msg;
 713                                                                 return CompilerError.WrongMessage;
 714                                                         }
 715
 716                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
 717                                                                 return CompilerError.MissingLocation;
 718                                                 }
 719                                                 result = CompilerError.Expected;
 720                                         } else if (line.IndexOf (error_prefix) != -1 &&
 721                                                 line.IndexOf (ignored_error) == -1)
 722                                                 result = CompilerError.Wrong;
 723                                 }
 724
 725                                 line = sr.ReadLine ();
 726                         }
 727
 728                         return result;
 729                 }
 730
 731                 bool HandleFailure (string file, CompilerError status)
 732                 {
 733                         switch (status) {
 734                                 case CompilerError.Expected:
 735                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
 736                                                 LogLine ("FIXED ISSUE");
 737                                                 return true;
 738                                         }
 739                                         LogLine ("OK");
 740                                         return true;
 741
 742                                 case CompilerError.Wrong:
 743                                         if (know_issues.Contains (file)) {
 744                                                 LogLine ("KNOWN ISSUE (Wrong error reported)");
 745                                                 know_issues.Remove (file);
 746                                                 return false;
 747                                         }
 748                                         if (no_error_list.Contains (file)) {
 749                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR CODE)");
 750                                                 no_error_list.Remove (file);
 751                                         }
 752                                         else {
 753                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
 754                                         }
 755                                         break;
 756
 757                                 case CompilerError.WrongMessage:
 758                                         if (know_issues.Contains (file)) {
 759                                                 LogLine ("KNOWN ISSUE (Wrong error message reported)");
 760                                                 know_issues.Remove (file);
 761                                                 return false;
 762                                         }
 763                                         if (no_error_list.Contains (file)) {
 764                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
 765                                                 no_error_list.Remove (file);
 766                                         }
 767                                         else {
 768                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
 769                                                 LogLine ("Exp: {0}", expected_message);
 770                                                 LogLine ("Was: {0}", error_message);
 771                                         }
 772                                         break;
 773
 774                                 case CompilerError.Missing:
 775                                         if (no_error_list.Contains (file)) {
 776                                                 LogLine ("KNOWN ISSUE (No error reported)");
 777                                                 no_error_list.Remove (file);
 778                                                 return false;
 779                                         }
 780
 781                                         if (know_issues.Contains (file)) {
 782                                                 LogLine ("REGRESSION (WRONG ERROR -> NO ERROR)");
 783                                                 know_issues.Remove (file);
 784                                         }
 785                                         else {
 786                                                 LogLine ("REGRESSION (CORRECT ERROR -> NO ERROR)");
 787                                         }
 788
 789                                         break;
 790
 791                                 case CompilerError.MissingLocation:
 792                                         if (know_issues.Contains (file)) {
 793                                                 LogLine ("KNOWN ISSUE (Missing error location)");
 794                                                 know_issues.Remove (file);
 795                                                 return false;
 796                                         }
 797                                         if (no_error_list.Contains (file)) {
 798                                                 LogLine ("REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
 799                                                 no_error_list.Remove (file);
 800                                         }
 801                                         else {
 802                                                 LogLine ("REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
 803                                         }
 804                                         break;
 805
 806                                 case CompilerError.Duplicate:
 807                                         // Will become an error soon
 808                                         LogLine("WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
 809                                         return true;
 810                         }
 811
 812                         regression.Add (file);
 813                         return false;
 814                 }
 815
 816                 public override void PrintSummary()
 817                 {
 818                         base.PrintSummary ();
 819
 820                         if (wrong_warning.Count > 0) {
 821                                 LogLine ("");
 822                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
 823                                 LogLine ("");
 824                                 foreach (DictionaryEntry de in wrong_warning)
 825                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
 826                         }
 827                 }
 828
 829         }
 830
 831         class Tester {
 832
 833                 static int Main(string[] args) {
 834                         if (args.Length != 5) {
 835                                 Console.Error.WriteLine ("Usage: TestRunner [negative|positive] test-pattern compiler know-issues log-file");
 836                                 return 1;
 837                         }
 838
 839                         string mode = args[0].ToLower ();
 840 #if NET_2_0
 841                         string test_pattern = args [1] == "0" ? "*cs*.cs" : "*test-*.cs"; //args [1];
 842 #else
 843                         string test_pattern = args [1] == "0" ? "cs*.cs" : "test-*.cs"; //args [1];
 844 #endif
 845                         string mcs = args [2];
 846                         string issue_file = args [3];
 847                         string log_fname = args [4];
 848
 849                         string[] files = Directory.GetFiles (".", test_pattern);
 850
 851                         ITester tester;
 852                         try {
 853                                 Console.WriteLine ("Loading: " + mcs);
 854                                 tester = new ReflectionTester (Assembly.LoadFile (mcs));
 855                         }
 856                         catch (Exception) {
 857                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
 858                                 if (!File.Exists (mcs)) {
 859                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
 860                                         return 1;
 861                                 }
 862                                 tester = new ProcessTester (mcs);
 863                         }
 864
 865                         Checker checker;
 866                         switch (mode) {
 867                                 case "negative":
 868                                         checker = new NegativeChecker (tester, log_fname, issue_file, true);
 869                                         break;
 870                                 case "positive":
 871                                         checker = new PositiveChecker (tester, log_fname, issue_file);
 872                                         break;
 873                                 default:
 874                                         Console.Error.WriteLine ("You must specify testing mode (positive or negative)");
 875                                         return 1;
 876                         }
 877
 878                         foreach (string s in files) {
 879                                 string filename = Path.GetFileName (s);
 880                                 if (Char.IsUpper (filename, 0)) { // Windows hack
 881                                         continue;
 882                                 }
 883
 884                                 if (filename.EndsWith ("-p2.cs"))
 885                                         continue;
 886
 887                                 checker.Do (filename);
 888                         }
 889
 890                         checker.PrintSummary ();
 891
 892                         checker.Dispose ();
 893
 894                         return checker.ResultCode;
 895                 }
 896         }
 897 }