mcs/tools/compiler-tester/compiler-tester.cs

   1 using System;
   2 using System.IO;
   3 using System.Diagnostics;
   4 using System.Reflection;
   5 using System.Text;
   6 using System.Collections;
   7
   8 namespace TestRunner {
   9
  10         interface ITester
  11         {
  12                 string Output { get; }
  13                 bool Invoke (string[] args);
  14                 bool IsWarning (int warningNumber);
  15         }
  16
  17         class ReflectionTester: ITester {
  18                 MethodInfo ep;
  19                 object[] method_arg;
  20                 StringWriter output;
  21                 int[] all_warnings;
  22
  23                 public ReflectionTester (Assembly a)
  24                 {
  25                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  26                         ep = t.GetMethod ("InvokeCompiler",
  27                                 BindingFlags.Static | BindingFlags.Public);
  28                         if (ep == null)
  29                                 throw new MissingMethodException ("static InvokeCompiler");
  30                         method_arg = new object [2];
  31
  32                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  33                         all_warnings = (int[])pi.GetValue (null, null);
  34                         Array.Sort (all_warnings);
  35                 }
  36
  37                 public string Output {
  38                         get {
  39                                 return output.GetStringBuilder ().ToString ();
  40                         }
  41                 }
  42
  43                 public bool Invoke(string[] args)
  44                 {
  45                         output = new StringWriter ();
  46                         method_arg [0] = args;
  47                         method_arg [1] = output;
  48                         return (bool)ep.Invoke (null, method_arg);
  49                 }
  50
  51                 public bool IsWarning (int warningNumber)
  52                 {
  53                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  54                 }
  55         }
  56
  57         class ProcessTester: ITester
  58         {
  59                 ProcessStartInfo pi;
  60                 string output;
  61
  62                 public ProcessTester (string p_path)
  63                 {
  64                         pi = new ProcessStartInfo ();
  65                         pi.FileName = p_path;
  66                         pi.CreateNoWindow = true;
  67                         pi.WindowStyle = ProcessWindowStyle.Hidden;
  68                         pi.RedirectStandardOutput = true;
  69                         pi.RedirectStandardError = true;
  70                         pi.UseShellExecute = false;
  71                 }
  72
  73                 public string Output {
  74                         get {
  75                                 return output;
  76                         }
  77                 }
  78
  79                 public bool Invoke(string[] args)
  80                 {
  81                         StringBuilder sb = new StringBuilder ("/nologo ");
  82                         foreach (string s in args) {
  83                                 sb.Append (s);
  84                                 sb.Append (" ");
  85                         }
  86                         pi.Arguments = sb.ToString ();
  87                         Process p = Process.Start (pi);
  88                         output = p.StandardError.ReadToEnd ();
  89                         if (output.Length == 0)
  90                             output = p.StandardOutput.ReadToEnd ();
  91                         p.WaitForExit ();
  92                         return p.ExitCode == 0;
  93                 }
  94
  95                 public bool IsWarning (int warningNumber)
  96                 {
  97                         throw new NotImplementedException ();
  98                 }
  99         }
 100
 101         class TestCase
 102         {
 103                 public readonly string FileName;
 104                 public readonly string[] CompilerOptions;
 105                 public readonly string[] Dependencies;
 106
 107                 public TestCase (string filename, string[] options, string[] deps)
 108                 {
 109                         this.FileName = filename;
 110                         this.CompilerOptions = options;
 111                         this.Dependencies = deps;
 112                 }
 113         }
 114
 115         class Checker: IDisposable
 116         {
 117                 protected ITester tester;
 118                 protected int success;
 119                 protected int total;
 120                 protected int ignored;
 121                 protected int syntax_errors;
 122                 string issue_file;
 123                 StreamWriter log_file;
 124                 // protected string[] compiler_options;
 125                 // protected string[] dependencies;
 126
 127                 protected ArrayList tests = new ArrayList ();
 128                 protected Hashtable test_hash = new Hashtable ();
 129                 protected ArrayList regression = new ArrayList ();
 130                 protected ArrayList know_issues = new ArrayList ();
 131                 protected ArrayList ignore_list = new ArrayList ();
 132                 protected ArrayList no_error_list = new ArrayList ();
 133
 134                 protected bool verbose;
 135
 136                 int total_known_issues;
 137
 138                 protected Checker (ITester tester, string log_file, string issue_file)
 139                 {
 140                         this.tester = tester;
 141                         this.issue_file = issue_file;
 142                         ReadWrongErrors (issue_file);
 143                         this.log_file = new StreamWriter (log_file, false);
 144                 }
 145
 146                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 147                                                         out string[] dependencies)
 148                 {
 149                         int row = 0;
 150                         compiler_options = null;
 151                         dependencies = null;
 152                         try {
 153                                 using (StreamReader sr = new StreamReader (file)) {
 154                                         String line;
 155                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 156                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 157                                                                       ref dependencies))
 158                                                         return false;
 159                                         }
 160                                 }
 161                         } catch {
 162                                 return false;
 163                         }
 164                         return true;
 165                 }
 166
 167                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 168                                                         ref string[] compiler_options,
 169                                                         ref string[] dependencies)
 170                 {
 171                         const string options = "// Compiler options:";
 172                         const string depends = "// Dependencies:";
 173
 174                         if (row == 1) {
 175                                 compiler_options = null;
 176                                 dependencies = null;
 177                         }
 178
 179                         int index = line.IndexOf (options);
 180                         if (index != -1) {
 181                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 182                                 for (int i = 0; i < compiler_options.Length; i++)
 183                                         compiler_options[i] = compiler_options[i].TrimStart ();
 184                         }
 185                         index = line.IndexOf (depends);
 186                         if (index != -1) {
 187                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 188                                 for (int i = 0; i < dependencies.Length; i++)
 189                                         dependencies[i] = dependencies[i].TrimStart ();
 190                         }
 191
 192                         return true;
 193                 }
 194
 195                 public bool Do (string filename)
 196                 {
 197                         if (test_hash.Contains (filename))
 198                                 return true;
 199
 200                         if (ignore_list.Contains (filename)) {
 201                                 ++ignored;
 202                                 LogFileLine (filename, "NOT TESTED");
 203                                 return false;
 204                         }
 205
 206                         string[] compiler_options, dependencies;
 207                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 208                                 LogFileLine (filename, "ERROR");
 209                                 return false;
 210                         }
 211
 212                         TestCase test = new TestCase (filename, compiler_options, dependencies);
 213                         test_hash.Add (filename, test);
 214
 215                         ++total;
 216                         if (dependencies != null) {
 217                                 foreach (string dependency in dependencies) {
 218                                         if (!Do (dependency)) {
 219                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 220                                                 return false;
 221                                         }
 222                                 }
 223                         }
 224
 225                         tests.Add (test);
 226
 227                         if (verbose) {
 228                                 Log (filename);
 229                                 Log ("...\t");
 230                         }
 231
 232                         return Check (test);
 233                 }
 234
 235                 protected virtual bool Check (TestCase test)
 236                 {
 237                         string[] test_args;
 238
 239                         if (test.CompilerOptions != null) {
 240                                 test_args = new string [1 + test.CompilerOptions.Length];
 241                                 test.CompilerOptions.CopyTo (test_args, 0);
 242                         } else {
 243                                 test_args = new string [1];
 244                         }
 245                         test_args [test_args.Length - 1] = test.FileName;
 246
 247                         return tester.Invoke (test_args);
 248                 }
 249
 250
 251                 void ReadWrongErrors (string file)
 252                 {
 253                         const string ignored = "IGNORE";
 254                         const string no_error = "NO ERROR";
 255
 256                         using (StreamReader sr = new StreamReader (file)) {
 257                                 string line;
 258                                 while ((line = sr.ReadLine()) != null) {
 259                                         if (line.StartsWith ("#"))
 260                                                 continue;
 261
 262                                         ArrayList active_cont = know_issues;
 263
 264                                         if (line.IndexOf (ignored) > 0)
 265                                                 active_cont = ignore_list;
 266                                         else if (line.IndexOf (no_error) > 0)
 267                                                 active_cont = no_error_list;
 268
 269                                         string file_name = line.Split (' ')[0];
 270                                         if (file_name.Length == 0)
 271                                                 continue;
 272
 273                                         active_cont.Add (file_name);
 274                                 }
 275                         }
 276                         total_known_issues = know_issues.Count;
 277                 }
 278
 279                 public virtual void PrintSummary ()
 280                 {
 281                         LogLine ("Done" + Environment.NewLine);
 282                         LogLine ("{0} test cases passed ({1:.##%})", success, (float) (success) / (float)total);
 283
 284                         if (syntax_errors > 0)
 285                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 286
 287                         if (ignored > 0)
 288                                 LogLine ("{0} test(s) ignored", ignored);
 289
 290                         if (total_known_issues - know_issues.Count > 0)
 291                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 292
 293                         know_issues.AddRange (no_error_list);
 294                         if (know_issues.Count > 0) {
 295                                 LogLine ("");
 296                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 297                                 foreach (string s in know_issues)
 298                                         LogLine (s);
 299                         }
 300                         if (regression.Count > 0) {
 301                                 LogLine ("");
 302                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 303                                 foreach (string s in regression)
 304                                         LogLine (s);
 305                         }
 306                 }
 307
 308                 public int ResultCode
 309                 {
 310                         get {
 311                                 return regression.Count == 0 ? 0 : 1;
 312                         }
 313                 }
 314
 315                 protected void Log (string msg, params object [] rest)
 316                 {
 317                         Console.Write (msg, rest);
 318                         log_file.Write (msg, rest);
 319                 }
 320
 321                 protected void LogLine (string msg, params object [] rest)
 322                 {
 323                         Console.WriteLine (msg, rest);
 324                         log_file.WriteLine (msg, rest);
 325                 }
 326
 327                 protected void LogFileLine (string file, string msg, params object [] args)
 328                 {
 329                         string s = file + "...\t" + string.Format (msg, args);
 330                         Console.WriteLine (s);
 331                         log_file.WriteLine (s);
 332                 }
 333
 334                 #region IDisposable Members
 335
 336                 public void Dispose()
 337                 {
 338                         log_file.Close ();
 339                 }
 340
 341                 #endregion
 342         }
 343
 344         class PositiveChecker: Checker
 345         {
 346                 readonly string files_folder;
 347                 readonly static object[] default_args = new object[1] { new string[] {} };
 348                 string doc_output;
 349
 350                 ProcessStartInfo pi;
 351                 readonly string mono;
 352
 353                 protected enum TestResult {
 354                         CompileError,
 355                         ExecError,
 356                         LoadError,
 357                         XmlError,
 358                         Success
 359                 }
 360
 361                 public PositiveChecker (ITester tester, string log_file, string issue_file):
 362                         base (tester, log_file, issue_file)
 363                 {
 364                         files_folder = Directory.GetCurrentDirectory ();
 365
 366                         pi = new ProcessStartInfo ();
 367                         pi.CreateNoWindow = true;
 368                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 369                         pi.RedirectStandardOutput = true;
 370                         pi.RedirectStandardError = true;
 371                         pi.UseShellExecute = false;
 372
 373                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 374                         if (mono != null) {
 375                                 pi.FileName = mono;
 376                         }
 377                 }
 378
 379                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 380                                                         out string[] dependencies) {
 381                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 382                                 return false;
 383
 384                         doc_output = null;
 385                         if (compiler_options == null)
 386                                 return true;
 387
 388                         foreach (string one_opt in compiler_options) {
 389                                 if (one_opt.StartsWith ("-doc:")) {
 390                                         doc_output = one_opt.Split (':')[1];
 391                                 }
 392                         }
 393                         return true;
 394                 }
 395
 396                 protected override bool Check(TestCase test)
 397                 {
 398                         string filename = test.FileName;
 399                         try {
 400                                 if (!base.Check (test)) {
 401                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 402                                         return false;
 403                                 }
 404                         }
 405                         catch (Exception e) {
 406                                 if (e.InnerException != null)
 407                                         e = e.InnerException;
 408
 409                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 410                                 return false;
 411                         }
 412
 413                         // Test setup
 414                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 415                                 if (verbose)
 416                                         LogLine ("OK");
 417                                 --total;
 418                                 return true;
 419                         }
 420
 421                         MethodInfo mi = null;
 422                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 423
 424                         // Enable .dll only tests (no execution required)
 425                         if (!File.Exists(file)) {
 426                                 HandleFailure (filename, TestResult.Success, null);
 427                                 return true;
 428                         }
 429
 430                         try {
 431                                 mi = Assembly.LoadFile (file).EntryPoint;
 432                         }
 433                         catch (FileNotFoundException) {
 434                                 if (File.Exists (file)) {
 435                                         Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 436                                 }
 437                         }
 438                         catch (Exception e) {
 439                                 HandleFailure (filename, TestResult.LoadError, e.ToString ());
 440                                 return false;
 441                         }
 442
 443                         if (!ExecuteFile (mi, file, filename))
 444                                 return false;
 445
 446                         if (doc_output != null) {
 447                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 448                                 try {
 449 #if !NET_2_1
 450                                         XmlComparer.Compare (ref_file, doc_output);
 451 #endif
 452                                 }
 453                                 catch (Exception e) {
 454                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 455                                         return false;
 456                                 }
 457                         }
 458
 459                         HandleFailure (filename, TestResult.Success, null);
 460                         return true;
 461                 }
 462
 463                 int ExecFile (string exe_name, string filename)
 464                 {
 465                         if (mono == null)
 466                                 pi.FileName = exe_name;
 467                         else
 468                                 pi.Arguments = exe_name;
 469
 470                         Process p = Process.Start (pi);
 471                         p.WaitForExit ();
 472                         return p.ExitCode;
 473                 }
 474
 475                 bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
 476                 {
 477                         TextWriter stdout = Console.Out;
 478                         TextWriter stderr = Console.Error;
 479                         Console.SetOut (TextWriter.Null);
 480                         Console.SetError (TextWriter.Null);
 481                         ParameterInfo[] pi = entry_point.GetParameters ();
 482                         object[] args = pi.Length == 0 ? null : default_args;
 483
 484                         object result = null;
 485                         try {
 486                                 try {
 487                                         result = entry_point.Invoke (null, args);
 488                                 } finally {
 489                                         Console.SetOut (stdout);
 490                                         Console.SetError (stderr);
 491                                 }
 492                         }
 493                         catch (Exception e) {
 494                                 int exit_code = ExecFile (exe_name, filename);
 495                                 if (exit_code == 0) {
 496                                         LogLine ("(appdomain method failed, external executable succeeded)");
 497                                         LogLine (e.ToString ());
 498                                         return true;
 499                                 }
 500                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 501                                 return false;
 502                         }
 503
 504                         if (result is int && (int)result != 0) {
 505                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 506                                 return false;
 507                         }
 508                         return true;
 509                 }
 510
 511                 void HandleFailure (string file, TestResult status, string extra)
 512                 {
 513                         switch (status) {
 514                                 case TestResult.Success:
 515                                         success++;
 516                                         if (know_issues.Contains (file)) {
 517                                                 LogFileLine (file, "FIXED ISSUE");
 518                                                 return;
 519                                         }
 520                                         if (verbose)
 521                                                 LogLine ("OK");
 522                                         return;
 523
 524                                 case TestResult.CompileError:
 525                                         if (know_issues.Contains (file)) {
 526                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 527                                                 know_issues.Remove (file);
 528                                                 return;
 529                                         }
 530                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 531                                         break;
 532
 533                                 case TestResult.ExecError:
 534                                         if (know_issues.Contains (file)) {
 535                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 536                                                 know_issues.Remove (file);
 537                                                 return;
 538                                         }
 539                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 540                                         break;
 541
 542                                 case TestResult.XmlError:
 543                                         if (know_issues.Contains (file)) {
 544                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
 545                                                 know_issues.Remove (file);
 546                                                 return;
 547                                         }
 548                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 549                                         break;
 550
 551                                 case TestResult.LoadError:
 552                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
 553                                         break;
 554                         }
 555
 556                         if (extra != null)
 557                                 LogLine ("{0}", extra);
 558
 559                         regression.Add (file);
 560                 }
 561         }
 562
 563         class NegativeChecker: Checker
 564         {
 565                 string expected_message;
 566                 string error_message;
 567                 bool check_msg;
 568                 bool check_error_line;
 569                 bool is_warning;
 570                 IDictionary wrong_warning;
 571
 572                 protected enum CompilerError {
 573                         Expected,
 574                         Wrong,
 575                         Missing,
 576                         WrongMessage,
 577                         MissingLocation,
 578                         Duplicate
 579                 }
 580
 581                 public NegativeChecker (ITester tester, string log_file, string issue_file, bool check_msg):
 582                         base (tester, log_file, issue_file)
 583                 {
 584                         this.check_msg = check_msg;
 585                         wrong_warning = new Hashtable ();
 586                 }
 587
 588                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
 589                                                         ref string[] compiler_options,
 590                                                         ref string[] dependencies)
 591                 {
 592                         if (row == 1) {
 593                                 expected_message = null;
 594
 595                                 int index = line.IndexOf (':');
 596                                 if (index == -1 || index > 15) {
 597                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
 598                                         ++syntax_errors;
 599                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
 600                                                               ref dependencies);
 601                                         return false;
 602                                 }
 603
 604                                 expected_message = line.Substring (index + 1).Trim ();
 605                         }
 606
 607                         if (row == 2) {
 608                                 string filtered = line.Replace(" ", "");
 609
 610                                 // Some error tests require to have different error text for different runtimes.
 611                                 if (filtered.StartsWith ("//GMCS")) {
 612                                         row = 1;
 613 #if !NET_2_0
 614                                         return true;
 615 #else
 616                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
 617 #endif
 618                                 }
 619
 620                                 check_error_line = !filtered.StartsWith ("//Line:0");
 621
 622                                 if (!filtered.StartsWith ("//Line:")) {
 623                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
 624                                         ++syntax_errors;
 625                                         return false;
 626                                 }
 627                         }
 628
 629                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
 630                                 return false;
 631
 632                         is_warning = false;
 633                         if (compiler_options != null) {
 634                                 foreach (string s in compiler_options) {
 635                                         if (s.EndsWith ("warnaserror"))
 636                                                 is_warning = true;
 637                                 }
 638                         }
 639                         return true;
 640                 }
 641
 642
 643                 protected override bool Check (TestCase test)
 644                 {
 645                         string filename = test.FileName;
 646
 647                         int start_char = 0;
 648                         while (Char.IsLetter (filename, start_char))
 649                                 ++start_char;
 650
 651                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
 652                         string expected = filename.Substring (start_char, end_char - start_char);
 653
 654                         try {
 655                                 if (base.Check (test)) {
 656                                         HandleFailure (filename, CompilerError.Missing);
 657                                         return false;
 658                                 }
 659                         }
 660                         catch (Exception e) {
 661                                 HandleFailure (filename, CompilerError.Missing);
 662                                 if (e.InnerException != null)
 663                                         e = e.InnerException;
 664
 665                                 Log (e.ToString ());
 666                                 return false;
 667                         }
 668
 669                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
 670                         if (tester.IsWarning (err_id)) {
 671                                 if (!is_warning)
 672                                         wrong_warning [err_id] = true;
 673                         } else {
 674                                 if (is_warning)
 675                                         wrong_warning [err_id] = false;
 676                         }
 677
 678                         CompilerError result_code = GetCompilerError (expected, tester.Output);
 679                         if (HandleFailure (filename, result_code)) {
 680                                 success++;
 681                                 return true;
 682                         }
 683
 684                         if (result_code == CompilerError.Wrong)
 685                                 LogLine (tester.Output);
 686
 687                         return false;
 688                 }
 689
 690                 CompilerError GetCompilerError (string expected, string buffer)
 691                 {
 692                         const string error_prefix = "CS";
 693                         const string ignored_error = "error CS5001";
 694                         string tested_text = "error " + error_prefix + expected;
 695                         StringReader sr = new StringReader (buffer);
 696                         string line = sr.ReadLine ();
 697                         ArrayList ld = new ArrayList ();
 698                         CompilerError result = CompilerError.Missing;
 699                         while (line != null) {
 700                                 if (ld.Contains (line)) {
 701                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
 702                                                 return CompilerError.Duplicate;
 703                                 }
 704                                 ld.Add (line);
 705
 706                                 if (result != CompilerError.Expected) {
 707                                         if (line.IndexOf (tested_text) != -1) {
 708                                                 if (check_msg) {
 709                                                         int first = line.IndexOf (':');
 710                                                         int second = line.IndexOf (':', first + 1);
 711                                                         if (second == -1 || !check_error_line)
 712                                                                 second = first;
 713
 714                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
 715                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
 716                                                                 error_message = msg;
 717                                                                 return CompilerError.WrongMessage;
 718                                                         }
 719
 720                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
 721                                                                 return CompilerError.MissingLocation;
 722                                                 }
 723                                                 result = CompilerError.Expected;
 724                                         } else if (line.IndexOf (error_prefix) != -1 &&
 725                                                 line.IndexOf (ignored_error) == -1)
 726                                                 result = CompilerError.Wrong;
 727                                 }
 728
 729                                 line = sr.ReadLine ();
 730                         }
 731
 732                         return result;
 733                 }
 734
 735                 bool HandleFailure (string file, CompilerError status)
 736                 {
 737                         switch (status) {
 738                                 case CompilerError.Expected:
 739                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
 740                                                 LogFileLine (file, "FIXED ISSUE");
 741                                                 return true;
 742                                         }
 743
 744                                         if (verbose)
 745                                                 LogLine ("OK");
 746                                         return true;
 747
 748                                 case CompilerError.Wrong:
 749                                         if (know_issues.Contains (file)) {
 750                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
 751                                                 know_issues.Remove (file);
 752                                                 return false;
 753                                         }
 754                                         if (no_error_list.Contains (file)) {
 755                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
 756                                                 no_error_list.Remove (file);
 757                                         }
 758                                         else {
 759                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
 760                                         }
 761                                         break;
 762
 763                                 case CompilerError.WrongMessage:
 764                                         if (know_issues.Contains (file)) {
 765                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
 766                                                 know_issues.Remove (file);
 767                                                 return false;
 768                                         }
 769                                         if (no_error_list.Contains (file)) {
 770                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
 771                                                 no_error_list.Remove (file);
 772                                         }
 773                                         else {
 774                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
 775                                                 LogLine ("Exp: {0}", expected_message);
 776                                                 LogLine ("Was: {0}", error_message);
 777                                         }
 778                                         break;
 779
 780                                 case CompilerError.Missing:
 781                                         if (no_error_list.Contains (file)) {
 782                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
 783                                                 no_error_list.Remove (file);
 784                                                 return false;
 785                                         }
 786
 787                                         if (know_issues.Contains (file)) {
 788                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
 789                                                 know_issues.Remove (file);
 790                                         }
 791                                         else {
 792                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
 793                                         }
 794
 795                                         break;
 796
 797                                 case CompilerError.MissingLocation:
 798                                         if (know_issues.Contains (file)) {
 799                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
 800                                                 know_issues.Remove (file);
 801                                                 return false;
 802                                         }
 803                                         if (no_error_list.Contains (file)) {
 804                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
 805                                                 no_error_list.Remove (file);
 806                                         }
 807                                         else {
 808                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
 809                                         }
 810                                         break;
 811
 812                                 case CompilerError.Duplicate:
 813                                         // Will become an error soon
 814                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
 815                                         return true;
 816                         }
 817
 818                         regression.Add (file);
 819                         return false;
 820                 }
 821
 822                 public override void PrintSummary()
 823                 {
 824                         base.PrintSummary ();
 825
 826                         if (wrong_warning.Count > 0) {
 827                                 LogLine ("");
 828                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
 829                                 LogLine ("");
 830                                 foreach (DictionaryEntry de in wrong_warning)
 831                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
 832                         }
 833                 }
 834
 835         }
 836
 837         class Tester {
 838
 839                 static int Main(string[] args) {
 840                         if (args.Length != 5) {
 841                                 Console.Error.WriteLine ("Usage: TestRunner [negative|positive] test-pattern compiler know-issues log-file");
 842                                 return 1;
 843                         }
 844
 845                         string mode = args[0].ToLower ();
 846 #if NET_2_1
 847                         string test_pattern ="fubar";
 848 #elif NET_2_0
 849                         string test_pattern = args [1] == "0" ? "*cs*.cs" : "*test-*.cs"; //args [1];
 850 #else
 851                         string test_pattern = args [1] == "0" ? "cs*.cs" : "test-*.cs"; //args [1];
 852 #endif
 853                         string mcs = args [2];
 854                         string issue_file = args [3];
 855                         string log_fname = args [4];
 856
 857                         string[] files = Directory.GetFiles (".", test_pattern);
 858
 859                         ITester tester;
 860                         try {
 861                                 Console.WriteLine ("Testing: " + mcs);
 862                                 tester = new ReflectionTester (Assembly.LoadFile (mcs));
 863                         }
 864                         catch (Exception) {
 865                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
 866                                 if (!File.Exists (mcs)) {
 867                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
 868                                         return 1;
 869                                 }
 870                                 tester = new ProcessTester (mcs);
 871                         }
 872
 873                         Checker checker;
 874                         switch (mode) {
 875                                 case "negative":
 876                                         checker = new NegativeChecker (tester, log_fname, issue_file, true);
 877                                         break;
 878                                 case "positive":
 879                                         checker = new PositiveChecker (tester, log_fname, issue_file);
 880                                         break;
 881                                 default:
 882                                         Console.Error.WriteLine ("You must specify testing mode (positive or negative)");
 883                                         return 1;
 884                         }
 885
 886                         foreach (string s in files) {
 887                                 string filename = Path.GetFileName (s);
 888                                 if (Char.IsUpper (filename, 0)) { // Windows hack
 889                                         continue;
 890                                 }
 891
 892                                 if (filename.EndsWith ("-p2.cs"))
 893                                         continue;
 894
 895                                 checker.Do (filename);
 896                         }
 897
 898                         checker.PrintSummary ();
 899
 900                         checker.Dispose ();
 901
 902                         return checker.ResultCode;
 903                 }
 904         }
 905 }