mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7
   8 //
   9 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
  10 //
  11 // Permission is hereby granted, free of charge, to any person obtaining
  12 // a copy of this software and associated documentation files (the
  13 // "Software"), to deal in the Software without restriction, including
  14 // without limitation the rights to use, copy, modify, merge, publish,
  15 // distribute, sublicense, and/or sell copies of the Software, and to
  16 // permit persons to whom the Software is furnished to do so, subject to
  17 // the following conditions:
  18 //
  19 // The above copyright notice and this permission notice shall be
  20 // included in all copies or substantial portions of the Software.
  21 //
  22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29 //
  30
  31 using System;
  32 using System.IO;
  33 using System.Diagnostics;
  34 using System.Reflection;
  35 using System.Text;
  36 using System.Collections;
  37 using System.Xml;
  38
  39 namespace TestRunner {
  40
  41         interface ITester
  42         {
  43                 string Output { get; }
  44                 bool Invoke (string[] args);
  45                 bool IsWarning (int warningNumber);
  46         }
  47
  48         class ReflectionTester: ITester {
  49                 MethodInfo ep;
  50                 object[] method_arg;
  51                 StringWriter output;
  52                 int[] all_warnings;
  53
  54                 public ReflectionTester (Assembly a)
  55                 {
  56                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  57
  58                         if (t == null)
  59                                 Console.Error.WriteLine ("null, huh?");
  60
  61                         ep = t.GetMethod ("InvokeCompiler",
  62                                 BindingFlags.Static | BindingFlags.Public);
  63                         if (ep == null)
  64                                 throw new MissingMethodException ("static InvokeCompiler");
  65                         method_arg = new object [2];
  66
  67                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  68                         all_warnings = (int[])pi.GetValue (null, null);
  69                         Array.Sort (all_warnings);
  70                 }
  71
  72                 public string Output {
  73                         get {
  74                                 return output.GetStringBuilder ().ToString ();
  75                         }
  76                 }
  77
  78                 public bool Invoke(string[] args)
  79                 {
  80                         output = new StringWriter ();
  81                         method_arg [0] = args;
  82                         method_arg [1] = output;
  83                         return (bool)ep.Invoke (null, method_arg);
  84                 }
  85
  86                 public bool IsWarning (int warningNumber)
  87                 {
  88                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  89                 }
  90         }
  91
  92 #if !NET_2_1
  93         class ProcessTester: ITester
  94         {
  95                 ProcessStartInfo pi;
  96                 string output;
  97
  98                 public ProcessTester (string p_path)
  99                 {
 100                         pi = new ProcessStartInfo ();
 101                         pi.FileName = p_path;
 102                         pi.CreateNoWindow = true;
 103                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 104                         pi.RedirectStandardOutput = true;
 105                         pi.RedirectStandardError = true;
 106                         pi.UseShellExecute = false;
 107                 }
 108
 109                 public string Output {
 110                         get {
 111                                 return output;
 112                         }
 113                 }
 114
 115                 public bool Invoke(string[] args)
 116                 {
 117                         StringBuilder sb = new StringBuilder ("/nologo ");
 118                         foreach (string s in args) {
 119                                 sb.Append (s);
 120                                 sb.Append (" ");
 121                         }
 122                         pi.Arguments = sb.ToString ();
 123                         Process p = Process.Start (pi);
 124                         output = p.StandardError.ReadToEnd ();
 125                         if (output.Length == 0)
 126                             output = p.StandardOutput.ReadToEnd ();
 127                         p.WaitForExit ();
 128                         return p.ExitCode == 0;
 129                 }
 130
 131                 public bool IsWarning (int warningNumber)
 132                 {
 133                         throw new NotImplementedException ();
 134                 }
 135         }
 136 #endif
 137
 138         class TestCase
 139         {
 140                 public readonly string FileName;
 141                 public readonly string[] CompilerOptions;
 142                 public readonly string[] Dependencies;
 143
 144                 public TestCase (string filename, string[] options, string[] deps)
 145                 {
 146                         this.FileName = filename;
 147                         this.CompilerOptions = options;
 148                         this.Dependencies = deps;
 149                 }
 150         }
 151
 152         class PositiveTestCase : TestCase
 153         {
 154                 public class VerificationData
 155                 {
 156                         public class MethodData
 157                         {
 158                                 public MethodData (MethodBase mi, int il_size)
 159                                 {
 160                                         this.Type = mi.DeclaringType.ToString ();
 161                                         this.MethodName = mi.ToString ();
 162                                         this.ILSize = il_size;
 163                                 }
 164
 165                                 public MethodData (string type_name, string method_name, int il_size)
 166                                 {
 167                                         this.Type = type_name;
 168                                         this.MethodName = method_name;
 169                                         this.ILSize = il_size;
 170                                 }
 171
 172                                 public string Type;
 173                                 public string MethodName;
 174                                 public int ILSize;
 175                                 public bool Checked;
 176                         }
 177
 178                         ArrayList methods;
 179                         public bool IsNewSet;
 180
 181                         public VerificationData (string test_file)
 182                         {
 183 #if NET_2_0
 184                                 this.test_file = test_file;
 185 #endif
 186                         }
 187
 188 #if NET_2_0
 189                         string test_file;
 190
 191                         public static VerificationData FromFile (string name, XmlReader r)
 192                         {
 193                                 VerificationData tc = new VerificationData (name);
 194                                 ArrayList methods = new ArrayList ();
 195                                 r.Read ();
 196                                 while (r.ReadToNextSibling ("type")) {
 197                                         string type_name = r ["name"];
 198                                         r.Read ();
 199                                         while (r.ReadToNextSibling ("method")) {
 200                                                 string m_name = r ["name"];
 201
 202                                                 r.ReadToDescendant ("size");
 203                                                 int il_size = r.ReadElementContentAsInt ();
 204                                                 methods.Add (new MethodData (type_name, m_name, il_size));
 205                                                 r.Read ();
 206                                         }
 207                                         r.Read ();
 208                                 }
 209
 210                                 tc.methods = methods;
 211                                 return tc;
 212                         }
 213
 214                         public void WriteCodeInfoTo (XmlWriter w)
 215                         {
 216                                 w.WriteStartElement ("test");
 217                                 w.WriteAttributeString ("name", test_file);
 218
 219                                 string type = null;
 220                                 foreach (MethodData data in methods) {
 221                                         if (!data.Checked)
 222                                                 continue;
 223
 224                                         if (type != data.Type) {
 225                                                 if (type != null)
 226                                                         w.WriteEndElement ();
 227
 228                                                 type = data.Type;
 229                                                 w.WriteStartElement ("type");
 230                                                 w.WriteAttributeString ("name", type);
 231                                         }
 232
 233                                         w.WriteStartElement ("method");
 234                                         w.WriteAttributeString ("name", data.MethodName);
 235                                         w.WriteStartElement ("size");
 236                                         w.WriteValue (data.ILSize);
 237                                         w.WriteEndElement ();
 238                                         w.WriteEndElement ();
 239                                 }
 240
 241                                 if (type != null)
 242                                         w.WriteEndElement ();
 243
 244                                 w.WriteEndElement ();
 245                         }
 246 #endif
 247
 248                         public MethodData FindMethodData (string method_name, string declaring_type)
 249                         {
 250                                 if (methods == null)
 251                                         return null;
 252
 253                                 foreach (MethodData md in methods) {
 254                                         if (md.MethodName == method_name && md.Type == declaring_type)
 255                                                 return md;
 256                                 }
 257
 258                                 return null;
 259                         }
 260
 261                         public void AddNewMethod (MethodBase mb, int il_size)
 262                         {
 263                                 if (methods == null)
 264                                         methods = new ArrayList ();
 265
 266                                 MethodData md = new MethodData (mb, il_size);
 267                                 md.Checked = true;
 268                                 methods.Add (md);
 269                         }
 270                 }
 271
 272                 VerificationData verif_data;
 273
 274                 public PositiveTestCase (string filename, string [] options, string [] deps)
 275                         : base (filename, options, deps)
 276                 {
 277                 }
 278
 279                 public void CreateNewTest ()
 280                 {
 281                         verif_data = new VerificationData (FileName);
 282                         verif_data.IsNewSet = true;
 283                 }
 284
 285                 public VerificationData VerificationProvider {
 286                         set {
 287                                 verif_data = value;
 288                         }
 289                         get {
 290                                 return verif_data;
 291                         }
 292                 }
 293
 294                 public bool CompareIL (MethodBase mi, PositiveChecker checker)
 295                 {
 296                         string m_name = mi.ToString ();
 297                         string decl_type = mi.DeclaringType.ToString ();
 298                         VerificationData.MethodData md = verif_data.FindMethodData (m_name, decl_type);
 299                         if (md == null) {
 300                                 verif_data.AddNewMethod (mi, GetILSize (mi));
 301                                 if (!verif_data.IsNewSet) {
 302                                         checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 303                                         return false;
 304                                 }
 305
 306                                 return true;
 307                         }
 308
 309                         if (md.Checked) {
 310                                 checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 311                                 return false;
 312                         }
 313
 314                         md.Checked = true;
 315
 316                         int il_size = GetILSize (mi);
 317                         if (md.ILSize == il_size)
 318                                 return true;
 319
 320                         if (md.ILSize > il_size) {
 321                                 checker.LogFileLine (FileName, "{0} (code size reduction {1} -> {2})", m_name, md.ILSize, il_size);
 322                                 md.ILSize = il_size;
 323                                 return true;
 324                         }
 325
 326                         checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError,
 327                                 string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
 328
 329                         md.ILSize = il_size;
 330
 331                         return false;
 332                 }
 333
 334                 static int GetILSize (MethodBase mi)
 335                 {
 336 #if NET_2_0
 337                         MethodBody body = mi.GetMethodBody ();
 338                         if (body != null)
 339                                 return body.GetILAsByteArray ().Length;
 340 #endif
 341                         return 0;
 342                 }
 343         }
 344
 345         class Checker: IDisposable
 346         {
 347                 protected ITester tester;
 348                 protected int success;
 349                 protected int total;
 350                 protected int ignored;
 351                 protected int syntax_errors;
 352                 string issue_file;
 353                 StreamWriter log_file;
 354                 protected string[] extra_compiler_options;
 355                 // protected string[] compiler_options;
 356                 // protected string[] dependencies;
 357
 358                 protected ArrayList tests = new ArrayList ();
 359                 protected Hashtable test_hash = new Hashtable ();
 360                 protected ArrayList regression = new ArrayList ();
 361                 protected ArrayList know_issues = new ArrayList ();
 362                 protected ArrayList ignore_list = new ArrayList ();
 363                 protected ArrayList no_error_list = new ArrayList ();
 364
 365                 protected bool verbose;
 366
 367                 int total_known_issues;
 368
 369                 protected Checker (ITester tester)
 370                 {
 371                         this.tester = tester;
 372                 }
 373
 374                 public string IssueFile {
 375                         set {
 376                                 this.issue_file = value;
 377                                 ReadWrongErrors (issue_file);
 378                         }
 379                 }
 380
 381                 public string LogFile {
 382                         set {
 383                                 this.log_file = new StreamWriter (value, false);
 384                         }
 385                 }
 386
 387                 public bool Verbose {
 388                         set {
 389                                 verbose = value;
 390                         }
 391                 }
 392
 393                 public string[] ExtraCompilerOptions {
 394                         set {
 395                                 extra_compiler_options = value;
 396                         }
 397                 }
 398
 399                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 400                                                         out string[] dependencies)
 401                 {
 402                         int row = 0;
 403                         compiler_options = null;
 404                         dependencies = null;
 405                         try {
 406                                 using (StreamReader sr = new StreamReader (file)) {
 407                                         String line;
 408                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 409                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 410                                                                       ref dependencies))
 411                                                         return false;
 412                                         }
 413                                 }
 414                         } catch {
 415                                 return false;
 416                         }
 417                         return true;
 418                 }
 419
 420                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 421                                                         ref string[] compiler_options,
 422                                                         ref string[] dependencies)
 423                 {
 424                         const string options = "// Compiler options:";
 425                         const string depends = "// Dependencies:";
 426
 427                         if (row == 1) {
 428                                 compiler_options = null;
 429                                 dependencies = null;
 430                         }
 431
 432                         int index = line.IndexOf (options);
 433                         if (index != -1) {
 434                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 435                                 for (int i = 0; i < compiler_options.Length; i++)
 436                                         compiler_options[i] = compiler_options[i].TrimStart ();
 437                         }
 438                         index = line.IndexOf (depends);
 439                         if (index != -1) {
 440                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 441                                 for (int i = 0; i < dependencies.Length; i++)
 442                                         dependencies[i] = dependencies[i].TrimStart ();
 443                         }
 444
 445                         return true;
 446                 }
 447
 448                 public bool Do (string filename)
 449                 {
 450                         if (test_hash.Contains (filename))
 451                                 return true;
 452
 453                         if (ignore_list.Contains (filename)) {
 454                                 ++ignored;
 455                                 LogFileLine (filename, "NOT TESTED");
 456                                 return false;
 457                         }
 458
 459                         string[] compiler_options, dependencies;
 460                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 461                                 LogFileLine (filename, "ERROR");
 462                                 return false;
 463                         }
 464
 465                         if (extra_compiler_options != null) {
 466                                 if (compiler_options == null)
 467                                         compiler_options = extra_compiler_options;
 468                                 else {
 469                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 470                                         extra_compiler_options.CopyTo (new_options, 0);
 471                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 472                                         compiler_options = new_options;
 473                                 }
 474                         }
 475
 476                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 477                         test_hash.Add (filename, test);
 478
 479                         ++total;
 480                         if (dependencies != null) {
 481                                 foreach (string dependency in dependencies) {
 482                                         if (!Do (dependency)) {
 483                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 484                                                 return false;
 485                                         }
 486                                 }
 487                         }
 488
 489                         tests.Add (test);
 490
 491                         return Check (test);
 492                 }
 493
 494                 protected virtual bool Check (TestCase test)
 495                 {
 496                         string[] test_args;
 497
 498                         if (test.CompilerOptions != null) {
 499                                 test_args = new string [2 + test.CompilerOptions.Length];
 500                                 test.CompilerOptions.CopyTo (test_args, 0);
 501                         } else {
 502                                 test_args = new string [2];
 503                         }
 504                         test_args [test_args.Length - 2] = test.FileName;
 505                         test_args [test_args.Length - 1] = "-debug";
 506
 507                         return tester.Invoke (test_args);
 508                 }
 509
 510                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 511                 {
 512                         return new TestCase (filename, options, deps);
 513                 }
 514
 515                 void ReadWrongErrors (string file)
 516                 {
 517                         const string ignored = "IGNORE";
 518                         const string no_error = "NO ERROR";
 519
 520                         using (StreamReader sr = new StreamReader (file)) {
 521                                 string line;
 522                                 while ((line = sr.ReadLine()) != null) {
 523                                         if (line.StartsWith ("#"))
 524                                                 continue;
 525
 526                                         ArrayList active_cont = know_issues;
 527
 528                                         if (line.IndexOf (ignored) > 0)
 529                                                 active_cont = ignore_list;
 530                                         else if (line.IndexOf (no_error) > 0)
 531                                                 active_cont = no_error_list;
 532
 533                                         string file_name = line.Split (' ')[0];
 534                                         if (file_name.Length == 0)
 535                                                 continue;
 536
 537                                         active_cont.Add (file_name);
 538                                 }
 539                         }
 540                         total_known_issues = know_issues.Count;
 541                 }
 542
 543                 protected virtual void PrintSummary ()
 544                 {
 545                         LogLine ("Done" + Environment.NewLine);
 546                         float rate = 0;
 547                         if (total > 0)
 548                                 rate = (float) (success) / (float)total;
 549                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 550
 551                         if (syntax_errors > 0)
 552                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 553
 554                         if (ignored > 0)
 555                                 LogLine ("{0} test(s) ignored", ignored);
 556
 557                         if (total_known_issues - know_issues.Count > 0)
 558                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 559
 560                         know_issues.AddRange (no_error_list);
 561                         if (know_issues.Count > 0) {
 562                                 LogLine ("");
 563                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 564                                 foreach (string s in know_issues)
 565                                         LogLine (s);
 566                         }
 567                         if (regression.Count > 0) {
 568                                 LogLine ("");
 569                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 570                                 foreach (string s in regression)
 571                                         LogLine (s);
 572                         }
 573                 }
 574
 575                 public int ResultCode
 576                 {
 577                         get {
 578                                 return regression.Count == 0 ? 0 : 1;
 579                         }
 580                 }
 581
 582                 protected void Log (string msg, params object [] rest)
 583                 {
 584                         Console.Write (msg, rest);
 585                         if (log_file != null)
 586                                 log_file.Write (msg, rest);
 587                 }
 588
 589                 protected void LogLine (string msg, params object [] rest)
 590                 {
 591                         Console.WriteLine (msg, rest);
 592                         if (log_file != null)
 593                                 log_file.WriteLine (msg, rest);
 594                 }
 595
 596                 public void LogFileLine (string file, string msg, params object [] args)
 597                 {
 598                         string s = file + "...\t" + string.Format (msg, args);
 599                         Console.WriteLine (s);
 600                         if (log_file != null)
 601                                 log_file.WriteLine (s);
 602                 }
 603
 604                 #region IDisposable Members
 605
 606                 public void Dispose()
 607                 {
 608                         if (log_file != null)
 609                                 log_file.Close ();
 610                 }
 611
 612                 #endregion
 613
 614                 public virtual void Initialize ()
 615                 {
 616                 }
 617
 618                 public virtual void CleanUp ()
 619                 {
 620                         PrintSummary ();
 621                 }
 622         }
 623
 624         class PositiveChecker: Checker
 625         {
 626                 readonly string files_folder;
 627                 readonly static object[] default_args = new object[1] { new string[] {} };
 628                 string doc_output;
 629                 string verif_file;
 630                 bool update_verif_file;
 631                 Hashtable verif_data;
 632
 633 #if !NET_2_1
 634                 ProcessStartInfo pi;
 635 #endif
 636                 readonly string mono;
 637
 638                 public enum TestResult {
 639                         CompileError,
 640                         ExecError,
 641                         LoadError,
 642                         XmlError,
 643                         Success,
 644                         ILError
 645                 }
 646
 647                 public PositiveChecker (ITester tester, string verif_file):
 648                         base (tester)
 649                 {
 650                         files_folder = Directory.GetCurrentDirectory ();
 651                         this.verif_file = verif_file;
 652
 653 #if !NET_2_1
 654                         pi = new ProcessStartInfo ();
 655                         pi.CreateNoWindow = true;
 656                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 657                         pi.RedirectStandardOutput = true;
 658                         pi.RedirectStandardError = true;
 659                         pi.UseShellExecute = false;
 660
 661                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 662                         if (mono != null) {
 663                                 pi.FileName = mono;
 664                         }
 665 #endif
 666                 }
 667
 668                 public bool UpdateVerificationDataFile {
 669                         set {
 670                                 update_verif_file = value;
 671                         }
 672                 }
 673
 674                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 675                                                         out string[] dependencies) {
 676                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 677                                 return false;
 678
 679                         doc_output = null;
 680                         if (compiler_options == null)
 681                                 return true;
 682
 683                         foreach (string one_opt in compiler_options) {
 684                                 if (one_opt.StartsWith ("-doc:")) {
 685                                         doc_output = one_opt.Split (':', '/')[1];
 686                                 }
 687                         }
 688                         return true;
 689                 }
 690
 691                 protected override bool Check(TestCase test)
 692                 {
 693                         string filename = test.FileName;
 694                         try {
 695                                 if (!base.Check (test)) {
 696                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 697                                         return false;
 698                                 }
 699                         }
 700                         catch (Exception e) {
 701                                 if (e.InnerException != null)
 702                                         e = e.InnerException;
 703
 704                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 705                                 return false;
 706                         }
 707
 708                         // Test setup
 709                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 710                                 if (verbose)
 711                                         LogFileLine (filename, "OK");
 712                                 --total;
 713                                 return true;
 714                         }
 715
 716                         MethodInfo mi = null;
 717                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 718
 719                         // Enable .dll only tests (no execution required)
 720                         if (!File.Exists(file)) {
 721                                 HandleFailure (filename, TestResult.Success, null);
 722                                 return true;
 723                         }
 724
 725                         Assembly assembly = null;
 726                         try {
 727                                 assembly = Assembly.LoadFile (file);
 728                                 mi = assembly.EntryPoint;
 729                         }
 730                         catch (FileNotFoundException) {
 731                                 if (File.Exists (file)) {
 732                                         Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 733                                 }
 734                         }
 735                         catch (Exception e) {
 736                                 HandleFailure (filename, TestResult.LoadError, e.ToString ());
 737                                 return false;
 738                         }
 739
 740                         if (!ExecuteFile (mi, file, filename))
 741                                 return false;
 742
 743                         if (doc_output != null) {
 744                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 745                                 try {
 746 #if !NET_2_1
 747                                         XmlComparer.Compare (ref_file, doc_output);
 748 #endif
 749                                 }
 750                                 catch (Exception e) {
 751                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 752                                         return false;
 753                                 }
 754                         } else {
 755                                 if (verif_file != null) {
 756                                         PositiveTestCase pt = (PositiveTestCase) test;
 757                                         pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data [filename];
 758                                         if (!CheckILSize (assembly, pt))
 759                                                 return false;
 760                                 }
 761                         }
 762
 763                         HandleFailure (filename, TestResult.Success, null);
 764                         return true;
 765                 }
 766
 767                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
 768                 {
 769                         return new PositiveTestCase (filename, options, deps);
 770                 }
 771
 772                 bool CheckILSize (Assembly assembly, PositiveTestCase test)
 773                 {
 774                         bool success = true;
 775                         Type[] types = assembly.GetTypes ();
 776                         foreach (Type t in types) {
 777                                 if (!t.IsClass && t.IsValueType)
 778                                         continue;
 779
 780                                 if (test.VerificationProvider == null) {
 781                                         if (!update_verif_file)
 782                                                 LogFileLine (test.FileName, "Missing IL verification data");
 783                                         test.CreateNewTest ();
 784                                 }
 785
 786                                 foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 787                                         MethodBase mi = m as MethodBase;
 788                                         if (mi == null)
 789                                                 continue;
 790
 791                                         if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 792                                                 continue;
 793
 794                                         success &= test.CompareIL (mi, this);
 795                                 }
 796                         }
 797
 798                         return success;
 799                 }
 800
 801                 bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
 802                 {
 803                         TextWriter stdout = Console.Out;
 804                         TextWriter stderr = Console.Error;
 805                         Console.SetOut (TextWriter.Null);
 806                         Console.SetError (TextWriter.Null);
 807                         ParameterInfo[] pi = entry_point.GetParameters ();
 808                         object[] args = pi.Length == 0 ? null : default_args;
 809
 810                         object result = null;
 811                         try {
 812                                 try {
 813                                         result = entry_point.Invoke (null, args);
 814                                 } finally {
 815                                         Console.SetOut (stdout);
 816                                         Console.SetError (stderr);
 817                                 }
 818                         }
 819                         catch (Exception e) {
 820                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 821                                 return false;
 822                         }
 823
 824                         if (result is int && (int)result != 0) {
 825                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 826                                 return false;
 827                         }
 828                         return true;
 829                 }
 830
 831                 public void HandleFailure (string file, TestResult status, string extra)
 832                 {
 833                         switch (status) {
 834                                 case TestResult.Success:
 835                                         success++;
 836                                         if (know_issues.Contains (file)) {
 837                                                 LogFileLine (file, "FIXED ISSUE");
 838                                                 return;
 839                                         }
 840                                         if (verbose)
 841                                                 LogFileLine (file, "OK");
 842                                         return;
 843
 844                                 case TestResult.CompileError:
 845                                         if (know_issues.Contains (file)) {
 846                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 847                                                 know_issues.Remove (file);
 848                                                 return;
 849                                         }
 850                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 851                                         break;
 852
 853                                 case TestResult.ExecError:
 854                                         if (know_issues.Contains (file)) {
 855                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 856                                                 know_issues.Remove (file);
 857                                                 return;
 858                                         }
 859                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 860                                         break;
 861
 862                                 case TestResult.XmlError:
 863                                         if (know_issues.Contains (file)) {
 864                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
 865                                                 know_issues.Remove (file);
 866                                                 return;
 867                                         }
 868                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 869                                         break;
 870
 871                                 case TestResult.LoadError:
 872                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
 873                                         break;
 874
 875                                 case TestResult.ILError:
 876                                         LogFileLine (file, "IL REGRESSION: " + extra);
 877                                         extra = null;
 878                                         break;
 879                         }
 880
 881                         if (extra != null)
 882                                 LogLine ("{0}", extra);
 883
 884                         if (!regression.Contains (file))
 885                                 regression.Add (file);
 886                 }
 887
 888                 public override void Initialize ()
 889                 {
 890                         if (verif_file != null) {
 891 #if NET_2_0
 892                                 LoadVerificationData (verif_file);
 893 #else
 894                                 throw new NotSupportedException ();
 895 #endif
 896                         }
 897
 898                         base.Initialize ();
 899                 }
 900
 901                 public override void CleanUp ()
 902                 {
 903                         base.CleanUp ();
 904
 905                         if (update_verif_file) {
 906 #if NET_2_0
 907                                 UpdateVerificationData (verif_file);
 908 #else
 909                                 throw new NotSupportedException ();
 910 #endif
 911                         }
 912                 }
 913
 914 #if NET_2_0
 915                 void LoadVerificationData (string file)
 916                 {
 917                         LogLine ("Loading verification data from `{0}' ...", file);
 918
 919                         using (XmlReader r = XmlReader.Create (file)) {
 920                                 r.ReadStartElement ("tests");
 921                                 verif_data = new Hashtable ();
 922
 923                                 while (r.Read ()) {
 924                                         if (r.Name != "test")
 925                                                 continue;
 926
 927                                         string name = r.GetAttribute ("name");
 928                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
 929                                         verif_data.Add (name, tc);
 930                                 }
 931                         }
 932                 }
 933
 934                 void UpdateVerificationData (string file)
 935                 {
 936                         LogLine ("Updating verification data `{0}' ...", file);
 937
 938                         XmlWriterSettings s = new XmlWriterSettings ();
 939                         s.Indent = true;
 940                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
 941                                 w.WriteStartDocument ();
 942                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
 943                                 w.WriteStartElement ("tests");
 944                                 foreach (PositiveTestCase tc in tests) {
 945                                         if (tc.VerificationProvider != null)
 946                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
 947                                 }
 948                                 w.WriteEndElement ();
 949                         }
 950                 }
 951 #endif
 952         }
 953
 954         class NegativeChecker: Checker
 955         {
 956                 string expected_message;
 957                 string error_message;
 958                 bool check_msg;
 959                 bool check_error_line;
 960                 bool is_warning;
 961                 IDictionary wrong_warning;
 962
 963                 protected enum CompilerError {
 964                         Expected,
 965                         Wrong,
 966                         Missing,
 967                         WrongMessage,
 968                         MissingLocation,
 969                         Duplicate
 970                 }
 971
 972                 public NegativeChecker (ITester tester, bool check_msg):
 973                         base (tester)
 974                 {
 975                         this.check_msg = check_msg;
 976                         wrong_warning = new Hashtable ();
 977                 }
 978
 979                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
 980                                                         ref string[] compiler_options,
 981                                                         ref string[] dependencies)
 982                 {
 983                         if (row == 1) {
 984                                 expected_message = null;
 985
 986                                 int index = line.IndexOf (':');
 987                                 if (index == -1 || index > 15) {
 988                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
 989                                         ++syntax_errors;
 990                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
 991                                                               ref dependencies);
 992                                         return false;
 993                                 }
 994
 995                                 expected_message = line.Substring (index + 1).Trim ();
 996                         }
 997
 998                         if (row == 2) {
 999                                 string filtered = line.Replace(" ", "");
1000
1001                                 // Some error tests require to have different error text for different runtimes.
1002                                 if (filtered.StartsWith ("//GMCS")) {
1003                                         row = 1;
1004 #if !NET_2_0
1005                                         return true;
1006 #else
1007                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1008 #endif
1009                                 }
1010
1011                                 check_error_line = !filtered.StartsWith ("//Line:0");
1012
1013                                 if (!filtered.StartsWith ("//Line:")) {
1014                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1015                                         ++syntax_errors;
1016                                         return false;
1017                                 }
1018                         }
1019
1020                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1021                                 return false;
1022
1023                         is_warning = false;
1024                         if (compiler_options != null) {
1025                                 foreach (string s in compiler_options) {
1026                                         if (s.EndsWith ("warnaserror"))
1027                                                 is_warning = true;
1028                                 }
1029                         }
1030                         return true;
1031                 }
1032
1033
1034                 protected override bool Check (TestCase test)
1035                 {
1036                         string filename = test.FileName;
1037
1038                         int start_char = 0;
1039                         while (Char.IsLetter (filename, start_char))
1040                                 ++start_char;
1041
1042                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1043                         string expected = filename.Substring (start_char, end_char - start_char);
1044
1045                         try {
1046                                 if (base.Check (test)) {
1047                                         HandleFailure (filename, CompilerError.Missing);
1048                                         return false;
1049                                 }
1050                         }
1051                         catch (Exception e) {
1052                                 HandleFailure (filename, CompilerError.Missing);
1053                                 if (e.InnerException != null)
1054                                         e = e.InnerException;
1055
1056                                 Log (e.ToString ());
1057                                 return false;
1058                         }
1059
1060                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1061                         if (tester.IsWarning (err_id)) {
1062                                 if (!is_warning)
1063                                         wrong_warning [err_id] = true;
1064                         } else {
1065                                 if (is_warning)
1066                                         wrong_warning [err_id] = false;
1067                         }
1068
1069                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1070                         if (HandleFailure (filename, result_code)) {
1071                                 success++;
1072                                 return true;
1073                         }
1074
1075                         if (result_code == CompilerError.Wrong)
1076                                 LogLine (tester.Output);
1077
1078                         return false;
1079                 }
1080
1081                 CompilerError GetCompilerError (string expected, string buffer)
1082                 {
1083                         const string error_prefix = "CS";
1084                         const string ignored_error = "error CS5001";
1085                         string tested_text = "error " + error_prefix + expected;
1086                         StringReader sr = new StringReader (buffer);
1087                         string line = sr.ReadLine ();
1088                         ArrayList ld = new ArrayList ();
1089                         CompilerError result = CompilerError.Missing;
1090                         while (line != null) {
1091                                 if (ld.Contains (line)) {
1092                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1093                                                 return CompilerError.Duplicate;
1094                                 }
1095                                 ld.Add (line);
1096
1097                                 if (result != CompilerError.Expected) {
1098                                         if (line.IndexOf (tested_text) != -1) {
1099                                                 if (check_msg) {
1100                                                         int first = line.IndexOf (':');
1101                                                         int second = line.IndexOf (':', first + 1);
1102                                                         if (second == -1 || !check_error_line)
1103                                                                 second = first;
1104
1105                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1106                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1107                                                                 error_message = msg;
1108                                                                 return CompilerError.WrongMessage;
1109                                                         }
1110
1111                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1112                                                                 return CompilerError.MissingLocation;
1113                                                 }
1114                                                 result = CompilerError.Expected;
1115                                         } else if (line.IndexOf (error_prefix) != -1 &&
1116                                                 line.IndexOf (ignored_error) == -1)
1117                                                 result = CompilerError.Wrong;
1118                                 }
1119
1120                                 line = sr.ReadLine ();
1121                         }
1122
1123                         return result;
1124                 }
1125
1126                 bool HandleFailure (string file, CompilerError status)
1127                 {
1128                         switch (status) {
1129                                 case CompilerError.Expected:
1130                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1131                                                 LogFileLine (file, "FIXED ISSUE");
1132                                                 return true;
1133                                         }
1134
1135                                         if (verbose)
1136                                                 LogFileLine (file, "OK");
1137                                         return true;
1138
1139                                 case CompilerError.Wrong:
1140                                         if (know_issues.Contains (file)) {
1141                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1142                                                 know_issues.Remove (file);
1143                                                 return false;
1144                                         }
1145                                         if (no_error_list.Contains (file)) {
1146                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1147                                                 no_error_list.Remove (file);
1148                                         }
1149                                         else {
1150                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1151                                         }
1152                                         break;
1153
1154                                 case CompilerError.WrongMessage:
1155                                         if (know_issues.Contains (file)) {
1156                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1157                                                 know_issues.Remove (file);
1158                                                 return false;
1159                                         }
1160                                         if (no_error_list.Contains (file)) {
1161                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1162                                                 no_error_list.Remove (file);
1163                                         }
1164                                         else {
1165                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1166                                                 LogLine ("Exp: {0}", expected_message);
1167                                                 LogLine ("Was: {0}", error_message);
1168                                         }
1169                                         break;
1170
1171                                 case CompilerError.Missing:
1172                                         if (no_error_list.Contains (file)) {
1173                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1174                                                 no_error_list.Remove (file);
1175                                                 return false;
1176                                         }
1177
1178                                         if (know_issues.Contains (file)) {
1179                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1180                                                 know_issues.Remove (file);
1181                                         }
1182                                         else {
1183                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1184                                         }
1185
1186                                         break;
1187
1188                                 case CompilerError.MissingLocation:
1189                                         if (know_issues.Contains (file)) {
1190                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1191                                                 know_issues.Remove (file);
1192                                                 return false;
1193                                         }
1194                                         if (no_error_list.Contains (file)) {
1195                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1196                                                 no_error_list.Remove (file);
1197                                         }
1198                                         else {
1199                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1200                                         }
1201                                         break;
1202
1203                                 case CompilerError.Duplicate:
1204                                         // Will become an error soon
1205                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1206                                         return true;
1207                         }
1208
1209                         regression.Add (file);
1210                         return false;
1211                 }
1212
1213                 protected override void PrintSummary()
1214                 {
1215                         base.PrintSummary ();
1216
1217                         if (wrong_warning.Count > 0) {
1218                                 LogLine ("");
1219                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1220                                 LogLine ("");
1221                                 foreach (DictionaryEntry de in wrong_warning)
1222                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1223                         }
1224                 }
1225
1226         }
1227
1228         class Tester {
1229
1230                 static int Main(string[] args)
1231                 {
1232                         string temp;
1233
1234                         if (GetOption ("help", args, false, out temp)) {
1235                                 Usage ();
1236                                 return 0;
1237                         }
1238
1239                         string compiler;
1240                         if (!GetOption ("compiler", args, true, out compiler)) {
1241                                 Usage ();
1242                                 return 1;
1243                         }
1244
1245                         ITester tester;
1246                         try {
1247                                 Console.WriteLine ("Loading " + compiler + " ...");
1248                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1249                         }
1250                         catch (Exception) {
1251 #if NET_2_1
1252                                 throw;
1253 #else
1254                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1255                                 if (!File.Exists (compiler)) {
1256                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1257                                         return 1;
1258                                 }
1259                                 tester = new ProcessTester (compiler);
1260 #endif
1261                         }
1262
1263                         string mode;
1264                         if (!GetOption ("mode", args, true, out mode)) {
1265                                 Usage ();
1266                                 return 1;
1267                         }
1268
1269                         Checker checker;
1270                         switch (mode) {
1271                                 case "neg":
1272                                         checker = new NegativeChecker (tester, true);
1273                                         break;
1274                                 case "pos":
1275                                         string iltest;
1276                                         GetOption ("il", args, false, out iltest);
1277                                         checker = new PositiveChecker (tester, iltest);
1278
1279                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1280                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1281                                         }
1282
1283                                         break;
1284                                 default:
1285                                         Console.Error.WriteLine ("Invalid -mode argument");
1286                                         return 1;
1287                         }
1288
1289
1290                         if (GetOption ("issues", args, true, out temp))
1291                                 checker.IssueFile = temp;
1292                         if (GetOption ("log", args, true, out temp))
1293                                 checker.LogFile = temp;
1294                         if (GetOption ("verbose", args, false, out temp))
1295                                 checker.Verbose = true;
1296                         if (GetOption ("compiler-options", args, true, out temp)) {
1297                                 string[] extra = temp.Split (' ');
1298                                 checker.ExtraCompilerOptions = extra;
1299                         }
1300
1301                         string test_pattern;
1302                         if (!GetOption ("files", args, true, out test_pattern)) {
1303                                 Usage ();
1304                                 return 1;
1305                         }
1306
1307                         string [] files = Directory.GetFiles (".", test_pattern);
1308                         if (files.Length == 0) {
1309                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1310                                 return 2;
1311                         }
1312
1313                         checker.Initialize ();
1314
1315                         foreach (string s in files) {
1316                                 string filename = Path.GetFileName (s);
1317                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1318                                         continue;
1319                                 }
1320
1321                                 if (filename.EndsWith ("-p2.cs"))
1322                                         continue;
1323
1324                                 checker.Do (filename);
1325                         }
1326
1327                         checker.CleanUp ();
1328
1329                         checker.Dispose ();
1330
1331                         return checker.ResultCode;
1332                 }
1333
1334                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1335                 {
1336                         opt = "-" + opt;
1337                         foreach (string a in args) {
1338                                 if (a.StartsWith (opt)) {
1339                                         int sep = a.IndexOf (':');
1340                                         if (sep > 0) {
1341                                                 value = a.Substring (sep + 1);
1342                                         } else {
1343                                                 value = null;
1344                                                 if (req_arg) {
1345                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1346                                                         return false;
1347                                                 }
1348                                         }
1349
1350                                         return true;
1351                                 }
1352                         }
1353
1354                         value = null;
1355                         return false;
1356                 }
1357
1358                 static void Usage ()
1359                 {
1360                         Console.WriteLine (
1361                                 "Mono compiler tester, (C) 2008 Novell, Inc.\n" +
1362                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1363                                 "   \n" +
1364                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1365                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1366                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1367                                 "   -issues:FILE     The list of expected failures\n" +
1368                                 "   -log:FILE        Writes any output also to the file\n" +
1369                                 "   -help            Lists all options\n" +
1370                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1371                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1372                                 "   -verbose         Prints more details during testing\n"
1373                                 );
1374                 }
1375         }
1376 }