mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
   8 // Copyright (C) 2012 Xamarin Inc (http://www.xamarin.com)
   9 //
  10 // Permission is hereby granted, free of charge, to any person obtaining
  11 // a copy of this software and associated documentation files (the
  12 // "Software"), to deal in the Software without restriction, including
  13 // without limitation the rights to use, copy, modify, merge, publish,
  14 // distribute, sublicense, and/or sell copies of the Software, and to
  15 // permit persons to whom the Software is furnished to do so, subject to
  16 // the following conditions:
  17 //
  18 // The above copyright notice and this permission notice shall be
  19 // included in all copies or substantial portions of the Software.
  20 //
  21 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  22 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  23 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  24 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  25 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  26 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  27 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  28 //
  29
  30 using System;
  31 using System.IO;
  32 using System.Diagnostics;
  33 using System.Reflection;
  34 using System.Text;
  35 using System.Collections;
  36 using System.Xml;
  37 using System.Collections.Generic;
  38 using Mono.CompilerServices.SymbolWriter;
  39 using System.Globalization;
  40
  41 namespace TestRunner {
  42
  43         interface ITester
  44         {
  45                 string Output { get; }
  46                 bool Invoke (string[] args);
  47                 bool IsWarning (int warningNumber);
  48         }
  49
  50         class ReflectionTester: ITester {
  51                 MethodInfo ep;
  52                 object[] method_arg;
  53                 StringWriter output;
  54                 int[] all_warnings;
  55
  56                 public ReflectionTester (Assembly a)
  57                 {
  58                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  59
  60                         if (t == null)
  61                                 Console.Error.WriteLine ("null, huh?");
  62
  63                         ep = t.GetMethod ("InvokeCompiler",
  64                                 BindingFlags.Static | BindingFlags.Public);
  65                         if (ep == null)
  66                                 throw new MissingMethodException ("static InvokeCompiler");
  67                         method_arg = new object [2];
  68
  69                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  70                         all_warnings = (int[])pi.GetValue (null, null);
  71                         Array.Sort (all_warnings);
  72                 }
  73
  74                 public string Output {
  75                         get {
  76                                 return output.GetStringBuilder ().ToString ();
  77                         }
  78                 }
  79
  80                 public bool Invoke(string[] args)
  81                 {
  82                         output = new StringWriter ();
  83                         method_arg [0] = args;
  84                         method_arg [1] = output;
  85                         return (bool)ep.Invoke (null, method_arg);
  86                 }
  87
  88                 public bool IsWarning (int warningNumber)
  89                 {
  90                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  91                 }
  92         }
  93
  94 #if !NET_2_1
  95         class ProcessTester: ITester
  96         {
  97                 ProcessStartInfo pi;
  98                 string output;
  99
 100                 public ProcessTester (string p_path)
 101                 {
 102                         pi = new ProcessStartInfo ();
 103                         pi.FileName = p_path;
 104                         pi.CreateNoWindow = true;
 105                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 106                         pi.RedirectStandardOutput = true;
 107                         pi.RedirectStandardError = true;
 108                         pi.UseShellExecute = false;
 109                 }
 110
 111                 public string Output {
 112                         get {
 113                                 return output;
 114                         }
 115                 }
 116
 117                 public bool Invoke(string[] args)
 118                 {
 119                         StringBuilder sb = new StringBuilder ("/nologo ");
 120                         foreach (string s in args) {
 121                                 sb.Append (s);
 122                                 sb.Append (" ");
 123                         }
 124                         pi.Arguments = sb.ToString ();
 125                         Process p = Process.Start (pi);
 126                         output = p.StandardError.ReadToEnd ();
 127                         if (output.Length == 0)
 128                             output = p.StandardOutput.ReadToEnd ();
 129                         p.WaitForExit ();
 130                         return p.ExitCode == 0;
 131                 }
 132
 133                 public bool IsWarning (int warningNumber)
 134                 {
 135                         throw new NotImplementedException ();
 136                 }
 137         }
 138 #endif
 139
 140         class TestCase : MarshalByRefObject
 141         {
 142                 public readonly string FileName;
 143                 public readonly string[] CompilerOptions;
 144                 public readonly string[] Dependencies;
 145
 146                 public TestCase (string filename, string[] options, string[] deps)
 147                 {
 148                         this.FileName = filename;
 149                         this.CompilerOptions = options;
 150                         this.Dependencies = deps;
 151                 }
 152         }
 153
 154         class PositiveTestCase : TestCase
 155         {
 156                 public class VerificationData : MarshalByRefObject
 157                 {
 158                         public class MethodData : MarshalByRefObject
 159                         {
 160                                 public MethodData (MethodBase mi, int il_size)
 161                                 {
 162                                         this.Type = mi.DeclaringType.ToString ();
 163                                         this.MethodName = mi.ToString ();
 164                                         this.MethodAttributes = (int) mi.Attributes;
 165                                         this.ILSize = il_size;
 166                                 }
 167
 168                                 public MethodData (string type_name, string method_name, int method_attributes, int il_size)
 169                                 {
 170                                         this.Type = type_name;
 171                                         this.MethodName = method_name;
 172                                         this.MethodAttributes = method_attributes;
 173                                         this.ILSize = il_size;
 174                                 }
 175
 176                                 public string Type;
 177                                 public string MethodName;
 178                                 public int ILSize;
 179                                 public bool Checked;
 180                                 public int MethodAttributes;
 181                         }
 182
 183                         ArrayList methods;
 184                         public bool IsNewSet;
 185
 186                         public VerificationData (string test_file)
 187                         {
 188                                 this.test_file = test_file;
 189                         }
 190
 191                         string test_file;
 192
 193                         public static VerificationData FromFile (string name, XmlReader r)
 194                         {
 195                                 VerificationData tc = new VerificationData (name);
 196                                 ArrayList methods = new ArrayList ();
 197                                 r.Read ();
 198                                 while (r.ReadToNextSibling ("type")) {
 199                                         string type_name = r ["name"];
 200                                         r.Read ();
 201                                         while (r.ReadToNextSibling ("method")) {
 202                                                 string m_name = r ["name"];
 203                                                 int method_attrs = int.Parse (r["attrs"]);
 204
 205                                                 r.ReadToDescendant ("size");
 206                                                 int il_size = r.ReadElementContentAsInt ();
 207                                                 methods.Add (new MethodData (type_name, m_name, method_attrs, il_size));
 208                                                 r.Read ();
 209                                         }
 210                                         r.Read ();
 211                                 }
 212
 213                                 tc.methods = methods;
 214                                 return tc;
 215                         }
 216
 217                         public void WriteCodeInfoTo (XmlWriter w)
 218                         {
 219                                 w.WriteStartElement ("test");
 220                                 w.WriteAttributeString ("name", test_file);
 221
 222                                 string type = null;
 223                                 foreach (MethodData data in methods) {
 224                                         if (!data.Checked)
 225                                                 continue;
 226
 227                                         if (type != data.Type) {
 228                                                 if (type != null)
 229                                                         w.WriteEndElement ();
 230
 231                                                 type = data.Type;
 232                                                 w.WriteStartElement ("type");
 233                                                 w.WriteAttributeString ("name", type);
 234                                         }
 235
 236                                         w.WriteStartElement ("method");
 237                                         w.WriteAttributeString ("name", data.MethodName);
 238                                         int v = data.MethodAttributes;
 239                                         w.WriteAttributeString ("attrs", v.ToString ());
 240                                         w.WriteStartElement ("size");
 241                                         w.WriteValue (data.ILSize);
 242                                         w.WriteEndElement ();
 243                                         w.WriteEndElement ();
 244                                 }
 245
 246                                 if (type != null)
 247                                         w.WriteEndElement ();
 248
 249                                 w.WriteEndElement ();
 250                         }
 251
 252                         public MethodData FindMethodData (string method_name, string declaring_type)
 253                         {
 254                                 if (methods == null)
 255                                         return null;
 256
 257                                 foreach (MethodData md in methods) {
 258                                         if (md.MethodName == method_name && md.Type == declaring_type)
 259                                                 return md;
 260                                 }
 261
 262                                 return null;
 263                         }
 264
 265                         public void AddNewMethod (MethodBase mb, int il_size)
 266                         {
 267                                 if (methods == null)
 268                                         methods = new ArrayList ();
 269
 270                                 MethodData md = new MethodData (mb, il_size);
 271                                 md.Checked = true;
 272                                 methods.Add (md);
 273                         }
 274                 }
 275
 276                 VerificationData verif_data;
 277
 278                 public PositiveTestCase (string filename, string [] options, string [] deps)
 279                         : base (filename, options, deps)
 280                 {
 281                 }
 282
 283                 public void CreateNewTest ()
 284                 {
 285                         verif_data = new VerificationData (FileName);
 286                         verif_data.IsNewSet = true;
 287                 }
 288
 289                 public VerificationData VerificationProvider {
 290                         set {
 291                                 verif_data = value;
 292                         }
 293                         get {
 294                                 return verif_data;
 295                         }
 296                 }
 297         }
 298
 299         class Checker: MarshalByRefObject, IDisposable
 300         {
 301                 protected ITester tester;
 302                 protected int success;
 303                 protected int total;
 304                 protected int ignored;
 305                 protected int syntax_errors;
 306                 string issue_file;
 307                 StreamWriter log_file;
 308                 protected string[] extra_compiler_options;
 309                 // protected string[] compiler_options;
 310                 // protected string[] dependencies;
 311
 312                 protected ArrayList tests = new ArrayList ();
 313                 protected Hashtable test_hash = new Hashtable ();
 314                 protected ArrayList regression = new ArrayList ();
 315                 protected ArrayList know_issues = new ArrayList ();
 316                 protected ArrayList ignore_list = new ArrayList ();
 317                 protected ArrayList no_error_list = new ArrayList ();
 318
 319                 protected bool verbose;
 320                 protected bool safe_execution;
 321
 322                 int total_known_issues;
 323
 324                 protected Checker (ITester tester)
 325                 {
 326                         this.tester = tester;
 327                 }
 328
 329                 public string IssueFile {
 330                         set {
 331                                 this.issue_file = value;
 332                                 ReadWrongErrors (issue_file);
 333                         }
 334                 }
 335
 336                 public string LogFile {
 337                         set {
 338                                 this.log_file = new StreamWriter (value, false);
 339                         }
 340                 }
 341
 342                 public bool Verbose {
 343                         set {
 344                                 verbose = value;
 345                         }
 346                 }
 347
 348                 public bool SafeExecution {
 349                         set {
 350                                 safe_execution = value;
 351                         }
 352                 }
 353
 354                 public string[] ExtraCompilerOptions {
 355                         set {
 356                                 extra_compiler_options = value;
 357                         }
 358                 }
 359
 360                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 361                                                         out string[] dependencies)
 362                 {
 363                         int row = 0;
 364                         compiler_options = null;
 365                         dependencies = null;
 366                         try {
 367                                 using (StreamReader sr = new StreamReader (file)) {
 368                                         String line;
 369                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 370                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 371                                                                       ref dependencies))
 372                                                         return false;
 373                                         }
 374                                 }
 375                         } catch {
 376                                 return false;
 377                         }
 378                         return true;
 379                 }
 380
 381                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 382                                                         ref string[] compiler_options,
 383                                                         ref string[] dependencies)
 384                 {
 385                         const string options = "// Compiler options:";
 386                         const string depends = "// Dependencies:";
 387
 388                         if (row == 1) {
 389                                 compiler_options = null;
 390                                 dependencies = null;
 391                         }
 392
 393                         int index = line.IndexOf (options);
 394                         if (index != -1) {
 395                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 396                                 for (int i = 0; i < compiler_options.Length; i++)
 397                                         compiler_options[i] = compiler_options[i].TrimStart ();
 398                         }
 399                         index = line.IndexOf (depends);
 400                         if (index != -1) {
 401                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 402                                 for (int i = 0; i < dependencies.Length; i++)
 403                                         dependencies[i] = dependencies[i].TrimStart ();
 404                         }
 405
 406                         return true;
 407                 }
 408
 409                 public bool Do (string filename)
 410                 {
 411                         if (test_hash.Contains (filename))
 412                                 return true;
 413
 414                         if (verbose)
 415                                 Log (filename + "...\t");
 416
 417                         if (ignore_list.Contains (filename)) {
 418                                 ++ignored;
 419                                 LogFileLine (filename, "NOT TESTED");
 420                                 return false;
 421                         }
 422
 423                         string[] compiler_options, dependencies;
 424                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 425                                 LogFileLine (filename, "ERROR");
 426                                 return false;
 427                         }
 428
 429                         if (extra_compiler_options != null) {
 430                                 if (compiler_options == null)
 431                                         compiler_options = extra_compiler_options;
 432                                 else {
 433                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 434                                         extra_compiler_options.CopyTo (new_options, 0);
 435                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 436                                         compiler_options = new_options;
 437                                 }
 438                         }
 439
 440                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 441                         test_hash.Add (filename, test);
 442
 443                         ++total;
 444                         if (dependencies != null) {
 445                                 foreach (string dependency in dependencies) {
 446                                         if (!Do (dependency)) {
 447                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 448                                                 return false;
 449                                         }
 450                                 }
 451                         }
 452
 453                         tests.Add (test);
 454
 455                         return Check (test);
 456                 }
 457
 458                 protected virtual bool Check (TestCase test)
 459                 {
 460                         string[] test_args;
 461
 462                         if (test.CompilerOptions != null) {
 463                                 test_args = new string [2 + test.CompilerOptions.Length];
 464                                 test.CompilerOptions.CopyTo (test_args, 0);
 465                         } else {
 466                                 test_args = new string [2];
 467                         }
 468                         test_args [test_args.Length - 2] = test.FileName;
 469                         test_args [test_args.Length - 1] = "-debug";
 470
 471                         return tester.Invoke (test_args);
 472                 }
 473
 474                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 475                 {
 476                         return new TestCase (filename, options, deps);
 477                 }
 478
 479                 void ReadWrongErrors (string file)
 480                 {
 481                         const string ignored = "IGNORE";
 482                         const string no_error = "NO ERROR";
 483
 484                         using (StreamReader sr = new StreamReader (file)) {
 485                                 string line;
 486                                 while ((line = sr.ReadLine()) != null) {
 487                                         if (line.StartsWith ("#"))
 488                                                 continue;
 489
 490                                         ArrayList active_cont = know_issues;
 491
 492                                         if (line.IndexOf (ignored) > 0)
 493                                                 active_cont = ignore_list;
 494                                         else if (line.IndexOf (no_error) > 0)
 495                                                 active_cont = no_error_list;
 496
 497                                         string file_name = line.Split (' ')[0];
 498                                         if (file_name.Length == 0)
 499                                                 continue;
 500
 501                                         active_cont.Add (file_name);
 502                                 }
 503                         }
 504                         total_known_issues = know_issues.Count;
 505                 }
 506
 507                 protected virtual void PrintSummary ()
 508                 {
 509                         LogLine ("Done" + Environment.NewLine);
 510                         float rate = 0;
 511                         if (total > 0)
 512                                 rate = (float) (success) / (float)total;
 513                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 514
 515                         if (syntax_errors > 0)
 516                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 517
 518                         if (ignored > 0)
 519                                 LogLine ("{0} test(s) ignored", ignored);
 520
 521                         if (total_known_issues - know_issues.Count > 0)
 522                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 523
 524                         know_issues.AddRange (no_error_list);
 525                         if (know_issues.Count > 0) {
 526                                 LogLine ("");
 527                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 528                                 foreach (string s in know_issues)
 529                                         LogLine (s);
 530                         }
 531                         if (regression.Count > 0) {
 532                                 LogLine ("");
 533                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 534                                 foreach (string s in regression)
 535                                         LogLine (s);
 536                         }
 537                 }
 538
 539                 public int ResultCode
 540                 {
 541                         get {
 542                                 return regression.Count == 0 ? 0 : 1;
 543                         }
 544                 }
 545
 546                 protected void Log (string msg, params object [] rest)
 547                 {
 548                         Console.Write (msg, rest);
 549                         if (log_file != null)
 550                                 log_file.Write (msg, rest);
 551                 }
 552
 553                 protected void LogLine (string msg)
 554                 {
 555                         Console.WriteLine (msg);
 556                         if (log_file != null)
 557                                 log_file.WriteLine (msg);
 558                 }
 559
 560                 protected void LogLine (string msg, params object [] rest)
 561                 {
 562                         Console.WriteLine (msg, rest);
 563                         if (log_file != null)
 564                                 log_file.WriteLine (msg, rest);
 565                 }
 566
 567                 public void LogFileLine (string file, string msg, params object [] args)
 568                 {
 569                         string s = verbose ?
 570                                 string.Format (msg, args) :
 571                                 file + "...\t" + string.Format (msg, args);
 572
 573                         Console.WriteLine (s);
 574                         if (log_file != null)
 575                                 log_file.WriteLine (s);
 576                 }
 577
 578                 #region IDisposable Members
 579
 580                 public void Dispose()
 581                 {
 582                         if (log_file != null)
 583                                 log_file.Close ();
 584                 }
 585
 586                 #endregion
 587
 588                 public virtual void Initialize ()
 589                 {
 590                 }
 591
 592                 public virtual void CleanUp ()
 593                 {
 594                         PrintSummary ();
 595                 }
 596         }
 597
 598         class PositiveChecker: Checker
 599         {
 600                 readonly string files_folder;
 601                 readonly static object[] default_args = new object[1] { new string[] {} };
 602                 string doc_output;
 603                 string verif_file;
 604                 bool update_verif_file;
 605                 Hashtable verif_data;
 606
 607 #if !NET_2_1
 608                 ProcessStartInfo pi;
 609 #endif
 610                 readonly string mono;
 611
 612                 public enum TestResult {
 613                         CompileError,
 614                         ExecError,
 615                         LoadError,
 616                         XmlError,
 617                         Success,
 618                         ILError,
 619                         DebugError,
 620                         MethodAttributesError
 621                 }
 622
 623                 public PositiveChecker (ITester tester, string verif_file):
 624                         base (tester)
 625                 {
 626                         files_folder = Directory.GetCurrentDirectory ();
 627                         this.verif_file = verif_file;
 628
 629 #if !NET_2_1
 630                         pi = new ProcessStartInfo ();
 631                         pi.CreateNoWindow = true;
 632                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 633                         pi.RedirectStandardOutput = true;
 634                         pi.RedirectStandardError = true;
 635                         pi.UseShellExecute = false;
 636
 637                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 638                         if (mono != null) {
 639                                 pi.FileName = mono;
 640                         }
 641 #endif
 642                 }
 643
 644                 public bool UpdateVerificationDataFile {
 645                         set {
 646                                 update_verif_file = value;
 647                         }
 648                         get {
 649                                 return update_verif_file;
 650                         }
 651                 }
 652
 653                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 654                                                         out string[] dependencies) {
 655                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 656                                 return false;
 657
 658                         doc_output = null;
 659                         if (compiler_options == null)
 660                                 return true;
 661
 662                         foreach (string one_opt in compiler_options) {
 663                                 if (one_opt.StartsWith ("-doc:")) {
 664                                         doc_output = one_opt.Split (':', '/')[1];
 665                                 }
 666                         }
 667                         return true;
 668                 }
 669
 670                 class DomainTester : MarshalByRefObject
 671                 {
 672                         public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
 673                         {
 674                                 Assembly assembly = Assembly.LoadFile (file);
 675
 676                                 bool success = true;
 677                                 Type[] types = assembly.GetTypes ();
 678                                 foreach (Type t in types) {
 679
 680                                         // Skip interfaces
 681                                         if (!t.IsClass && !t.IsValueType)
 682                                                 continue;
 683
 684                                         if (test.VerificationProvider == null) {
 685                                                 if (!checker.UpdateVerificationDataFile)
 686                                                         checker.LogFileLine (test.FileName, "Missing IL verification data");
 687                                                 test.CreateNewTest ();
 688                                         }
 689
 690                                         foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 691                                                 MethodBase mi = m as MethodBase;
 692                                                 if (mi == null)
 693                                                         continue;
 694
 695                                                 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 696                                                         continue;
 697
 698                                                 success &= CompareIL (mi, test, checker);
 699                                         }
 700                                 }
 701
 702                                 return success;
 703                         }
 704
 705                         bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
 706                         {
 707                                 string m_name = mi.ToString ();
 708                                 string decl_type = mi.DeclaringType.ToString ();
 709                                 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
 710
 711                                 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
 712                                 if (md == null) {
 713                                         data_provider.AddNewMethod (mi, GetILSize (mi));
 714                                         if (!data_provider.IsNewSet) {
 715                                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 716                                                 return false;
 717                                         }
 718
 719                                         return true;
 720                                 }
 721
 722                                 if (md.Checked) {
 723                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 724                                         return false;
 725                                 }
 726
 727                                 md.Checked = true;
 728
 729                                 if (md.MethodAttributes != (int) mi.Attributes) {
 730                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.MethodAttributesError,
 731                                                 string.Format ("{0} ({1} -> {2})", decl_type + ": " + m_name, md.MethodAttributes, mi.Attributes));
 732                                 }
 733
 734                                 md.MethodAttributes = (int) mi.Attributes;
 735
 736                                 int il_size = GetILSize (mi);
 737                                 if (md.ILSize == il_size)
 738                                         return true;
 739
 740                                 if (md.ILSize > il_size) {
 741                                         checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size);
 742                                         md.ILSize = il_size;
 743                                         return true;
 744                                 }
 745
 746                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
 747                                         string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
 748
 749                                 md.ILSize = il_size;
 750
 751                                 return false;
 752                         }
 753
 754                         static int GetILSize (MethodBase mi)
 755                         {
 756                                 MethodBody body = mi.GetMethodBody ();
 757                                 if (body != null)
 758                                         return body.GetILAsByteArray ().Length;
 759
 760                                 return 0;
 761                         }
 762
 763                         bool ExecuteFile (MethodInfo entry_point, string filename)
 764                         {
 765                                 TextWriter stdout = Console.Out;
 766                                 TextWriter stderr = Console.Error;
 767                                 Console.SetOut (TextWriter.Null);
 768                                 Console.SetError (TextWriter.Null);
 769                                 ParameterInfo[] pi = entry_point.GetParameters ();
 770                                 object[] args = pi.Length == 0 ? null : default_args;
 771
 772                                 object result = null;
 773                                 try {
 774                                         try {
 775                                                 result = entry_point.Invoke (null, args);
 776                                         } finally {
 777                                                 Console.SetOut (stdout);
 778                                                 Console.SetError (stderr);
 779                                         }
 780                                 } catch (Exception e) {
 781                                         throw new ApplicationException (e.ToString ());
 782                                 }
 783
 784                                 if (result is int && (int) result != 0)
 785                                         throw new ApplicationException ("Wrong return code: " + result.ToString ());
 786
 787                                 return true;
 788                         }
 789
 790                         public bool Test (string file)
 791                         {
 792                                 Assembly assembly = Assembly.LoadFile (file);
 793                                 return ExecuteFile (assembly.EntryPoint, file);
 794                         }
 795                 }
 796
 797                 protected override bool Check(TestCase test)
 798                 {
 799                         string filename = test.FileName;
 800                         try {
 801                                 if (!base.Check (test)) {
 802                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 803                                         return false;
 804                                 }
 805                         }
 806                         catch (Exception e) {
 807                                 if (e.InnerException != null)
 808                                         e = e.InnerException;
 809
 810                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 811                                 return false;
 812                         }
 813
 814                         // Test setup
 815                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 816                                 if (verbose)
 817                                         LogFileLine (filename, "OK");
 818                                 --total;
 819                                 return true;
 820                         }
 821
 822                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 823
 824                         // Enable .dll only tests (no execution required)
 825                         if (!File.Exists(file)) {
 826                                 HandleFailure (filename, TestResult.Success, null);
 827                                 return true;
 828                         }
 829
 830                         AppDomain domain = null;
 831 #if !NET_2_1
 832                         if (safe_execution) {
 833                                 // Create a new AppDomain, with the current directory as the base.
 834                                 AppDomainSetup setupInfo = new AppDomainSetup ();
 835                                 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
 836                                 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
 837                                 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
 838                         }
 839 #endif
 840                         try {
 841                                 DomainTester tester;
 842                                 try {
 843 #if !NET_2_1
 844                                         if (domain != null)
 845                                                 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
 846                                         else
 847 #endif
 848                                                 tester = new DomainTester ();
 849
 850                                         if (!tester.Test (file))
 851                                                 return false;
 852
 853                                 } catch (ApplicationException e) {
 854                                         HandleFailure (filename, TestResult.ExecError, e.Message);
 855                                         return false;
 856                                 } catch (Exception e) {
 857                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 858                                         return false;
 859                                 }
 860
 861                                 if (doc_output != null) {
 862                                         string ref_file = filename.Replace (".cs", "-ref.xml");
 863                                         try {
 864 #if !NET_2_1
 865                                                 new XmlComparer ("doc").Compare (ref_file, doc_output);
 866 #endif
 867                                         } catch (Exception e) {
 868                                                 HandleFailure (filename, TestResult.XmlError, e.Message);
 869                                                 return false;
 870                                         }
 871                                 } else {
 872                                         if (verif_file != null) {
 873                                                 PositiveTestCase pt = (PositiveTestCase) test;
 874                                                 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
 875
 876                                                 if (!tester.CheckILSize (pt, this, file))
 877                                                         return false;
 878                                         }
 879
 880                                         if (filename.StartsWith ("test-debug", StringComparison.OrdinalIgnoreCase)) {
 881                                                 var mdb_file_name = file + ".mdb";
 882                                                 MonoSymbolFile mdb_file = MonoSymbolFile.ReadSymbolFile (mdb_file_name);
 883                                                 var mdb_xml_file = mdb_file_name + ".xml";
 884                                                 ConvertSymbolFileToXml (mdb_file, mdb_xml_file);
 885
 886                                                 var ref_file = filename.Replace(".cs", "-ref.xml");
 887                                                 try {
 888                                                         new XmlComparer ("symbols").Compare (ref_file, mdb_xml_file);
 889                                                 } catch (Exception e) {
 890                                                         HandleFailure (filename, TestResult.DebugError, e.Message);
 891                                                         return false;
 892                                                 }
 893                                         }
 894
 895                                 }
 896                         } finally {
 897                                 if (domain != null)
 898                                         AppDomain.Unload (domain);
 899                         }
 900
 901                         HandleFailure (filename, TestResult.Success, null);
 902                         return true;
 903                 }
 904
 905                 static void ConvertSymbolFileToXml (MonoSymbolFile symbolFile, string xmlFile)
 906                 {
 907                         using (XmlTextWriter writer = new XmlTextWriter (xmlFile, Encoding.UTF8)) {
 908                                 writer.Formatting = Formatting.Indented;
 909
 910                                 writer.WriteStartDocument ();
 911
 912                                 writer.WriteStartElement ("symbols");
 913
 914                                 writer.WriteStartElement ("files");
 915                                 foreach (var file in symbolFile.Sources) {
 916                                         writer.WriteStartElement ("file");
 917                                         writer.WriteAttributeString ("id", file.Index.ToString ());
 918                                         writer.WriteAttributeString ("name", Path.GetFileName (file.FileName));
 919                                         var checksum = file.Checksum;
 920                                         if (checksum != null)
 921                                                 writer.WriteAttributeString ("checksum", ChecksumToString (checksum));
 922
 923                                         writer.WriteEndElement ();
 924                                 }
 925                                 writer.WriteEndElement ();
 926
 927                                 writer.WriteStartElement ("methods");
 928                                 foreach (var method in symbolFile.Methods) {
 929                                         writer.WriteStartElement ("method");
 930                                         writer.WriteAttributeString ("token", IntToHex (method.Token));
 931
 932                                         var il_entries = method.GetLineNumberTable ();
 933                                         writer.WriteStartElement ("sequencepoints");
 934                                         foreach (var entry in il_entries.LineNumbers) {
 935                                                 writer.WriteStartElement ("entry");
 936                                                 writer.WriteAttributeString ("il", IntToHex (entry.Offset));
 937                                                 writer.WriteAttributeString ("row", entry.Row.ToString ());
 938                                                 writer.WriteAttributeString ("file_ref", entry.File.ToString ());
 939                                                 writer.WriteAttributeString ("hidden", BoolToString (entry.IsHidden));
 940                                                 writer.WriteEndElement ();
 941                                         }
 942                                         writer.WriteEndElement ();
 943
 944                                         writer.WriteStartElement ("locals");
 945                                         foreach (var local in method.GetLocals ()) {
 946                                                 writer.WriteStartElement ("entry");
 947                                                 writer.WriteAttributeString ("name", local.Name);
 948                                                 writer.WriteAttributeString ("il_index", local.Index.ToString ());
 949                                                 writer.WriteAttributeString ("scope_ref", local.BlockIndex.ToString ());
 950                                                 writer.WriteEndElement ();
 951                                         }
 952                                         writer.WriteEndElement ();
 953
 954                                         writer.WriteStartElement ("scopes");
 955                                         foreach (var scope in method.GetCodeBlocks ()) {
 956                                                 writer.WriteStartElement ("entry");
 957                                                 writer.WriteAttributeString ("index", scope.Index.ToString ());
 958                                                 writer.WriteAttributeString ("start", IntToHex (scope.StartOffset));
 959                                                 writer.WriteAttributeString ("end", IntToHex (scope.EndOffset));
 960                                                 writer.WriteEndElement ();
 961                                         }
 962                                         writer.WriteEndElement ();
 963
 964                                         writer.WriteEndElement ();
 965                                 }
 966                                 writer.WriteEndElement ();
 967
 968                                 writer.WriteEndElement ();
 969                                 writer.WriteEndDocument ();
 970                         }
 971                 }
 972
 973                 static string ChecksumToString (byte[] checksum)
 974                 {
 975                         var sb = new StringBuilder (checksum.Length * 2);
 976                         for (int i = 0; i < checksum.Length; i++) {
 977                                 sb.Append ("0123456789abcdef"[checksum[i] >> 4]);
 978                                 sb.Append ("0123456789abcdef"[checksum[i] & 0x0F]);
 979                         }
 980
 981                         return sb.ToString ();
 982                 }
 983
 984                 static string IntToHex (int value)
 985                 {
 986                         return "0x" + value.ToString ("x", CultureInfo.InvariantCulture);
 987                 }
 988
 989                 static string BoolToString (bool value)
 990                 {
 991                         return value ? "true" : "false";
 992                 }
 993
 994                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
 995                 {
 996                         return new PositiveTestCase (filename, options, deps);
 997                 }
 998
 999                 public void HandleFailure (string file, TestResult status, string extra)
1000                 {
1001                         switch (status) {
1002                                 case TestResult.Success:
1003                                         success++;
1004                                         if (know_issues.Contains (file)) {
1005                                                 LogFileLine (file, "FIXED ISSUE");
1006                                                 return;
1007                                         }
1008                                         if (verbose)
1009                                                 LogFileLine (file, "OK");
1010                                         return;
1011
1012                                 case TestResult.CompileError:
1013                                         if (know_issues.Contains (file)) {
1014                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
1015                                                 know_issues.Remove (file);
1016                                                 return;
1017                                         }
1018                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
1019                                         break;
1020
1021                                 case TestResult.ExecError:
1022                                         if (know_issues.Contains (file)) {
1023                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
1024                                                 know_issues.Remove (file);
1025                                                 return;
1026                                         }
1027                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
1028                                         break;
1029
1030                                 case TestResult.XmlError:
1031                                         if (know_issues.Contains (file)) {
1032                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
1033                                                 know_issues.Remove (file);
1034                                                 return;
1035                                         }
1036                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
1037                                         break;
1038
1039                                 case TestResult.LoadError:
1040                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
1041                                         break;
1042
1043                                 case TestResult.MethodAttributesError:
1044                                 case TestResult.ILError:
1045                                         if (!update_verif_file) {
1046                                                 LogFileLine (file, "IL REGRESSION: " + extra);
1047                                         }
1048                                         extra = null;
1049                                         break;
1050
1051                                 case TestResult.DebugError:
1052                                         LogFileLine (file, "REGRESSION (SUCCESS -> SYMBOL FILE ERROR)");
1053                                         break;
1054                         }
1055
1056                         if (extra != null)
1057                                 LogLine ("{0}", extra);
1058
1059                         if (!regression.Contains (file))
1060                                 regression.Add (file);
1061                 }
1062
1063                 public override void Initialize ()
1064                 {
1065                         if (verif_file != null) {
1066                                 LoadVerificationData (verif_file);
1067                         }
1068
1069                         base.Initialize ();
1070                 }
1071
1072                 public override void CleanUp ()
1073                 {
1074                         base.CleanUp ();
1075
1076                         if (update_verif_file) {
1077                                 UpdateVerificationData (verif_file);
1078                         }
1079                 }
1080
1081                 void LoadVerificationData (string file)
1082                 {
1083                         verif_data = new Hashtable ();
1084
1085                         if (!File.Exists (file)) {
1086                                 LogLine ("Writing verification data to `{0}' ...", file);
1087                                 return;
1088                         }
1089
1090                         LogLine ("Loading verification data from `{0}' ...", file);
1091
1092                         using (XmlReader r = XmlReader.Create (file)) {
1093                                 r.ReadStartElement ("tests");
1094
1095                                 while (r.Read ()) {
1096                                         if (r.Name != "test")
1097                                                 continue;
1098
1099                                         string name = r.GetAttribute ("name");
1100                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
1101                                         verif_data.Add (name, tc);
1102                                 }
1103                         }
1104                 }
1105
1106                 void UpdateVerificationData (string file)
1107                 {
1108                         LogLine ("Updating verification data `{0}' ...", file);
1109
1110                         XmlWriterSettings s = new XmlWriterSettings ();
1111                         s.Indent = true;
1112                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
1113                                 w.WriteStartDocument ();
1114                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
1115                                 w.WriteStartElement ("tests");
1116                                 foreach (PositiveTestCase tc in tests) {
1117                                         if (tc.VerificationProvider != null)
1118                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
1119                                 }
1120                                 w.WriteEndElement ();
1121                         }
1122                 }
1123         }
1124
1125         class NegativeChecker: Checker
1126         {
1127                 string expected_message;
1128                 string error_message;
1129                 bool check_msg;
1130                 bool check_error_line;
1131                 bool is_warning;
1132                 IDictionary wrong_warning;
1133
1134                 protected enum CompilerError {
1135                         Expected,
1136                         Wrong,
1137                         Missing,
1138                         WrongMessage,
1139                         MissingLocation,
1140                         Duplicate
1141                 }
1142
1143                 public NegativeChecker (ITester tester, bool check_msg):
1144                         base (tester)
1145                 {
1146                         this.check_msg = check_msg;
1147                         wrong_warning = new Hashtable ();
1148                 }
1149
1150                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1151                                                         ref string[] compiler_options,
1152                                                         ref string[] dependencies)
1153                 {
1154                         if (row == 1) {
1155                                 expected_message = null;
1156
1157                                 int index = line.IndexOf (':');
1158                                 if (index == -1 || index > 15) {
1159                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1160                                         ++syntax_errors;
1161                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1162                                                               ref dependencies);
1163                                         return false;
1164                                 }
1165
1166                                 expected_message = line.Substring (index + 1).Trim ();
1167                         }
1168
1169                         if (row == 2) {
1170                                 string filtered = line.Replace(" ", "");
1171
1172                                 // Some error tests require to have different error text for different runtimes.
1173                                 if (filtered.StartsWith ("//GMCS")) {
1174                                         row = 1;
1175                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1176                                 }
1177
1178                                 check_error_line = !filtered.StartsWith ("//Line:0");
1179
1180                                 if (!filtered.StartsWith ("//Line:")) {
1181                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1182                                         ++syntax_errors;
1183                                         return false;
1184                                 }
1185                         }
1186
1187                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1188                                 return false;
1189
1190                         is_warning = false;
1191                         if (compiler_options != null) {
1192                                 foreach (string s in compiler_options) {
1193                                         if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1194                                                 is_warning = true;
1195                                 }
1196                         }
1197                         return true;
1198                 }
1199
1200
1201                 protected override bool Check (TestCase test)
1202                 {
1203                         string filename = test.FileName;
1204
1205                         int start_char = 0;
1206                         while (Char.IsLetter (filename, start_char))
1207                                 ++start_char;
1208
1209                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1210                         string expected = filename.Substring (start_char, end_char - start_char);
1211
1212                         try {
1213                                 if (base.Check (test)) {
1214                                         HandleFailure (filename, CompilerError.Missing);
1215                                         return false;
1216                                 }
1217                         }
1218                         catch (Exception e) {
1219                                 HandleFailure (filename, CompilerError.Missing);
1220                                 if (e.InnerException != null)
1221                                         e = e.InnerException;
1222
1223                                 Log (e.ToString ());
1224                                 return false;
1225                         }
1226
1227                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1228                         if (tester.IsWarning (err_id)) {
1229                                 if (!is_warning)
1230                                         wrong_warning [err_id] = true;
1231                         } else {
1232                                 if (is_warning)
1233                                         wrong_warning [err_id] = false;
1234                         }
1235
1236                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1237                         if (HandleFailure (filename, result_code)) {
1238                                 success++;
1239                                 return true;
1240                         }
1241
1242                         if (result_code == CompilerError.Wrong)
1243                                 LogLine (tester.Output);
1244
1245                         return false;
1246                 }
1247
1248                 CompilerError GetCompilerError (string expected, string buffer)
1249                 {
1250                         const string error_prefix = "CS";
1251                         const string ignored_error = "error CS5001";
1252                         string tested_text = "error " + error_prefix + expected;
1253                         StringReader sr = new StringReader (buffer);
1254                         string line = sr.ReadLine ();
1255                         ArrayList ld = new ArrayList ();
1256                         CompilerError result = CompilerError.Missing;
1257                         while (line != null) {
1258                                 if (ld.Contains (line) && result == CompilerError.Expected) {
1259                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1260                                                 return CompilerError.Duplicate;
1261                                 }
1262                                 ld.Add (line);
1263
1264                                 if (result != CompilerError.Expected) {
1265                                         if (line.IndexOf (tested_text) != -1) {
1266                                                 if (check_msg) {
1267                                                         int first = line.IndexOf (':');
1268                                                         int second = line.IndexOf (':', first + 1);
1269                                                         if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
1270                                                                 if (check_error_line) {
1271                                                                         second = line.IndexOf (':', second + 1);
1272                                                                 }
1273                                                         } else if (second == -1 || !check_error_line) {
1274                                                                 second = first;
1275                                                         }
1276
1277                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1278                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1279                                                                 error_message = msg;
1280                                                                 return CompilerError.WrongMessage;
1281                                                         }
1282
1283                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1284                                                                 return CompilerError.MissingLocation;
1285                                                 }
1286                                                 result = CompilerError.Expected;
1287                                         } else if (line.IndexOf (error_prefix) != -1 &&
1288                                                 line.IndexOf (ignored_error) == -1)
1289                                                 result = CompilerError.Wrong;
1290                                 }
1291
1292                                 line = sr.ReadLine ();
1293                         }
1294
1295                         return result;
1296                 }
1297
1298                 bool HandleFailure (string file, CompilerError status)
1299                 {
1300                         switch (status) {
1301                                 case CompilerError.Expected:
1302                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1303                                                 LogFileLine (file, "FIXED ISSUE");
1304                                                 return true;
1305                                         }
1306
1307                                         if (verbose)
1308                                                 LogFileLine (file, "OK");
1309                                         return true;
1310
1311                                 case CompilerError.Wrong:
1312                                         if (know_issues.Contains (file)) {
1313                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1314                                                 know_issues.Remove (file);
1315                                                 return false;
1316                                         }
1317                                         if (no_error_list.Contains (file)) {
1318                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1319                                                 no_error_list.Remove (file);
1320                                         }
1321                                         else {
1322                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1323                                         }
1324                                         break;
1325
1326                                 case CompilerError.WrongMessage:
1327                                         if (know_issues.Contains (file)) {
1328                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1329                                                 know_issues.Remove (file);
1330                                                 return false;
1331                                         }
1332                                         if (no_error_list.Contains (file)) {
1333                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1334                                                 no_error_list.Remove (file);
1335                                         }
1336                                         else {
1337                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1338                                                 LogLine ("Exp: {0}", expected_message);
1339                                                 LogLine ("Was: {0}", error_message);
1340                                         }
1341                                         break;
1342
1343                                 case CompilerError.Missing:
1344                                         if (no_error_list.Contains (file)) {
1345                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1346                                                 no_error_list.Remove (file);
1347                                                 return false;
1348                                         }
1349
1350                                         if (know_issues.Contains (file)) {
1351                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1352                                                 know_issues.Remove (file);
1353                                         }
1354                                         else {
1355                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1356                                         }
1357
1358                                         break;
1359
1360                                 case CompilerError.MissingLocation:
1361                                         if (know_issues.Contains (file)) {
1362                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1363                                                 know_issues.Remove (file);
1364                                                 return false;
1365                                         }
1366                                         if (no_error_list.Contains (file)) {
1367                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1368                                                 no_error_list.Remove (file);
1369                                         }
1370                                         else {
1371                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1372                                         }
1373                                         break;
1374
1375                                 case CompilerError.Duplicate:
1376                                         // Will become an error soon
1377                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1378                                         return true;
1379                         }
1380
1381                         regression.Add (file);
1382                         return false;
1383                 }
1384
1385                 protected override void PrintSummary()
1386                 {
1387                         base.PrintSummary ();
1388
1389                         if (wrong_warning.Count > 0) {
1390                                 LogLine ("");
1391                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1392                                 LogLine ("");
1393                                 foreach (DictionaryEntry de in wrong_warning)
1394                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1395                         }
1396                 }
1397
1398         }
1399
1400         class Tester {
1401
1402                 static int Main(string[] args)
1403                 {
1404                         string temp;
1405
1406                         if (GetOption ("help", args, false, out temp)) {
1407                                 Usage ();
1408                                 return 0;
1409                         }
1410
1411                         string compiler;
1412                         if (!GetOption ("compiler", args, true, out compiler)) {
1413                                 Usage ();
1414                                 return 1;
1415                         }
1416
1417                         ITester tester;
1418                         try {
1419                                 Console.WriteLine ("Loading " + compiler + " ...");
1420                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1421                         }
1422                         catch (Exception) {
1423 #if NET_2_1
1424                                 throw;
1425 #else
1426                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1427                                 if (!File.Exists (compiler)) {
1428                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1429                                         return 1;
1430                                 }
1431                                 tester = new ProcessTester (compiler);
1432 #endif
1433                         }
1434
1435                         string mode;
1436                         if (!GetOption ("mode", args, true, out mode)) {
1437                                 Usage ();
1438                                 return 1;
1439                         }
1440
1441                         Checker checker;
1442                         bool positive;
1443                         switch (mode) {
1444                                 case "neg":
1445                                         checker = new NegativeChecker (tester, true);
1446                                         positive = false;
1447                                         break;
1448                                 case "pos":
1449                                         string iltest;
1450                                         GetOption ("il", args, false, out iltest);
1451                                         checker = new PositiveChecker (tester, iltest);
1452                                         positive = true;
1453
1454                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1455                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1456                                         }
1457
1458                                         break;
1459                                 default:
1460                                         Console.Error.WriteLine ("Invalid -mode argument");
1461                                         return 1;
1462                         }
1463
1464
1465                         if (GetOption ("issues", args, true, out temp))
1466                                 checker.IssueFile = temp;
1467                         if (GetOption ("log", args, true, out temp))
1468                                 checker.LogFile = temp;
1469                         if (GetOption ("verbose", args, false, out temp))
1470                                 checker.Verbose = true;
1471                         if (GetOption ("safe-execution", args, false, out temp))
1472                                 checker.SafeExecution = true;
1473                         if (GetOption ("compiler-options", args, true, out temp)) {
1474                                 string[] extra = temp.Split (' ');
1475                                 checker.ExtraCompilerOptions = extra;
1476                         }
1477
1478                         string test_pattern;
1479                         if (!GetOption ("files", args, true, out test_pattern)) {
1480                                 Usage ();
1481                                 return 1;
1482                         }
1483
1484                         var files = new List<string> ();
1485                         switch (test_pattern) {
1486                         case "v1":
1487                                 files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
1488                                 break;
1489                         case "v2":
1490                                 files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
1491                                 goto case "v1";
1492                         case "v4":
1493                                 files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
1494                                 goto case "v2";
1495                         default:
1496                                 files.AddRange (Directory.GetFiles (".", test_pattern));
1497                                 break;
1498                         }
1499
1500                         if (files.Count == 0) {
1501                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1502                                 return 2;
1503                         }
1504
1505                         checker.Initialize ();
1506 /*
1507                         files.Sort ((a, b) => {
1508                                 if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1509                                         if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
1510                                                 return -1;
1511                                 } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1512                                         if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
1513                                                 return 1;
1514                                 }
1515
1516                                 return a.CompareTo (b);
1517                         });
1518 */
1519                         foreach (string s in files) {
1520                                 string filename = Path.GetFileName (s);
1521                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1522                                         continue;
1523                                 }
1524
1525                                 if (filename.EndsWith ("-p2.cs"))
1526                                         continue;
1527
1528                                 checker.Do (filename);
1529                         }
1530
1531                         checker.CleanUp ();
1532
1533                         checker.Dispose ();
1534
1535                         return checker.ResultCode;
1536                 }
1537
1538                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1539                 {
1540                         opt = "-" + opt;
1541                         foreach (string a in args) {
1542                                 if (a.StartsWith (opt)) {
1543                                         int sep = a.IndexOf (':');
1544                                         if (sep > 0) {
1545                                                 value = a.Substring (sep + 1);
1546                                         } else {
1547                                                 value = null;
1548                                                 if (req_arg) {
1549                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1550                                                         return false;
1551                                                 }
1552                                         }
1553
1554                                         return true;
1555                                 }
1556                         }
1557
1558                         value = null;
1559                         return false;
1560                 }
1561
1562                 static void Usage ()
1563                 {
1564                         Console.WriteLine (
1565                                 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1566                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1567                                 "   \n" +
1568                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1569                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1570                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1571                                 "   -issues:FILE     The list of expected failures\n" +
1572                                 "   -log:FILE        Writes any output also to the file\n" +
1573                                 "   -help            Lists all options\n" +
1574                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1575                                 "   -safe-execution  Runs compiled executables in separate app-domain\n" +
1576                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1577                                 "   -verbose         Prints more details during testing\n"
1578                                 );
1579                 }
1580         }
1581 }