mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
   8 // Copyright (C) 2012 Xamarin Inc (http://www.xamarin.com)
   9 //
  10 // Permission is hereby granted, free of charge, to any person obtaining
  11 // a copy of this software and associated documentation files (the
  12 // "Software"), to deal in the Software without restriction, including
  13 // without limitation the rights to use, copy, modify, merge, publish,
  14 // distribute, sublicense, and/or sell copies of the Software, and to
  15 // permit persons to whom the Software is furnished to do so, subject to
  16 // the following conditions:
  17 //
  18 // The above copyright notice and this permission notice shall be
  19 // included in all copies or substantial portions of the Software.
  20 //
  21 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  22 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  23 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  24 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  25 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  26 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  27 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  28 //
  29
  30 using System;
  31 using System.IO;
  32 using System.Diagnostics;
  33 using System.Reflection;
  34 using System.Text;
  35 using System.Collections;
  36 using System.Xml;
  37 using System.Collections.Generic;
  38 using Mono.CompilerServices.SymbolWriter;
  39 using System.Globalization;
  40
  41 namespace TestRunner {
  42
  43         interface ITester
  44         {
  45                 string Output { get; }
  46                 bool Invoke (string[] args);
  47                 bool IsWarning (int warningNumber);
  48         }
  49
  50         class ReflectionTester: ITester {
  51                 MethodInfo ep;
  52                 object[] method_arg;
  53                 StringWriter output;
  54                 int[] all_warnings;
  55
  56                 public ReflectionTester (Assembly a)
  57                 {
  58                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  59
  60                         if (t == null)
  61                                 Console.Error.WriteLine ("null, huh?");
  62
  63                         ep = t.GetMethod ("InvokeCompiler",
  64                                 BindingFlags.Static | BindingFlags.Public);
  65                         if (ep == null)
  66                                 throw new MissingMethodException ("static InvokeCompiler");
  67                         method_arg = new object [2];
  68
  69                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  70                         all_warnings = (int[])pi.GetValue (null, null);
  71                         Array.Sort (all_warnings);
  72                 }
  73
  74                 public string Output {
  75                         get {
  76                                 return output.GetStringBuilder ().ToString ();
  77                         }
  78                 }
  79
  80                 public bool Invoke(string[] args)
  81                 {
  82                         output = new StringWriter ();
  83                         method_arg [0] = args;
  84                         method_arg [1] = output;
  85                         return (bool)ep.Invoke (null, method_arg);
  86                 }
  87
  88                 public bool IsWarning (int warningNumber)
  89                 {
  90                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  91                 }
  92         }
  93
  94 #if !NET_2_1
  95         class ProcessTester: ITester
  96         {
  97                 ProcessStartInfo pi;
  98                 string output;
  99
 100                 public ProcessTester (string p_path)
 101                 {
 102                         pi = new ProcessStartInfo ();
 103                         pi.FileName = p_path;
 104                         pi.CreateNoWindow = true;
 105                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 106                         pi.RedirectStandardOutput = true;
 107                         pi.RedirectStandardError = true;
 108                         pi.UseShellExecute = false;
 109                 }
 110
 111                 public string Output {
 112                         get {
 113                                 return output;
 114                         }
 115                 }
 116
 117                 public bool Invoke(string[] args)
 118                 {
 119                         StringBuilder sb = new StringBuilder ("/nologo ");
 120                         foreach (string s in args) {
 121                                 sb.Append (s);
 122                                 sb.Append (" ");
 123                         }
 124                         pi.Arguments = sb.ToString ();
 125                         Process p = Process.Start (pi);
 126                         output = p.StandardError.ReadToEnd ();
 127                         if (output.Length == 0)
 128                             output = p.StandardOutput.ReadToEnd ();
 129                         p.WaitForExit ();
 130                         return p.ExitCode == 0;
 131                 }
 132
 133                 public bool IsWarning (int warningNumber)
 134                 {
 135                         throw new NotImplementedException ();
 136                 }
 137         }
 138 #endif
 139
 140         class TestCase : MarshalByRefObject
 141         {
 142                 public readonly string FileName;
 143                 public readonly string[] CompilerOptions;
 144                 public readonly string[] Dependencies;
 145
 146                 public TestCase (string filename, string[] options, string[] deps)
 147                 {
 148                         this.FileName = filename;
 149                         this.CompilerOptions = options;
 150                         this.Dependencies = deps;
 151                 }
 152         }
 153
 154         class PositiveTestCase : TestCase
 155         {
 156                 public class VerificationData : MarshalByRefObject
 157                 {
 158                         public class MethodData : MarshalByRefObject
 159                         {
 160                                 public MethodData (MethodBase mi, int il_size)
 161                                 {
 162                                         this.Type = mi.DeclaringType.ToString ();
 163                                         this.MethodName = mi.ToString ();
 164                                         this.ILSize = il_size;
 165                                 }
 166
 167                                 public MethodData (string type_name, string method_name, int il_size)
 168                                 {
 169                                         this.Type = type_name;
 170                                         this.MethodName = method_name;
 171                                         this.ILSize = il_size;
 172                                 }
 173
 174                                 public string Type;
 175                                 public string MethodName;
 176                                 public int ILSize;
 177                                 public bool Checked;
 178                         }
 179
 180                         ArrayList methods;
 181                         public bool IsNewSet;
 182
 183                         public VerificationData (string test_file)
 184                         {
 185                                 this.test_file = test_file;
 186                         }
 187
 188                         string test_file;
 189
 190                         public static VerificationData FromFile (string name, XmlReader r)
 191                         {
 192                                 VerificationData tc = new VerificationData (name);
 193                                 ArrayList methods = new ArrayList ();
 194                                 r.Read ();
 195                                 while (r.ReadToNextSibling ("type")) {
 196                                         string type_name = r ["name"];
 197                                         r.Read ();
 198                                         while (r.ReadToNextSibling ("method")) {
 199                                                 string m_name = r ["name"];
 200
 201                                                 r.ReadToDescendant ("size");
 202                                                 int il_size = r.ReadElementContentAsInt ();
 203                                                 methods.Add (new MethodData (type_name, m_name, il_size));
 204                                                 r.Read ();
 205                                         }
 206                                         r.Read ();
 207                                 }
 208
 209                                 tc.methods = methods;
 210                                 return tc;
 211                         }
 212
 213                         public void WriteCodeInfoTo (XmlWriter w)
 214                         {
 215                                 w.WriteStartElement ("test");
 216                                 w.WriteAttributeString ("name", test_file);
 217
 218                                 string type = null;
 219                                 foreach (MethodData data in methods) {
 220                                         if (!data.Checked)
 221                                                 continue;
 222
 223                                         if (type != data.Type) {
 224                                                 if (type != null)
 225                                                         w.WriteEndElement ();
 226
 227                                                 type = data.Type;
 228                                                 w.WriteStartElement ("type");
 229                                                 w.WriteAttributeString ("name", type);
 230                                         }
 231
 232                                         w.WriteStartElement ("method");
 233                                         w.WriteAttributeString ("name", data.MethodName);
 234                                         w.WriteStartElement ("size");
 235                                         w.WriteValue (data.ILSize);
 236                                         w.WriteEndElement ();
 237                                         w.WriteEndElement ();
 238                                 }
 239
 240                                 if (type != null)
 241                                         w.WriteEndElement ();
 242
 243                                 w.WriteEndElement ();
 244                         }
 245
 246                         public MethodData FindMethodData (string method_name, string declaring_type)
 247                         {
 248                                 if (methods == null)
 249                                         return null;
 250
 251                                 foreach (MethodData md in methods) {
 252                                         if (md.MethodName == method_name && md.Type == declaring_type)
 253                                                 return md;
 254                                 }
 255
 256                                 return null;
 257                         }
 258
 259                         public void AddNewMethod (MethodBase mb, int il_size)
 260                         {
 261                                 if (methods == null)
 262                                         methods = new ArrayList ();
 263
 264                                 MethodData md = new MethodData (mb, il_size);
 265                                 md.Checked = true;
 266                                 methods.Add (md);
 267                         }
 268                 }
 269
 270                 VerificationData verif_data;
 271
 272                 public PositiveTestCase (string filename, string [] options, string [] deps)
 273                         : base (filename, options, deps)
 274                 {
 275                 }
 276
 277                 public void CreateNewTest ()
 278                 {
 279                         verif_data = new VerificationData (FileName);
 280                         verif_data.IsNewSet = true;
 281                 }
 282
 283                 public VerificationData VerificationProvider {
 284                         set {
 285                                 verif_data = value;
 286                         }
 287                         get {
 288                                 return verif_data;
 289                         }
 290                 }
 291         }
 292
 293         class Checker: MarshalByRefObject, IDisposable
 294         {
 295                 protected ITester tester;
 296                 protected int success;
 297                 protected int total;
 298                 protected int ignored;
 299                 protected int syntax_errors;
 300                 string issue_file;
 301                 StreamWriter log_file;
 302                 protected string[] extra_compiler_options;
 303                 // protected string[] compiler_options;
 304                 // protected string[] dependencies;
 305
 306                 protected ArrayList tests = new ArrayList ();
 307                 protected Hashtable test_hash = new Hashtable ();
 308                 protected ArrayList regression = new ArrayList ();
 309                 protected ArrayList know_issues = new ArrayList ();
 310                 protected ArrayList ignore_list = new ArrayList ();
 311                 protected ArrayList no_error_list = new ArrayList ();
 312
 313                 protected bool verbose;
 314                 protected bool safe_execution;
 315
 316                 int total_known_issues;
 317
 318                 protected Checker (ITester tester)
 319                 {
 320                         this.tester = tester;
 321                 }
 322
 323                 public string IssueFile {
 324                         set {
 325                                 this.issue_file = value;
 326                                 ReadWrongErrors (issue_file);
 327                         }
 328                 }
 329
 330                 public string LogFile {
 331                         set {
 332                                 this.log_file = new StreamWriter (value, false);
 333                         }
 334                 }
 335
 336                 public bool Verbose {
 337                         set {
 338                                 verbose = value;
 339                         }
 340                 }
 341
 342                 public bool SafeExecution {
 343                         set {
 344                                 safe_execution = value;
 345                         }
 346                 }
 347
 348                 public string[] ExtraCompilerOptions {
 349                         set {
 350                                 extra_compiler_options = value;
 351                         }
 352                 }
 353
 354                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 355                                                         out string[] dependencies)
 356                 {
 357                         int row = 0;
 358                         compiler_options = null;
 359                         dependencies = null;
 360                         try {
 361                                 using (StreamReader sr = new StreamReader (file)) {
 362                                         String line;
 363                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 364                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 365                                                                       ref dependencies))
 366                                                         return false;
 367                                         }
 368                                 }
 369                         } catch {
 370                                 return false;
 371                         }
 372                         return true;
 373                 }
 374
 375                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 376                                                         ref string[] compiler_options,
 377                                                         ref string[] dependencies)
 378                 {
 379                         const string options = "// Compiler options:";
 380                         const string depends = "// Dependencies:";
 381
 382                         if (row == 1) {
 383                                 compiler_options = null;
 384                                 dependencies = null;
 385                         }
 386
 387                         int index = line.IndexOf (options);
 388                         if (index != -1) {
 389                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 390                                 for (int i = 0; i < compiler_options.Length; i++)
 391                                         compiler_options[i] = compiler_options[i].TrimStart ();
 392                         }
 393                         index = line.IndexOf (depends);
 394                         if (index != -1) {
 395                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 396                                 for (int i = 0; i < dependencies.Length; i++)
 397                                         dependencies[i] = dependencies[i].TrimStart ();
 398                         }
 399
 400                         return true;
 401                 }
 402
 403                 public bool Do (string filename)
 404                 {
 405                         if (test_hash.Contains (filename))
 406                                 return true;
 407
 408                         if (verbose)
 409                                 Log (filename + "...\t");
 410
 411                         if (ignore_list.Contains (filename)) {
 412                                 ++ignored;
 413                                 LogFileLine (filename, "NOT TESTED");
 414                                 return false;
 415                         }
 416
 417                         string[] compiler_options, dependencies;
 418                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 419                                 LogFileLine (filename, "ERROR");
 420                                 return false;
 421                         }
 422
 423                         if (extra_compiler_options != null) {
 424                                 if (compiler_options == null)
 425                                         compiler_options = extra_compiler_options;
 426                                 else {
 427                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 428                                         extra_compiler_options.CopyTo (new_options, 0);
 429                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 430                                         compiler_options = new_options;
 431                                 }
 432                         }
 433
 434                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 435                         test_hash.Add (filename, test);
 436
 437                         ++total;
 438                         if (dependencies != null) {
 439                                 foreach (string dependency in dependencies) {
 440                                         if (!Do (dependency)) {
 441                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 442                                                 return false;
 443                                         }
 444                                 }
 445                         }
 446
 447                         tests.Add (test);
 448
 449                         return Check (test);
 450                 }
 451
 452                 protected virtual bool Check (TestCase test)
 453                 {
 454                         string[] test_args;
 455
 456                         if (test.CompilerOptions != null) {
 457                                 test_args = new string [2 + test.CompilerOptions.Length];
 458                                 test.CompilerOptions.CopyTo (test_args, 0);
 459                         } else {
 460                                 test_args = new string [2];
 461                         }
 462                         test_args [test_args.Length - 2] = test.FileName;
 463                         test_args [test_args.Length - 1] = "-debug";
 464
 465                         return tester.Invoke (test_args);
 466                 }
 467
 468                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 469                 {
 470                         return new TestCase (filename, options, deps);
 471                 }
 472
 473                 void ReadWrongErrors (string file)
 474                 {
 475                         const string ignored = "IGNORE";
 476                         const string no_error = "NO ERROR";
 477
 478                         using (StreamReader sr = new StreamReader (file)) {
 479                                 string line;
 480                                 while ((line = sr.ReadLine()) != null) {
 481                                         if (line.StartsWith ("#"))
 482                                                 continue;
 483
 484                                         ArrayList active_cont = know_issues;
 485
 486                                         if (line.IndexOf (ignored) > 0)
 487                                                 active_cont = ignore_list;
 488                                         else if (line.IndexOf (no_error) > 0)
 489                                                 active_cont = no_error_list;
 490
 491                                         string file_name = line.Split (' ')[0];
 492                                         if (file_name.Length == 0)
 493                                                 continue;
 494
 495                                         active_cont.Add (file_name);
 496                                 }
 497                         }
 498                         total_known_issues = know_issues.Count;
 499                 }
 500
 501                 protected virtual void PrintSummary ()
 502                 {
 503                         LogLine ("Done" + Environment.NewLine);
 504                         float rate = 0;
 505                         if (total > 0)
 506                                 rate = (float) (success) / (float)total;
 507                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 508
 509                         if (syntax_errors > 0)
 510                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 511
 512                         if (ignored > 0)
 513                                 LogLine ("{0} test(s) ignored", ignored);
 514
 515                         if (total_known_issues - know_issues.Count > 0)
 516                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 517
 518                         know_issues.AddRange (no_error_list);
 519                         if (know_issues.Count > 0) {
 520                                 LogLine ("");
 521                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 522                                 foreach (string s in know_issues)
 523                                         LogLine (s);
 524                         }
 525                         if (regression.Count > 0) {
 526                                 LogLine ("");
 527                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 528                                 foreach (string s in regression)
 529                                         LogLine (s);
 530                         }
 531                 }
 532
 533                 public int ResultCode
 534                 {
 535                         get {
 536                                 return regression.Count == 0 ? 0 : 1;
 537                         }
 538                 }
 539
 540                 protected void Log (string msg, params object [] rest)
 541                 {
 542                         Console.Write (msg, rest);
 543                         if (log_file != null)
 544                                 log_file.Write (msg, rest);
 545                 }
 546
 547                 protected void LogLine (string msg)
 548                 {
 549                         Console.WriteLine (msg);
 550                         if (log_file != null)
 551                                 log_file.WriteLine (msg);
 552                 }
 553
 554                 protected void LogLine (string msg, params object [] rest)
 555                 {
 556                         Console.WriteLine (msg, rest);
 557                         if (log_file != null)
 558                                 log_file.WriteLine (msg, rest);
 559                 }
 560
 561                 public void LogFileLine (string file, string msg, params object [] args)
 562                 {
 563                         string s = verbose ?
 564                                 string.Format (msg, args) :
 565                                 file + "...\t" + string.Format (msg, args);
 566
 567                         Console.WriteLine (s);
 568                         if (log_file != null)
 569                                 log_file.WriteLine (s);
 570                 }
 571
 572                 #region IDisposable Members
 573
 574                 public void Dispose()
 575                 {
 576                         if (log_file != null)
 577                                 log_file.Close ();
 578                 }
 579
 580                 #endregion
 581
 582                 public virtual void Initialize ()
 583                 {
 584                 }
 585
 586                 public virtual void CleanUp ()
 587                 {
 588                         PrintSummary ();
 589                 }
 590         }
 591
 592         class PositiveChecker: Checker
 593         {
 594                 readonly string files_folder;
 595                 readonly static object[] default_args = new object[1] { new string[] {} };
 596                 string doc_output;
 597                 string verif_file;
 598                 bool update_verif_file;
 599                 Hashtable verif_data;
 600
 601 #if !NET_2_1
 602                 ProcessStartInfo pi;
 603 #endif
 604                 readonly string mono;
 605
 606                 public enum TestResult {
 607                         CompileError,
 608                         ExecError,
 609                         LoadError,
 610                         XmlError,
 611                         Success,
 612                         ILError,
 613                         DebugError
 614                 }
 615
 616                 public PositiveChecker (ITester tester, string verif_file):
 617                         base (tester)
 618                 {
 619                         files_folder = Directory.GetCurrentDirectory ();
 620                         this.verif_file = verif_file;
 621
 622 #if !NET_2_1
 623                         pi = new ProcessStartInfo ();
 624                         pi.CreateNoWindow = true;
 625                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 626                         pi.RedirectStandardOutput = true;
 627                         pi.RedirectStandardError = true;
 628                         pi.UseShellExecute = false;
 629
 630                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 631                         if (mono != null) {
 632                                 pi.FileName = mono;
 633                         }
 634 #endif
 635                 }
 636
 637                 public bool UpdateVerificationDataFile {
 638                         set {
 639                                 update_verif_file = value;
 640                         }
 641                         get {
 642                                 return update_verif_file;
 643                         }
 644                 }
 645
 646                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 647                                                         out string[] dependencies) {
 648                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 649                                 return false;
 650
 651                         doc_output = null;
 652                         if (compiler_options == null)
 653                                 return true;
 654
 655                         foreach (string one_opt in compiler_options) {
 656                                 if (one_opt.StartsWith ("-doc:")) {
 657                                         doc_output = one_opt.Split (':', '/')[1];
 658                                 }
 659                         }
 660                         return true;
 661                 }
 662
 663                 class DomainTester : MarshalByRefObject
 664                 {
 665                         public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
 666                         {
 667                                 Assembly assembly = Assembly.LoadFile (file);
 668
 669                                 bool success = true;
 670                                 Type[] types = assembly.GetTypes ();
 671                                 foreach (Type t in types) {
 672
 673                                         // Skip interfaces
 674                                         if (!t.IsClass && !t.IsValueType)
 675                                                 continue;
 676
 677                                         if (test.VerificationProvider == null) {
 678                                                 if (!checker.UpdateVerificationDataFile)
 679                                                         checker.LogFileLine (test.FileName, "Missing IL verification data");
 680                                                 test.CreateNewTest ();
 681                                         }
 682
 683                                         foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 684                                                 MethodBase mi = m as MethodBase;
 685                                                 if (mi == null)
 686                                                         continue;
 687
 688                                                 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 689                                                         continue;
 690
 691                                                 success &= CompareIL (mi, test, checker);
 692                                         }
 693                                 }
 694
 695                                 return success;
 696                         }
 697
 698                         bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
 699                         {
 700                                 string m_name = mi.ToString ();
 701                                 string decl_type = mi.DeclaringType.ToString ();
 702                                 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
 703
 704                                 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
 705                                 if (md == null) {
 706                                         data_provider.AddNewMethod (mi, GetILSize (mi));
 707                                         if (!data_provider.IsNewSet) {
 708                                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 709                                                 return false;
 710                                         }
 711
 712                                         return true;
 713                                 }
 714
 715                                 if (md.Checked) {
 716                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 717                                         return false;
 718                                 }
 719
 720                                 md.Checked = true;
 721
 722                                 int il_size = GetILSize (mi);
 723                                 if (md.ILSize == il_size)
 724                                         return true;
 725
 726                                 if (md.ILSize > il_size) {
 727                                         checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size);
 728                                         md.ILSize = il_size;
 729                                         return true;
 730                                 }
 731
 732                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
 733                                         string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
 734
 735                                 md.ILSize = il_size;
 736
 737                                 return false;
 738                         }
 739
 740                         static int GetILSize (MethodBase mi)
 741                         {
 742                                 MethodBody body = mi.GetMethodBody ();
 743                                 if (body != null)
 744                                         return body.GetILAsByteArray ().Length;
 745
 746                                 return 0;
 747                         }
 748
 749                         bool ExecuteFile (MethodInfo entry_point, string filename)
 750                         {
 751                                 TextWriter stdout = Console.Out;
 752                                 TextWriter stderr = Console.Error;
 753                                 Console.SetOut (TextWriter.Null);
 754                                 Console.SetError (TextWriter.Null);
 755                                 ParameterInfo[] pi = entry_point.GetParameters ();
 756                                 object[] args = pi.Length == 0 ? null : default_args;
 757
 758                                 object result = null;
 759                                 try {
 760                                         try {
 761                                                 result = entry_point.Invoke (null, args);
 762                                         } finally {
 763                                                 Console.SetOut (stdout);
 764                                                 Console.SetError (stderr);
 765                                         }
 766                                 } catch (Exception e) {
 767                                         throw new ApplicationException (e.ToString ());
 768                                 }
 769
 770                                 if (result is int && (int) result != 0)
 771                                         throw new ApplicationException ("Wrong return code: " + result.ToString ());
 772
 773                                 return true;
 774                         }
 775
 776                         public bool Test (string file)
 777                         {
 778                                 Assembly assembly = Assembly.LoadFile (file);
 779                                 return ExecuteFile (assembly.EntryPoint, file);
 780                         }
 781                 }
 782
 783                 protected override bool Check(TestCase test)
 784                 {
 785                         string filename = test.FileName;
 786                         try {
 787                                 if (!base.Check (test)) {
 788                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 789                                         return false;
 790                                 }
 791                         }
 792                         catch (Exception e) {
 793                                 if (e.InnerException != null)
 794                                         e = e.InnerException;
 795
 796                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 797                                 return false;
 798                         }
 799
 800                         // Test setup
 801                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 802                                 if (verbose)
 803                                         LogFileLine (filename, "OK");
 804                                 --total;
 805                                 return true;
 806                         }
 807
 808                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 809
 810                         // Enable .dll only tests (no execution required)
 811                         if (!File.Exists(file)) {
 812                                 HandleFailure (filename, TestResult.Success, null);
 813                                 return true;
 814                         }
 815
 816                         AppDomain domain = null;
 817 #if !NET_2_1
 818                         if (safe_execution) {
 819                                 // Create a new AppDomain, with the current directory as the base.
 820                                 AppDomainSetup setupInfo = new AppDomainSetup ();
 821                                 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
 822                                 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
 823                                 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
 824                         }
 825 #endif
 826                         try {
 827                                 DomainTester tester;
 828                                 try {
 829 #if !NET_2_1
 830                                         if (domain != null)
 831                                                 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
 832                                         else
 833 #endif
 834                                                 tester = new DomainTester ();
 835
 836                                         if (!tester.Test (file))
 837                                                 return false;
 838
 839                                 } catch (ApplicationException e) {
 840                                         HandleFailure (filename, TestResult.ExecError, e.Message);
 841                                         return false;
 842                                 } catch (Exception e) {
 843                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 844                                         return false;
 845                                 }
 846
 847                                 if (doc_output != null) {
 848                                         string ref_file = filename.Replace (".cs", "-ref.xml");
 849                                         try {
 850 #if !NET_2_1
 851                                                 new XmlComparer ("doc").Compare (ref_file, doc_output);
 852 #endif
 853                                         } catch (Exception e) {
 854                                                 HandleFailure (filename, TestResult.XmlError, e.Message);
 855                                                 return false;
 856                                         }
 857                                 } else {
 858                                         if (verif_file != null) {
 859                                                 PositiveTestCase pt = (PositiveTestCase) test;
 860                                                 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
 861
 862                                                 if (!tester.CheckILSize (pt, this, file))
 863                                                         return false;
 864                                         }
 865
 866                                         if (filename.StartsWith ("test-debug", StringComparison.OrdinalIgnoreCase)) {
 867                                                 MonoSymbolFile mdb_file = MonoSymbolFile.ReadSymbolFile (file + ".mdb");
 868                                                 var mdb_xml_file = mdb_file.FileName + ".xml";
 869                                                 ConvertSymbolFileToXml (mdb_file, mdb_xml_file);
 870
 871                                                 var ref_file = filename.Replace(".cs", "-ref.xml");
 872                                                 try {
 873                                                         new XmlComparer ("symbols").Compare (ref_file, mdb_xml_file);
 874                                                 } catch (Exception e) {
 875                                                         HandleFailure (filename, TestResult.DebugError, e.Message);
 876                                                         return false;
 877                                                 }
 878                                         }
 879
 880                                 }
 881                         } finally {
 882                                 if (domain != null)
 883                                         AppDomain.Unload (domain);
 884                         }
 885
 886                         HandleFailure (filename, TestResult.Success, null);
 887                         return true;
 888                 }
 889
 890                 static void ConvertSymbolFileToXml (MonoSymbolFile symbolFile, string xmlFile)
 891                 {
 892                         using (XmlTextWriter writer = new XmlTextWriter (xmlFile, Encoding.UTF8)) {
 893                                 writer.Formatting = Formatting.Indented;
 894
 895                                 writer.WriteStartDocument ();
 896
 897                                 writer.WriteStartElement ("symbols");
 898
 899                                 writer.WriteStartElement ("files");
 900                                 foreach (var file in symbolFile.Sources) {
 901                                         writer.WriteStartElement ("file");
 902                                         writer.WriteAttributeString ("id", file.Index.ToString ());
 903                                         writer.WriteAttributeString ("name", Path.GetFileName (file.FileName));
 904                                         writer.WriteEndElement ();
 905                                 }
 906                                 writer.WriteEndElement ();
 907
 908                                 writer.WriteStartElement ("methods");
 909                                 foreach (var method in symbolFile.Methods) {
 910                                         writer.WriteStartElement ("method");
 911                                         writer.WriteAttributeString ("token", IntToHex (method.Token));
 912
 913                                         var il_entries = method.GetLineNumberTable ();
 914                                         writer.WriteStartElement ("sequencepoints");
 915                                         foreach (var entry in il_entries.LineNumbers) {
 916                                                 writer.WriteStartElement ("entry");
 917                                                 writer.WriteAttributeString ("il", IntToHex (entry.Offset));
 918                                                 writer.WriteAttributeString ("row", entry.Row.ToString ());
 919                                                 writer.WriteAttributeString ("file_ref", entry.File.ToString ());
 920                                                 writer.WriteAttributeString ("hidden", BoolToString (entry.IsHidden));
 921                                                 writer.WriteEndElement ();
 922                                         }
 923                                         writer.WriteEndElement ();
 924
 925                                         writer.WriteStartElement ("locals");
 926                                         foreach (var local in method.GetLocals ()) {
 927                                                 writer.WriteStartElement ("entry");
 928                                                 writer.WriteAttributeString ("name", local.Name);
 929                                                 writer.WriteAttributeString ("il_index", local.Index.ToString ());
 930                                                 writer.WriteAttributeString ("scope_ref", local.BlockIndex.ToString ());
 931                                                 writer.WriteEndElement ();
 932                                         }
 933                                         writer.WriteEndElement ();
 934
 935                                         writer.WriteStartElement ("scopes");
 936                                         foreach (var scope in method.GetCodeBlocks ()) {
 937                                                 writer.WriteStartElement ("entry");
 938                                                 writer.WriteAttributeString ("index", scope.Index.ToString ());
 939                                                 writer.WriteAttributeString ("start", IntToHex (scope.StartOffset));
 940                                                 writer.WriteAttributeString ("end", IntToHex (scope.EndOffset));
 941                                                 writer.WriteEndElement ();
 942                                         }
 943                                         writer.WriteEndElement ();
 944
 945                                         writer.WriteEndElement ();
 946                                 }
 947                                 writer.WriteEndElement ();
 948
 949                                 writer.WriteEndElement ();
 950                                 writer.WriteEndDocument ();
 951                         }
 952                 }
 953
 954                 static string IntToHex (int value)
 955                 {
 956                         return "0x" + value.ToString ("x", CultureInfo.InvariantCulture);
 957                 }
 958
 959                 static string BoolToString (bool value)
 960                 {
 961                         return value ? "true" : "false";
 962                 }
 963
 964                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
 965                 {
 966                         return new PositiveTestCase (filename, options, deps);
 967                 }
 968
 969                 public void HandleFailure (string file, TestResult status, string extra)
 970                 {
 971                         switch (status) {
 972                                 case TestResult.Success:
 973                                         success++;
 974                                         if (know_issues.Contains (file)) {
 975                                                 LogFileLine (file, "FIXED ISSUE");
 976                                                 return;
 977                                         }
 978                                         if (verbose)
 979                                                 LogFileLine (file, "OK");
 980                                         return;
 981
 982                                 case TestResult.CompileError:
 983                                         if (know_issues.Contains (file)) {
 984                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 985                                                 know_issues.Remove (file);
 986                                                 return;
 987                                         }
 988                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 989                                         break;
 990
 991                                 case TestResult.ExecError:
 992                                         if (know_issues.Contains (file)) {
 993                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 994                                                 know_issues.Remove (file);
 995                                                 return;
 996                                         }
 997                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 998                                         break;
 999
1000                                 case TestResult.XmlError:
1001                                         if (know_issues.Contains (file)) {
1002                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
1003                                                 know_issues.Remove (file);
1004                                                 return;
1005                                         }
1006                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
1007                                         break;
1008
1009                                 case TestResult.LoadError:
1010                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
1011                                         break;
1012
1013                                 case TestResult.ILError:
1014                                         if (!update_verif_file) {
1015                                                 LogFileLine (file, "IL REGRESSION: " + extra);
1016                                         }
1017                                         extra = null;
1018                                         break;
1019
1020                                 case TestResult.DebugError:
1021                                         LogFileLine (file, "REGRESSION (SUCCESS -> SYMBOL FILE ERROR)");
1022                                         break;
1023                         }
1024
1025                         if (extra != null)
1026                                 LogLine ("{0}", extra);
1027
1028                         if (!regression.Contains (file))
1029                                 regression.Add (file);
1030                 }
1031
1032                 public override void Initialize ()
1033                 {
1034                         if (verif_file != null) {
1035                                 LoadVerificationData (verif_file);
1036                         }
1037
1038                         base.Initialize ();
1039                 }
1040
1041                 public override void CleanUp ()
1042                 {
1043                         base.CleanUp ();
1044
1045                         if (update_verif_file) {
1046                                 UpdateVerificationData (verif_file);
1047                         }
1048                 }
1049
1050                 void LoadVerificationData (string file)
1051                 {
1052                         LogLine ("Loading verification data from `{0}' ...", file);
1053
1054                         using (XmlReader r = XmlReader.Create (file)) {
1055                                 r.ReadStartElement ("tests");
1056                                 verif_data = new Hashtable ();
1057
1058                                 while (r.Read ()) {
1059                                         if (r.Name != "test")
1060                                                 continue;
1061
1062                                         string name = r.GetAttribute ("name");
1063                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
1064                                         verif_data.Add (name, tc);
1065                                 }
1066                         }
1067                 }
1068
1069                 void UpdateVerificationData (string file)
1070                 {
1071                         LogLine ("Updating verification data `{0}' ...", file);
1072
1073                         XmlWriterSettings s = new XmlWriterSettings ();
1074                         s.Indent = true;
1075                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
1076                                 w.WriteStartDocument ();
1077                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
1078                                 w.WriteStartElement ("tests");
1079                                 foreach (PositiveTestCase tc in tests) {
1080                                         if (tc.VerificationProvider != null)
1081                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
1082                                 }
1083                                 w.WriteEndElement ();
1084                         }
1085                 }
1086         }
1087
1088         class NegativeChecker: Checker
1089         {
1090                 string expected_message;
1091                 string error_message;
1092                 bool check_msg;
1093                 bool check_error_line;
1094                 bool is_warning;
1095                 IDictionary wrong_warning;
1096
1097                 protected enum CompilerError {
1098                         Expected,
1099                         Wrong,
1100                         Missing,
1101                         WrongMessage,
1102                         MissingLocation,
1103                         Duplicate
1104                 }
1105
1106                 public NegativeChecker (ITester tester, bool check_msg):
1107                         base (tester)
1108                 {
1109                         this.check_msg = check_msg;
1110                         wrong_warning = new Hashtable ();
1111                 }
1112
1113                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1114                                                         ref string[] compiler_options,
1115                                                         ref string[] dependencies)
1116                 {
1117                         if (row == 1) {
1118                                 expected_message = null;
1119
1120                                 int index = line.IndexOf (':');
1121                                 if (index == -1 || index > 15) {
1122                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1123                                         ++syntax_errors;
1124                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1125                                                               ref dependencies);
1126                                         return false;
1127                                 }
1128
1129                                 expected_message = line.Substring (index + 1).Trim ();
1130                         }
1131
1132                         if (row == 2) {
1133                                 string filtered = line.Replace(" ", "");
1134
1135                                 // Some error tests require to have different error text for different runtimes.
1136                                 if (filtered.StartsWith ("//GMCS")) {
1137                                         row = 1;
1138                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1139                                 }
1140
1141                                 check_error_line = !filtered.StartsWith ("//Line:0");
1142
1143                                 if (!filtered.StartsWith ("//Line:")) {
1144                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1145                                         ++syntax_errors;
1146                                         return false;
1147                                 }
1148                         }
1149
1150                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1151                                 return false;
1152
1153                         is_warning = false;
1154                         if (compiler_options != null) {
1155                                 foreach (string s in compiler_options) {
1156                                         if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1157                                                 is_warning = true;
1158                                 }
1159                         }
1160                         return true;
1161                 }
1162
1163
1164                 protected override bool Check (TestCase test)
1165                 {
1166                         string filename = test.FileName;
1167
1168                         int start_char = 0;
1169                         while (Char.IsLetter (filename, start_char))
1170                                 ++start_char;
1171
1172                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1173                         string expected = filename.Substring (start_char, end_char - start_char);
1174
1175                         try {
1176                                 if (base.Check (test)) {
1177                                         HandleFailure (filename, CompilerError.Missing);
1178                                         return false;
1179                                 }
1180                         }
1181                         catch (Exception e) {
1182                                 HandleFailure (filename, CompilerError.Missing);
1183                                 if (e.InnerException != null)
1184                                         e = e.InnerException;
1185
1186                                 Log (e.ToString ());
1187                                 return false;
1188                         }
1189
1190                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1191                         if (tester.IsWarning (err_id)) {
1192                                 if (!is_warning)
1193                                         wrong_warning [err_id] = true;
1194                         } else {
1195                                 if (is_warning)
1196                                         wrong_warning [err_id] = false;
1197                         }
1198
1199                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1200                         if (HandleFailure (filename, result_code)) {
1201                                 success++;
1202                                 return true;
1203                         }
1204
1205                         if (result_code == CompilerError.Wrong)
1206                                 LogLine (tester.Output);
1207
1208                         return false;
1209                 }
1210
1211                 CompilerError GetCompilerError (string expected, string buffer)
1212                 {
1213                         const string error_prefix = "CS";
1214                         const string ignored_error = "error CS5001";
1215                         string tested_text = "error " + error_prefix + expected;
1216                         StringReader sr = new StringReader (buffer);
1217                         string line = sr.ReadLine ();
1218                         ArrayList ld = new ArrayList ();
1219                         CompilerError result = CompilerError.Missing;
1220                         while (line != null) {
1221                                 if (ld.Contains (line) && result == CompilerError.Expected) {
1222                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1223                                                 return CompilerError.Duplicate;
1224                                 }
1225                                 ld.Add (line);
1226
1227                                 if (result != CompilerError.Expected) {
1228                                         if (line.IndexOf (tested_text) != -1) {
1229                                                 if (check_msg) {
1230                                                         int first = line.IndexOf (':');
1231                                                         int second = line.IndexOf (':', first + 1);
1232                                                         if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
1233                                                                 if (check_error_line) {
1234                                                                         second = line.IndexOf (':', second + 1);
1235                                                                 }
1236                                                         } else if (second == -1 || !check_error_line) {
1237                                                                 second = first;
1238                                                         }
1239
1240                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1241                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1242                                                                 error_message = msg;
1243                                                                 return CompilerError.WrongMessage;
1244                                                         }
1245
1246                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1247                                                                 return CompilerError.MissingLocation;
1248                                                 }
1249                                                 result = CompilerError.Expected;
1250                                         } else if (line.IndexOf (error_prefix) != -1 &&
1251                                                 line.IndexOf (ignored_error) == -1)
1252                                                 result = CompilerError.Wrong;
1253                                 }
1254
1255                                 line = sr.ReadLine ();
1256                         }
1257
1258                         return result;
1259                 }
1260
1261                 bool HandleFailure (string file, CompilerError status)
1262                 {
1263                         switch (status) {
1264                                 case CompilerError.Expected:
1265                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1266                                                 LogFileLine (file, "FIXED ISSUE");
1267                                                 return true;
1268                                         }
1269
1270                                         if (verbose)
1271                                                 LogFileLine (file, "OK");
1272                                         return true;
1273
1274                                 case CompilerError.Wrong:
1275                                         if (know_issues.Contains (file)) {
1276                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1277                                                 know_issues.Remove (file);
1278                                                 return false;
1279                                         }
1280                                         if (no_error_list.Contains (file)) {
1281                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1282                                                 no_error_list.Remove (file);
1283                                         }
1284                                         else {
1285                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1286                                         }
1287                                         break;
1288
1289                                 case CompilerError.WrongMessage:
1290                                         if (know_issues.Contains (file)) {
1291                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1292                                                 know_issues.Remove (file);
1293                                                 return false;
1294                                         }
1295                                         if (no_error_list.Contains (file)) {
1296                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1297                                                 no_error_list.Remove (file);
1298                                         }
1299                                         else {
1300                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1301                                                 LogLine ("Exp: {0}", expected_message);
1302                                                 LogLine ("Was: {0}", error_message);
1303                                         }
1304                                         break;
1305
1306                                 case CompilerError.Missing:
1307                                         if (no_error_list.Contains (file)) {
1308                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1309                                                 no_error_list.Remove (file);
1310                                                 return false;
1311                                         }
1312
1313                                         if (know_issues.Contains (file)) {
1314                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1315                                                 know_issues.Remove (file);
1316                                         }
1317                                         else {
1318                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1319                                         }
1320
1321                                         break;
1322
1323                                 case CompilerError.MissingLocation:
1324                                         if (know_issues.Contains (file)) {
1325                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1326                                                 know_issues.Remove (file);
1327                                                 return false;
1328                                         }
1329                                         if (no_error_list.Contains (file)) {
1330                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1331                                                 no_error_list.Remove (file);
1332                                         }
1333                                         else {
1334                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1335                                         }
1336                                         break;
1337
1338                                 case CompilerError.Duplicate:
1339                                         // Will become an error soon
1340                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1341                                         return true;
1342                         }
1343
1344                         regression.Add (file);
1345                         return false;
1346                 }
1347
1348                 protected override void PrintSummary()
1349                 {
1350                         base.PrintSummary ();
1351
1352                         if (wrong_warning.Count > 0) {
1353                                 LogLine ("");
1354                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1355                                 LogLine ("");
1356                                 foreach (DictionaryEntry de in wrong_warning)
1357                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1358                         }
1359                 }
1360
1361         }
1362
1363         class Tester {
1364
1365                 static int Main(string[] args)
1366                 {
1367                         string temp;
1368
1369                         if (GetOption ("help", args, false, out temp)) {
1370                                 Usage ();
1371                                 return 0;
1372                         }
1373
1374                         string compiler;
1375                         if (!GetOption ("compiler", args, true, out compiler)) {
1376                                 Usage ();
1377                                 return 1;
1378                         }
1379
1380                         ITester tester;
1381                         try {
1382                                 Console.WriteLine ("Loading " + compiler + " ...");
1383                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1384                         }
1385                         catch (Exception) {
1386 #if NET_2_1
1387                                 throw;
1388 #else
1389                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1390                                 if (!File.Exists (compiler)) {
1391                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1392                                         return 1;
1393                                 }
1394                                 tester = new ProcessTester (compiler);
1395 #endif
1396                         }
1397
1398                         string mode;
1399                         if (!GetOption ("mode", args, true, out mode)) {
1400                                 Usage ();
1401                                 return 1;
1402                         }
1403
1404                         Checker checker;
1405                         bool positive;
1406                         switch (mode) {
1407                                 case "neg":
1408                                         checker = new NegativeChecker (tester, true);
1409                                         positive = false;
1410                                         break;
1411                                 case "pos":
1412                                         string iltest;
1413                                         GetOption ("il", args, false, out iltest);
1414                                         checker = new PositiveChecker (tester, iltest);
1415                                         positive = true;
1416
1417                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1418                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1419                                         }
1420
1421                                         break;
1422                                 default:
1423                                         Console.Error.WriteLine ("Invalid -mode argument");
1424                                         return 1;
1425                         }
1426
1427
1428                         if (GetOption ("issues", args, true, out temp))
1429                                 checker.IssueFile = temp;
1430                         if (GetOption ("log", args, true, out temp))
1431                                 checker.LogFile = temp;
1432                         if (GetOption ("verbose", args, false, out temp))
1433                                 checker.Verbose = true;
1434                         if (GetOption ("safe-execution", args, false, out temp))
1435                                 checker.SafeExecution = true;
1436                         if (GetOption ("compiler-options", args, true, out temp)) {
1437                                 string[] extra = temp.Split (' ');
1438                                 checker.ExtraCompilerOptions = extra;
1439                         }
1440
1441                         string test_pattern;
1442                         if (!GetOption ("files", args, true, out test_pattern)) {
1443                                 Usage ();
1444                                 return 1;
1445                         }
1446
1447                         var files = new List<string> ();
1448                         switch (test_pattern) {
1449                         case "v1":
1450                                 files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
1451                                 break;
1452                         case "v2":
1453                                 files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
1454                                 goto case "v1";
1455                         case "v4":
1456                                 files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
1457                                 goto case "v2";
1458                         default:
1459                                 files.AddRange (Directory.GetFiles (".", test_pattern));
1460                                 break;
1461                         }
1462
1463                         if (files.Count == 0) {
1464                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1465                                 return 2;
1466                         }
1467
1468                         checker.Initialize ();
1469 /*
1470                         files.Sort ((a, b) => {
1471                                 if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1472                                         if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
1473                                                 return -1;
1474                                 } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1475                                         if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
1476                                                 return 1;
1477                                 }
1478
1479                                 return a.CompareTo (b);
1480                         });
1481 */
1482                         foreach (string s in files) {
1483                                 string filename = Path.GetFileName (s);
1484                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1485                                         continue;
1486                                 }
1487
1488                                 if (filename.EndsWith ("-p2.cs"))
1489                                         continue;
1490
1491                                 checker.Do (filename);
1492                         }
1493
1494                         checker.CleanUp ();
1495
1496                         checker.Dispose ();
1497
1498                         return checker.ResultCode;
1499                 }
1500
1501                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1502                 {
1503                         opt = "-" + opt;
1504                         foreach (string a in args) {
1505                                 if (a.StartsWith (opt)) {
1506                                         int sep = a.IndexOf (':');
1507                                         if (sep > 0) {
1508                                                 value = a.Substring (sep + 1);
1509                                         } else {
1510                                                 value = null;
1511                                                 if (req_arg) {
1512                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1513                                                         return false;
1514                                                 }
1515                                         }
1516
1517                                         return true;
1518                                 }
1519                         }
1520
1521                         value = null;
1522                         return false;
1523                 }
1524
1525                 static void Usage ()
1526                 {
1527                         Console.WriteLine (
1528                                 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1529                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1530                                 "   \n" +
1531                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1532                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1533                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1534                                 "   -issues:FILE     The list of expected failures\n" +
1535                                 "   -log:FILE        Writes any output also to the file\n" +
1536                                 "   -help            Lists all options\n" +
1537                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1538                                 "   -safe-execution  Runs compiled executables in separate app-domain\n" +
1539                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1540                                 "   -verbose         Prints more details during testing\n"
1541                                 );
1542                 }
1543         }
1544 }