mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7
   8 //
   9 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
  10 //
  11 // Permission is hereby granted, free of charge, to any person obtaining
  12 // a copy of this software and associated documentation files (the
  13 // "Software"), to deal in the Software without restriction, including
  14 // without limitation the rights to use, copy, modify, merge, publish,
  15 // distribute, sublicense, and/or sell copies of the Software, and to
  16 // permit persons to whom the Software is furnished to do so, subject to
  17 // the following conditions:
  18 //
  19 // The above copyright notice and this permission notice shall be
  20 // included in all copies or substantial portions of the Software.
  21 //
  22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29 //
  30
  31 using System;
  32 using System.IO;
  33 using System.Diagnostics;
  34 using System.Reflection;
  35 using System.Text;
  36 using System.Collections;
  37 using System.Xml;
  38 using System.Collections.Generic;
  39
  40 namespace TestRunner {
  41
  42         interface ITester
  43         {
  44                 string Output { get; }
  45                 bool Invoke (string[] args);
  46                 bool IsWarning (int warningNumber);
  47         }
  48
  49         class ReflectionTester: ITester {
  50                 MethodInfo ep;
  51                 object[] method_arg;
  52                 StringWriter output;
  53                 int[] all_warnings;
  54
  55                 public ReflectionTester (Assembly a)
  56                 {
  57                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  58
  59                         if (t == null)
  60                                 Console.Error.WriteLine ("null, huh?");
  61
  62                         ep = t.GetMethod ("InvokeCompiler",
  63                                 BindingFlags.Static | BindingFlags.Public);
  64                         if (ep == null)
  65                                 throw new MissingMethodException ("static InvokeCompiler");
  66                         method_arg = new object [2];
  67
  68                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  69                         all_warnings = (int[])pi.GetValue (null, null);
  70                         Array.Sort (all_warnings);
  71                 }
  72
  73                 public string Output {
  74                         get {
  75                                 return output.GetStringBuilder ().ToString ();
  76                         }
  77                 }
  78
  79                 public bool Invoke(string[] args)
  80                 {
  81                         output = new StringWriter ();
  82                         method_arg [0] = args;
  83                         method_arg [1] = output;
  84                         return (bool)ep.Invoke (null, method_arg);
  85                 }
  86
  87                 public bool IsWarning (int warningNumber)
  88                 {
  89                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  90                 }
  91         }
  92
  93 #if !NET_2_1
  94         class ProcessTester: ITester
  95         {
  96                 ProcessStartInfo pi;
  97                 string output;
  98
  99                 public ProcessTester (string p_path)
 100                 {
 101                         pi = new ProcessStartInfo ();
 102                         pi.FileName = p_path;
 103                         pi.CreateNoWindow = true;
 104                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 105                         pi.RedirectStandardOutput = true;
 106                         pi.RedirectStandardError = true;
 107                         pi.UseShellExecute = false;
 108                 }
 109
 110                 public string Output {
 111                         get {
 112                                 return output;
 113                         }
 114                 }
 115
 116                 public bool Invoke(string[] args)
 117                 {
 118                         StringBuilder sb = new StringBuilder ("/nologo ");
 119                         foreach (string s in args) {
 120                                 sb.Append (s);
 121                                 sb.Append (" ");
 122                         }
 123                         pi.Arguments = sb.ToString ();
 124                         Process p = Process.Start (pi);
 125                         output = p.StandardError.ReadToEnd ();
 126                         if (output.Length == 0)
 127                             output = p.StandardOutput.ReadToEnd ();
 128                         p.WaitForExit ();
 129                         return p.ExitCode == 0;
 130                 }
 131
 132                 public bool IsWarning (int warningNumber)
 133                 {
 134                         throw new NotImplementedException ();
 135                 }
 136         }
 137 #endif
 138
 139         class TestCase : MarshalByRefObject
 140         {
 141                 public readonly string FileName;
 142                 public readonly string[] CompilerOptions;
 143                 public readonly string[] Dependencies;
 144
 145                 public TestCase (string filename, string[] options, string[] deps)
 146                 {
 147                         this.FileName = filename;
 148                         this.CompilerOptions = options;
 149                         this.Dependencies = deps;
 150                 }
 151         }
 152
 153         class PositiveTestCase : TestCase
 154         {
 155                 public class VerificationData : MarshalByRefObject
 156                 {
 157                         public class MethodData : MarshalByRefObject
 158                         {
 159                                 public MethodData (MethodBase mi, int il_size)
 160                                 {
 161                                         this.Type = mi.DeclaringType.ToString ();
 162                                         this.MethodName = mi.ToString ();
 163                                         this.ILSize = il_size;
 164                                 }
 165
 166                                 public MethodData (string type_name, string method_name, int il_size)
 167                                 {
 168                                         this.Type = type_name;
 169                                         this.MethodName = method_name;
 170                                         this.ILSize = il_size;
 171                                 }
 172
 173                                 public string Type;
 174                                 public string MethodName;
 175                                 public int ILSize;
 176                                 public bool Checked;
 177                         }
 178
 179                         ArrayList methods;
 180                         public bool IsNewSet;
 181
 182                         public VerificationData (string test_file)
 183                         {
 184 #if NET_2_0
 185                                 this.test_file = test_file;
 186 #endif
 187                         }
 188
 189 #if NET_2_0
 190                         string test_file;
 191
 192                         public static VerificationData FromFile (string name, XmlReader r)
 193                         {
 194                                 VerificationData tc = new VerificationData (name);
 195                                 ArrayList methods = new ArrayList ();
 196                                 r.Read ();
 197                                 while (r.ReadToNextSibling ("type")) {
 198                                         string type_name = r ["name"];
 199                                         r.Read ();
 200                                         while (r.ReadToNextSibling ("method")) {
 201                                                 string m_name = r ["name"];
 202
 203                                                 r.ReadToDescendant ("size");
 204                                                 int il_size = r.ReadElementContentAsInt ();
 205                                                 methods.Add (new MethodData (type_name, m_name, il_size));
 206                                                 r.Read ();
 207                                         }
 208                                         r.Read ();
 209                                 }
 210
 211                                 tc.methods = methods;
 212                                 return tc;
 213                         }
 214
 215                         public void WriteCodeInfoTo (XmlWriter w)
 216                         {
 217                                 w.WriteStartElement ("test");
 218                                 w.WriteAttributeString ("name", test_file);
 219
 220                                 string type = null;
 221                                 foreach (MethodData data in methods) {
 222                                         if (!data.Checked)
 223                                                 continue;
 224
 225                                         if (type != data.Type) {
 226                                                 if (type != null)
 227                                                         w.WriteEndElement ();
 228
 229                                                 type = data.Type;
 230                                                 w.WriteStartElement ("type");
 231                                                 w.WriteAttributeString ("name", type);
 232                                         }
 233
 234                                         w.WriteStartElement ("method");
 235                                         w.WriteAttributeString ("name", data.MethodName);
 236                                         w.WriteStartElement ("size");
 237                                         w.WriteValue (data.ILSize);
 238                                         w.WriteEndElement ();
 239                                         w.WriteEndElement ();
 240                                 }
 241
 242                                 if (type != null)
 243                                         w.WriteEndElement ();
 244
 245                                 w.WriteEndElement ();
 246                         }
 247 #endif
 248
 249                         public MethodData FindMethodData (string method_name, string declaring_type)
 250                         {
 251                                 if (methods == null)
 252                                         return null;
 253
 254                                 foreach (MethodData md in methods) {
 255                                         if (md.MethodName == method_name && md.Type == declaring_type)
 256                                                 return md;
 257                                 }
 258
 259                                 return null;
 260                         }
 261
 262                         public void AddNewMethod (MethodBase mb, int il_size)
 263                         {
 264                                 if (methods == null)
 265                                         methods = new ArrayList ();
 266
 267                                 MethodData md = new MethodData (mb, il_size);
 268                                 md.Checked = true;
 269                                 methods.Add (md);
 270                         }
 271                 }
 272
 273                 VerificationData verif_data;
 274
 275                 public PositiveTestCase (string filename, string [] options, string [] deps)
 276                         : base (filename, options, deps)
 277                 {
 278                 }
 279
 280                 public void CreateNewTest ()
 281                 {
 282                         verif_data = new VerificationData (FileName);
 283                         verif_data.IsNewSet = true;
 284                 }
 285
 286                 public VerificationData VerificationProvider {
 287                         set {
 288                                 verif_data = value;
 289                         }
 290                         get {
 291                                 return verif_data;
 292                         }
 293                 }
 294         }
 295
 296         class Checker: MarshalByRefObject, IDisposable
 297         {
 298                 protected ITester tester;
 299                 protected int success;
 300                 protected int total;
 301                 protected int ignored;
 302                 protected int syntax_errors;
 303                 string issue_file;
 304                 StreamWriter log_file;
 305                 protected string[] extra_compiler_options;
 306                 // protected string[] compiler_options;
 307                 // protected string[] dependencies;
 308
 309                 protected ArrayList tests = new ArrayList ();
 310                 protected Hashtable test_hash = new Hashtable ();
 311                 protected ArrayList regression = new ArrayList ();
 312                 protected ArrayList know_issues = new ArrayList ();
 313                 protected ArrayList ignore_list = new ArrayList ();
 314                 protected ArrayList no_error_list = new ArrayList ();
 315
 316                 protected bool verbose;
 317                 protected bool safe_execution;
 318
 319                 int total_known_issues;
 320
 321                 protected Checker (ITester tester)
 322                 {
 323                         this.tester = tester;
 324                 }
 325
 326                 public string IssueFile {
 327                         set {
 328                                 this.issue_file = value;
 329                                 ReadWrongErrors (issue_file);
 330                         }
 331                 }
 332
 333                 public string LogFile {
 334                         set {
 335                                 this.log_file = new StreamWriter (value, false);
 336                         }
 337                 }
 338
 339                 public bool Verbose {
 340                         set {
 341                                 verbose = value;
 342                         }
 343                 }
 344
 345                 public bool SafeExecution {
 346                         set {
 347                                 safe_execution = value;
 348                         }
 349                 }
 350
 351                 public string[] ExtraCompilerOptions {
 352                         set {
 353                                 extra_compiler_options = value;
 354                         }
 355                 }
 356
 357                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 358                                                         out string[] dependencies)
 359                 {
 360                         int row = 0;
 361                         compiler_options = null;
 362                         dependencies = null;
 363                         try {
 364                                 using (StreamReader sr = new StreamReader (file)) {
 365                                         String line;
 366                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 367                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 368                                                                       ref dependencies))
 369                                                         return false;
 370                                         }
 371                                 }
 372                         } catch {
 373                                 return false;
 374                         }
 375                         return true;
 376                 }
 377
 378                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 379                                                         ref string[] compiler_options,
 380                                                         ref string[] dependencies)
 381                 {
 382                         const string options = "// Compiler options:";
 383                         const string depends = "// Dependencies:";
 384
 385                         if (row == 1) {
 386                                 compiler_options = null;
 387                                 dependencies = null;
 388                         }
 389
 390                         int index = line.IndexOf (options);
 391                         if (index != -1) {
 392                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 393                                 for (int i = 0; i < compiler_options.Length; i++)
 394                                         compiler_options[i] = compiler_options[i].TrimStart ();
 395                         }
 396                         index = line.IndexOf (depends);
 397                         if (index != -1) {
 398                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 399                                 for (int i = 0; i < dependencies.Length; i++)
 400                                         dependencies[i] = dependencies[i].TrimStart ();
 401                         }
 402
 403                         return true;
 404                 }
 405
 406                 public bool Do (string filename)
 407                 {
 408                         if (test_hash.Contains (filename))
 409                                 return true;
 410
 411                         if (verbose)
 412                                 Log (filename + "...\t");
 413
 414                         if (ignore_list.Contains (filename)) {
 415                                 ++ignored;
 416                                 LogFileLine (filename, "NOT TESTED");
 417                                 return false;
 418                         }
 419
 420                         string[] compiler_options, dependencies;
 421                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 422                                 LogFileLine (filename, "ERROR");
 423                                 return false;
 424                         }
 425
 426                         if (extra_compiler_options != null) {
 427                                 if (compiler_options == null)
 428                                         compiler_options = extra_compiler_options;
 429                                 else {
 430                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 431                                         extra_compiler_options.CopyTo (new_options, 0);
 432                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 433                                         compiler_options = new_options;
 434                                 }
 435                         }
 436
 437                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 438                         test_hash.Add (filename, test);
 439
 440                         ++total;
 441                         if (dependencies != null) {
 442                                 foreach (string dependency in dependencies) {
 443                                         if (!Do (dependency)) {
 444                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 445                                                 return false;
 446                                         }
 447                                 }
 448                         }
 449
 450                         tests.Add (test);
 451
 452                         return Check (test);
 453                 }
 454
 455                 protected virtual bool Check (TestCase test)
 456                 {
 457                         string[] test_args;
 458
 459                         if (test.CompilerOptions != null) {
 460                                 test_args = new string [2 + test.CompilerOptions.Length];
 461                                 test.CompilerOptions.CopyTo (test_args, 0);
 462                         } else {
 463                                 test_args = new string [2];
 464                         }
 465                         test_args [test_args.Length - 2] = test.FileName;
 466                         test_args [test_args.Length - 1] = "-debug";
 467
 468                         return tester.Invoke (test_args);
 469                 }
 470
 471                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 472                 {
 473                         return new TestCase (filename, options, deps);
 474                 }
 475
 476                 void ReadWrongErrors (string file)
 477                 {
 478                         const string ignored = "IGNORE";
 479                         const string no_error = "NO ERROR";
 480
 481                         using (StreamReader sr = new StreamReader (file)) {
 482                                 string line;
 483                                 while ((line = sr.ReadLine()) != null) {
 484                                         if (line.StartsWith ("#"))
 485                                                 continue;
 486
 487                                         ArrayList active_cont = know_issues;
 488
 489                                         if (line.IndexOf (ignored) > 0)
 490                                                 active_cont = ignore_list;
 491                                         else if (line.IndexOf (no_error) > 0)
 492                                                 active_cont = no_error_list;
 493
 494                                         string file_name = line.Split (' ')[0];
 495                                         if (file_name.Length == 0)
 496                                                 continue;
 497
 498                                         active_cont.Add (file_name);
 499                                 }
 500                         }
 501                         total_known_issues = know_issues.Count;
 502                 }
 503
 504                 protected virtual void PrintSummary ()
 505                 {
 506                         LogLine ("Done" + Environment.NewLine);
 507                         float rate = 0;
 508                         if (total > 0)
 509                                 rate = (float) (success) / (float)total;
 510                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 511
 512                         if (syntax_errors > 0)
 513                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 514
 515                         if (ignored > 0)
 516                                 LogLine ("{0} test(s) ignored", ignored);
 517
 518                         if (total_known_issues - know_issues.Count > 0)
 519                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 520
 521                         know_issues.AddRange (no_error_list);
 522                         if (know_issues.Count > 0) {
 523                                 LogLine ("");
 524                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 525                                 foreach (string s in know_issues)
 526                                         LogLine (s);
 527                         }
 528                         if (regression.Count > 0) {
 529                                 LogLine ("");
 530                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 531                                 foreach (string s in regression)
 532                                         LogLine (s);
 533                         }
 534                 }
 535
 536                 public int ResultCode
 537                 {
 538                         get {
 539                                 return regression.Count == 0 ? 0 : 1;
 540                         }
 541                 }
 542
 543                 protected void Log (string msg, params object [] rest)
 544                 {
 545                         Console.Write (msg, rest);
 546                         if (log_file != null)
 547                                 log_file.Write (msg, rest);
 548                 }
 549
 550                 protected void LogLine (string msg)
 551                 {
 552                         Console.WriteLine (msg);
 553                         if (log_file != null)
 554                                 log_file.WriteLine (msg);
 555                 }
 556
 557                 protected void LogLine (string msg, params object [] rest)
 558                 {
 559                         Console.WriteLine (msg, rest);
 560                         if (log_file != null)
 561                                 log_file.WriteLine (msg, rest);
 562                 }
 563
 564                 public void LogFileLine (string file, string msg, params object [] args)
 565                 {
 566                         string s = verbose ?
 567                                 string.Format (msg, args) :
 568                                 file + "...\t" + string.Format (msg, args);
 569
 570                         Console.WriteLine (s);
 571                         if (log_file != null)
 572                                 log_file.WriteLine (s);
 573                 }
 574
 575                 #region IDisposable Members
 576
 577                 public void Dispose()
 578                 {
 579                         if (log_file != null)
 580                                 log_file.Close ();
 581                 }
 582
 583                 #endregion
 584
 585                 public virtual void Initialize ()
 586                 {
 587                 }
 588
 589                 public virtual void CleanUp ()
 590                 {
 591                         PrintSummary ();
 592                 }
 593         }
 594
 595         class PositiveChecker: Checker
 596         {
 597                 readonly string files_folder;
 598                 readonly static object[] default_args = new object[1] { new string[] {} };
 599                 string doc_output;
 600                 string verif_file;
 601                 bool update_verif_file;
 602                 Hashtable verif_data;
 603
 604 #if !NET_2_1
 605                 ProcessStartInfo pi;
 606 #endif
 607                 readonly string mono;
 608
 609                 public enum TestResult {
 610                         CompileError,
 611                         ExecError,
 612                         LoadError,
 613                         XmlError,
 614                         Success,
 615                         ILError
 616                 }
 617
 618                 public PositiveChecker (ITester tester, string verif_file):
 619                         base (tester)
 620                 {
 621                         files_folder = Directory.GetCurrentDirectory ();
 622                         this.verif_file = verif_file;
 623
 624 #if !NET_2_1
 625                         pi = new ProcessStartInfo ();
 626                         pi.CreateNoWindow = true;
 627                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 628                         pi.RedirectStandardOutput = true;
 629                         pi.RedirectStandardError = true;
 630                         pi.UseShellExecute = false;
 631
 632                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 633                         if (mono != null) {
 634                                 pi.FileName = mono;
 635                         }
 636 #endif
 637                 }
 638
 639                 public bool UpdateVerificationDataFile {
 640                         set {
 641                                 update_verif_file = value;
 642                         }
 643                         get {
 644                                 return update_verif_file;
 645                         }
 646                 }
 647
 648                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 649                                                         out string[] dependencies) {
 650                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 651                                 return false;
 652
 653                         doc_output = null;
 654                         if (compiler_options == null)
 655                                 return true;
 656
 657                         foreach (string one_opt in compiler_options) {
 658                                 if (one_opt.StartsWith ("-doc:")) {
 659                                         doc_output = one_opt.Split (':', '/')[1];
 660                                 }
 661                         }
 662                         return true;
 663                 }
 664
 665                 class DomainTester : MarshalByRefObject
 666                 {
 667                         public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
 668                         {
 669                                 Assembly assembly = Assembly.LoadFile (file);
 670
 671                                 bool success = true;
 672                                 Type[] types = assembly.GetTypes ();
 673                                 foreach (Type t in types) {
 674
 675                                         // Skip interfaces
 676                                         if (!t.IsClass && !t.IsValueType)
 677                                                 continue;
 678
 679                                         if (test.VerificationProvider == null) {
 680                                                 if (!checker.UpdateVerificationDataFile)
 681                                                         checker.LogFileLine (test.FileName, "Missing IL verification data");
 682                                                 test.CreateNewTest ();
 683                                         }
 684
 685                                         foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 686                                                 MethodBase mi = m as MethodBase;
 687                                                 if (mi == null)
 688                                                         continue;
 689
 690                                                 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 691                                                         continue;
 692
 693                                                 success &= CompareIL (mi, test, checker);
 694                                         }
 695                                 }
 696
 697                                 return success;
 698                         }
 699
 700                         bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
 701                         {
 702                                 string m_name = mi.ToString ();
 703                                 string decl_type = mi.DeclaringType.ToString ();
 704                                 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
 705
 706                                 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
 707                                 if (md == null) {
 708                                         data_provider.AddNewMethod (mi, GetILSize (mi));
 709                                         if (!data_provider.IsNewSet) {
 710                                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 711                                                 return false;
 712                                         }
 713
 714                                         return true;
 715                                 }
 716
 717                                 if (md.Checked) {
 718                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 719                                         return false;
 720                                 }
 721
 722                                 md.Checked = true;
 723
 724                                 int il_size = GetILSize (mi);
 725                                 if (md.ILSize == il_size)
 726                                         return true;
 727
 728                                 if (md.ILSize > il_size) {
 729                                         checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size);
 730                                         md.ILSize = il_size;
 731                                         return true;
 732                                 }
 733
 734                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
 735                                         string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
 736
 737                                 md.ILSize = il_size;
 738
 739                                 return false;
 740                         }
 741
 742                         static int GetILSize (MethodBase mi)
 743                         {
 744 #if NET_2_0
 745                                 MethodBody body = mi.GetMethodBody ();
 746                                 if (body != null)
 747                                         return body.GetILAsByteArray ().Length;
 748 #endif
 749                                 return 0;
 750                         }
 751
 752                         bool ExecuteFile (MethodInfo entry_point, string filename)
 753                         {
 754                                 TextWriter stdout = Console.Out;
 755                                 TextWriter stderr = Console.Error;
 756                                 Console.SetOut (TextWriter.Null);
 757                                 Console.SetError (TextWriter.Null);
 758                                 ParameterInfo[] pi = entry_point.GetParameters ();
 759                                 object[] args = pi.Length == 0 ? null : default_args;
 760
 761                                 object result = null;
 762                                 try {
 763                                         try {
 764                                                 result = entry_point.Invoke (null, args);
 765                                         } finally {
 766                                                 Console.SetOut (stdout);
 767                                                 Console.SetError (stderr);
 768                                         }
 769                                 } catch (Exception e) {
 770                                         throw new ApplicationException (e.ToString ());
 771                                 }
 772
 773                                 if (result is int && (int) result != 0)
 774                                         throw new ApplicationException ("Wrong return code: " + result.ToString ());
 775
 776                                 return true;
 777                         }
 778
 779                         public bool Test (string file)
 780                         {
 781                                 Assembly assembly = Assembly.LoadFile (file);
 782                                 return ExecuteFile (assembly.EntryPoint, file);
 783                         }
 784                 }
 785
 786                 protected override bool Check(TestCase test)
 787                 {
 788                         string filename = test.FileName;
 789                         try {
 790                                 if (!base.Check (test)) {
 791                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 792                                         return false;
 793                                 }
 794                         }
 795                         catch (Exception e) {
 796                                 if (e.InnerException != null)
 797                                         e = e.InnerException;
 798
 799                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 800                                 return false;
 801                         }
 802
 803                         // Test setup
 804                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 805                                 if (verbose)
 806                                         LogFileLine (filename, "OK");
 807                                 --total;
 808                                 return true;
 809                         }
 810
 811                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 812
 813                         // Enable .dll only tests (no execution required)
 814                         if (!File.Exists(file)) {
 815                                 HandleFailure (filename, TestResult.Success, null);
 816                                 return true;
 817                         }
 818
 819                         AppDomain domain = null;
 820 #if !NET_2_1
 821                         if (safe_execution) {
 822                                 // Create a new AppDomain, with the current directory as the base.
 823                                 AppDomainSetup setupInfo = new AppDomainSetup ();
 824                                 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
 825                                 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
 826                                 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
 827                         }
 828 #endif
 829                         try {
 830                                 DomainTester tester;
 831                                 try {
 832 #if !NET_2_1
 833                                         if (domain != null)
 834                                                 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
 835                                         else
 836 #endif
 837                                                 tester = new DomainTester ();
 838
 839                                         if (!tester.Test (file))
 840                                                 return false;
 841
 842                                 } catch (ApplicationException e) {
 843                                         HandleFailure (filename, TestResult.ExecError, e.Message);
 844                                         return false;
 845                                 } catch (Exception e) {
 846                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 847                                         return false;
 848                                 }
 849
 850                                 if (doc_output != null) {
 851                                         string ref_file = filename.Replace (".cs", "-ref.xml");
 852                                         try {
 853 #if !NET_2_1
 854                                                 XmlComparer.Compare (ref_file, doc_output);
 855 #endif
 856                                         } catch (Exception e) {
 857                                                 HandleFailure (filename, TestResult.XmlError, e.Message);
 858                                                 return false;
 859                                         }
 860                                 } else {
 861                                         if (verif_file != null) {
 862                                                 PositiveTestCase pt = (PositiveTestCase) test;
 863                                                 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
 864
 865                                                 if (!tester.CheckILSize (pt, this, file))
 866                                                         return false;
 867                                         }
 868                                 }
 869                         } finally {
 870                                 if (domain != null)
 871                                         AppDomain.Unload (domain);
 872                         }
 873
 874                         HandleFailure (filename, TestResult.Success, null);
 875                         return true;
 876                 }
 877
 878                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
 879                 {
 880                         return new PositiveTestCase (filename, options, deps);
 881                 }
 882
 883                 public void HandleFailure (string file, TestResult status, string extra)
 884                 {
 885                         switch (status) {
 886                                 case TestResult.Success:
 887                                         success++;
 888                                         if (know_issues.Contains (file)) {
 889                                                 LogFileLine (file, "FIXED ISSUE");
 890                                                 return;
 891                                         }
 892                                         if (verbose)
 893                                                 LogFileLine (file, "OK");
 894                                         return;
 895
 896                                 case TestResult.CompileError:
 897                                         if (know_issues.Contains (file)) {
 898                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 899                                                 know_issues.Remove (file);
 900                                                 return;
 901                                         }
 902                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 903                                         break;
 904
 905                                 case TestResult.ExecError:
 906                                         if (know_issues.Contains (file)) {
 907                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 908                                                 know_issues.Remove (file);
 909                                                 return;
 910                                         }
 911                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 912                                         break;
 913
 914                                 case TestResult.XmlError:
 915                                         if (know_issues.Contains (file)) {
 916                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
 917                                                 know_issues.Remove (file);
 918                                                 return;
 919                                         }
 920                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 921                                         break;
 922
 923                                 case TestResult.LoadError:
 924                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
 925                                         break;
 926
 927                                 case TestResult.ILError:
 928                                         if (!update_verif_file) {
 929                                                 LogFileLine (file, "IL REGRESSION: " + extra);
 930                                         }
 931                                         extra = null;
 932                                         break;
 933                         }
 934
 935                         if (extra != null)
 936                                 LogLine ("{0}", extra);
 937
 938                         if (!regression.Contains (file))
 939                                 regression.Add (file);
 940                 }
 941
 942                 public override void Initialize ()
 943                 {
 944                         if (verif_file != null) {
 945 #if NET_2_0
 946                                 LoadVerificationData (verif_file);
 947 #else
 948                                 throw new NotSupportedException ();
 949 #endif
 950                         }
 951
 952                         base.Initialize ();
 953                 }
 954
 955                 public override void CleanUp ()
 956                 {
 957                         base.CleanUp ();
 958
 959                         if (update_verif_file) {
 960 #if NET_2_0
 961                                 UpdateVerificationData (verif_file);
 962 #else
 963                                 throw new NotSupportedException ();
 964 #endif
 965                         }
 966                 }
 967
 968 #if NET_2_0
 969                 void LoadVerificationData (string file)
 970                 {
 971                         LogLine ("Loading verification data from `{0}' ...", file);
 972
 973                         using (XmlReader r = XmlReader.Create (file)) {
 974                                 r.ReadStartElement ("tests");
 975                                 verif_data = new Hashtable ();
 976
 977                                 while (r.Read ()) {
 978                                         if (r.Name != "test")
 979                                                 continue;
 980
 981                                         string name = r.GetAttribute ("name");
 982                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
 983                                         verif_data.Add (name, tc);
 984                                 }
 985                         }
 986                 }
 987
 988                 void UpdateVerificationData (string file)
 989                 {
 990                         LogLine ("Updating verification data `{0}' ...", file);
 991
 992                         XmlWriterSettings s = new XmlWriterSettings ();
 993                         s.Indent = true;
 994                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
 995                                 w.WriteStartDocument ();
 996                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
 997                                 w.WriteStartElement ("tests");
 998                                 foreach (PositiveTestCase tc in tests) {
 999                                         if (tc.VerificationProvider != null)
1000                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
1001                                 }
1002                                 w.WriteEndElement ();
1003                         }
1004                 }
1005 #endif
1006         }
1007
1008         class NegativeChecker: Checker
1009         {
1010                 string expected_message;
1011                 string error_message;
1012                 bool check_msg;
1013                 bool check_error_line;
1014                 bool is_warning;
1015                 IDictionary wrong_warning;
1016
1017                 protected enum CompilerError {
1018                         Expected,
1019                         Wrong,
1020                         Missing,
1021                         WrongMessage,
1022                         MissingLocation,
1023                         Duplicate
1024                 }
1025
1026                 public NegativeChecker (ITester tester, bool check_msg):
1027                         base (tester)
1028                 {
1029                         this.check_msg = check_msg;
1030                         wrong_warning = new Hashtable ();
1031                 }
1032
1033                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1034                                                         ref string[] compiler_options,
1035                                                         ref string[] dependencies)
1036                 {
1037                         if (row == 1) {
1038                                 expected_message = null;
1039
1040                                 int index = line.IndexOf (':');
1041                                 if (index == -1 || index > 15) {
1042                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1043                                         ++syntax_errors;
1044                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1045                                                               ref dependencies);
1046                                         return false;
1047                                 }
1048
1049                                 expected_message = line.Substring (index + 1).Trim ();
1050                         }
1051
1052                         if (row == 2) {
1053                                 string filtered = line.Replace(" ", "");
1054
1055                                 // Some error tests require to have different error text for different runtimes.
1056                                 if (filtered.StartsWith ("//GMCS")) {
1057                                         row = 1;
1058 #if !NET_2_0
1059                                         return true;
1060 #else
1061                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1062 #endif
1063                                 }
1064
1065                                 check_error_line = !filtered.StartsWith ("//Line:0");
1066
1067                                 if (!filtered.StartsWith ("//Line:")) {
1068                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1069                                         ++syntax_errors;
1070                                         return false;
1071                                 }
1072                         }
1073
1074                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1075                                 return false;
1076
1077                         is_warning = false;
1078                         if (compiler_options != null) {
1079                                 foreach (string s in compiler_options) {
1080                                         if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1081                                                 is_warning = true;
1082                                 }
1083                         }
1084                         return true;
1085                 }
1086
1087
1088                 protected override bool Check (TestCase test)
1089                 {
1090                         string filename = test.FileName;
1091
1092                         int start_char = 0;
1093                         while (Char.IsLetter (filename, start_char))
1094                                 ++start_char;
1095
1096                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1097                         string expected = filename.Substring (start_char, end_char - start_char);
1098
1099                         try {
1100                                 if (base.Check (test)) {
1101                                         HandleFailure (filename, CompilerError.Missing);
1102                                         return false;
1103                                 }
1104                         }
1105                         catch (Exception e) {
1106                                 HandleFailure (filename, CompilerError.Missing);
1107                                 if (e.InnerException != null)
1108                                         e = e.InnerException;
1109
1110                                 Log (e.ToString ());
1111                                 return false;
1112                         }
1113
1114                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1115                         if (tester.IsWarning (err_id)) {
1116                                 if (!is_warning)
1117                                         wrong_warning [err_id] = true;
1118                         } else {
1119                                 if (is_warning)
1120                                         wrong_warning [err_id] = false;
1121                         }
1122
1123                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1124                         if (HandleFailure (filename, result_code)) {
1125                                 success++;
1126                                 return true;
1127                         }
1128
1129                         if (result_code == CompilerError.Wrong)
1130                                 LogLine (tester.Output);
1131
1132                         return false;
1133                 }
1134
1135                 CompilerError GetCompilerError (string expected, string buffer)
1136                 {
1137                         const string error_prefix = "CS";
1138                         const string ignored_error = "error CS5001";
1139                         string tested_text = "error " + error_prefix + expected;
1140                         StringReader sr = new StringReader (buffer);
1141                         string line = sr.ReadLine ();
1142                         ArrayList ld = new ArrayList ();
1143                         CompilerError result = CompilerError.Missing;
1144                         while (line != null) {
1145                                 if (ld.Contains (line) && result == CompilerError.Expected) {
1146                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1147                                                 return CompilerError.Duplicate;
1148                                 }
1149                                 ld.Add (line);
1150
1151                                 if (result != CompilerError.Expected) {
1152                                         if (line.IndexOf (tested_text) != -1) {
1153                                                 if (check_msg) {
1154                                                         int first = line.IndexOf (':');
1155                                                         int second = line.IndexOf (':', first + 1);
1156                                                         if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
1157                                                                 if (check_error_line) {
1158                                                                         second = line.IndexOf (':', second + 1);
1159                                                                 }
1160                                                         } else if (second == -1 || !check_error_line) {
1161                                                                 second = first;
1162                                                         }
1163
1164                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1165                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1166                                                                 error_message = msg;
1167                                                                 return CompilerError.WrongMessage;
1168                                                         }
1169
1170                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1171                                                                 return CompilerError.MissingLocation;
1172                                                 }
1173                                                 result = CompilerError.Expected;
1174                                         } else if (line.IndexOf (error_prefix) != -1 &&
1175                                                 line.IndexOf (ignored_error) == -1)
1176                                                 result = CompilerError.Wrong;
1177                                 }
1178
1179                                 line = sr.ReadLine ();
1180                         }
1181
1182                         return result;
1183                 }
1184
1185                 bool HandleFailure (string file, CompilerError status)
1186                 {
1187                         switch (status) {
1188                                 case CompilerError.Expected:
1189                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1190                                                 LogFileLine (file, "FIXED ISSUE");
1191                                                 return true;
1192                                         }
1193
1194                                         if (verbose)
1195                                                 LogFileLine (file, "OK");
1196                                         return true;
1197
1198                                 case CompilerError.Wrong:
1199                                         if (know_issues.Contains (file)) {
1200                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1201                                                 know_issues.Remove (file);
1202                                                 return false;
1203                                         }
1204                                         if (no_error_list.Contains (file)) {
1205                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1206                                                 no_error_list.Remove (file);
1207                                         }
1208                                         else {
1209                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1210                                         }
1211                                         break;
1212
1213                                 case CompilerError.WrongMessage:
1214                                         if (know_issues.Contains (file)) {
1215                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1216                                                 know_issues.Remove (file);
1217                                                 return false;
1218                                         }
1219                                         if (no_error_list.Contains (file)) {
1220                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1221                                                 no_error_list.Remove (file);
1222                                         }
1223                                         else {
1224                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1225                                                 LogLine ("Exp: {0}", expected_message);
1226                                                 LogLine ("Was: {0}", error_message);
1227                                         }
1228                                         break;
1229
1230                                 case CompilerError.Missing:
1231                                         if (no_error_list.Contains (file)) {
1232                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1233                                                 no_error_list.Remove (file);
1234                                                 return false;
1235                                         }
1236
1237                                         if (know_issues.Contains (file)) {
1238                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1239                                                 know_issues.Remove (file);
1240                                         }
1241                                         else {
1242                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1243                                         }
1244
1245                                         break;
1246
1247                                 case CompilerError.MissingLocation:
1248                                         if (know_issues.Contains (file)) {
1249                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1250                                                 know_issues.Remove (file);
1251                                                 return false;
1252                                         }
1253                                         if (no_error_list.Contains (file)) {
1254                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1255                                                 no_error_list.Remove (file);
1256                                         }
1257                                         else {
1258                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1259                                         }
1260                                         break;
1261
1262                                 case CompilerError.Duplicate:
1263                                         // Will become an error soon
1264                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1265                                         return true;
1266                         }
1267
1268                         regression.Add (file);
1269                         return false;
1270                 }
1271
1272                 protected override void PrintSummary()
1273                 {
1274                         base.PrintSummary ();
1275
1276                         if (wrong_warning.Count > 0) {
1277                                 LogLine ("");
1278                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1279                                 LogLine ("");
1280                                 foreach (DictionaryEntry de in wrong_warning)
1281                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1282                         }
1283                 }
1284
1285         }
1286
1287         class Tester {
1288
1289                 static int Main(string[] args)
1290                 {
1291                         string temp;
1292
1293                         if (GetOption ("help", args, false, out temp)) {
1294                                 Usage ();
1295                                 return 0;
1296                         }
1297
1298                         string compiler;
1299                         if (!GetOption ("compiler", args, true, out compiler)) {
1300                                 Usage ();
1301                                 return 1;
1302                         }
1303
1304                         ITester tester;
1305                         try {
1306                                 Console.WriteLine ("Loading " + compiler + " ...");
1307                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1308                         }
1309                         catch (Exception) {
1310 #if NET_2_1
1311                                 throw;
1312 #else
1313                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1314                                 if (!File.Exists (compiler)) {
1315                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1316                                         return 1;
1317                                 }
1318                                 tester = new ProcessTester (compiler);
1319 #endif
1320                         }
1321
1322                         string mode;
1323                         if (!GetOption ("mode", args, true, out mode)) {
1324                                 Usage ();
1325                                 return 1;
1326                         }
1327
1328                         Checker checker;
1329                         bool positive;
1330                         switch (mode) {
1331                                 case "neg":
1332                                         checker = new NegativeChecker (tester, true);
1333                                         positive = false;
1334                                         break;
1335                                 case "pos":
1336                                         string iltest;
1337                                         GetOption ("il", args, false, out iltest);
1338                                         checker = new PositiveChecker (tester, iltest);
1339                                         positive = true;
1340
1341                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1342                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1343                                         }
1344
1345                                         break;
1346                                 default:
1347                                         Console.Error.WriteLine ("Invalid -mode argument");
1348                                         return 1;
1349                         }
1350
1351
1352                         if (GetOption ("issues", args, true, out temp))
1353                                 checker.IssueFile = temp;
1354                         if (GetOption ("log", args, true, out temp))
1355                                 checker.LogFile = temp;
1356                         if (GetOption ("verbose", args, false, out temp))
1357                                 checker.Verbose = true;
1358                         if (GetOption ("safe-execution", args, false, out temp))
1359                                 checker.SafeExecution = true;
1360                         if (GetOption ("compiler-options", args, true, out temp)) {
1361                                 string[] extra = temp.Split (' ');
1362                                 checker.ExtraCompilerOptions = extra;
1363                         }
1364
1365                         string test_pattern;
1366                         if (!GetOption ("files", args, true, out test_pattern)) {
1367                                 Usage ();
1368                                 return 1;
1369                         }
1370
1371                         var files = new List<string> ();
1372                         switch (test_pattern) {
1373                         case "v1":
1374                                 files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
1375                                 break;
1376                         case "v2":
1377                                 files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
1378                                 goto case "v1";
1379                         case "v4":
1380                                 files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
1381                                 goto case "v2";
1382                         default:
1383                                 files.AddRange (Directory.GetFiles (".", test_pattern));
1384                                 break;
1385                         }
1386
1387                         if (files.Count == 0) {
1388                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1389                                 return 2;
1390                         }
1391
1392                         checker.Initialize ();
1393 /*
1394                         files.Sort ((a, b) => {
1395                                 if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1396                                         if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
1397                                                 return -1;
1398                                 } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1399                                         if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
1400                                                 return 1;
1401                                 }
1402
1403                                 return a.CompareTo (b);
1404                         });
1405 */
1406                         foreach (string s in files) {
1407                                 string filename = Path.GetFileName (s);
1408                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1409                                         continue;
1410                                 }
1411
1412                                 if (filename.EndsWith ("-p2.cs"))
1413                                         continue;
1414
1415                                 checker.Do (filename);
1416                         }
1417
1418                         checker.CleanUp ();
1419
1420                         checker.Dispose ();
1421
1422                         return checker.ResultCode;
1423                 }
1424
1425                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1426                 {
1427                         opt = "-" + opt;
1428                         foreach (string a in args) {
1429                                 if (a.StartsWith (opt)) {
1430                                         int sep = a.IndexOf (':');
1431                                         if (sep > 0) {
1432                                                 value = a.Substring (sep + 1);
1433                                         } else {
1434                                                 value = null;
1435                                                 if (req_arg) {
1436                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1437                                                         return false;
1438                                                 }
1439                                         }
1440
1441                                         return true;
1442                                 }
1443                         }
1444
1445                         value = null;
1446                         return false;
1447                 }
1448
1449                 static void Usage ()
1450                 {
1451                         Console.WriteLine (
1452                                 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1453                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1454                                 "   \n" +
1455                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1456                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1457                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1458                                 "   -issues:FILE     The list of expected failures\n" +
1459                                 "   -log:FILE        Writes any output also to the file\n" +
1460                                 "   -help            Lists all options\n" +
1461                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1462                                 "   -safe-execution  Runs compiled executables in separate app-domain\n" +
1463                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1464                                 "   -verbose         Prints more details during testing\n"
1465                                 );
1466                 }
1467         }
1468 }