mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7
   8 //
   9 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
  10 //
  11 // Permission is hereby granted, free of charge, to any person obtaining
  12 // a copy of this software and associated documentation files (the
  13 // "Software"), to deal in the Software without restriction, including
  14 // without limitation the rights to use, copy, modify, merge, publish,
  15 // distribute, sublicense, and/or sell copies of the Software, and to
  16 // permit persons to whom the Software is furnished to do so, subject to
  17 // the following conditions:
  18 //
  19 // The above copyright notice and this permission notice shall be
  20 // included in all copies or substantial portions of the Software.
  21 //
  22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29 //
  30
  31 using System;
  32 using System.IO;
  33 using System.Diagnostics;
  34 using System.Reflection;
  35 using System.Text;
  36 using System.Collections;
  37 using System.Xml;
  38
  39 namespace TestRunner {
  40
  41         interface ITester
  42         {
  43                 string Output { get; }
  44                 bool Invoke (string[] args);
  45                 bool IsWarning (int warningNumber);
  46         }
  47
  48         class ReflectionTester: ITester {
  49                 MethodInfo ep;
  50                 object[] method_arg;
  51                 StringWriter output;
  52                 int[] all_warnings;
  53
  54                 public ReflectionTester (Assembly a)
  55                 {
  56                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  57
  58                         if (t == null)
  59                                 Console.Error.WriteLine ("null, huh?");
  60
  61                         ep = t.GetMethod ("InvokeCompiler",
  62                                 BindingFlags.Static | BindingFlags.Public);
  63                         if (ep == null)
  64                                 throw new MissingMethodException ("static InvokeCompiler");
  65                         method_arg = new object [2];
  66
  67                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  68                         all_warnings = (int[])pi.GetValue (null, null);
  69                         Array.Sort (all_warnings);
  70                 }
  71
  72                 public string Output {
  73                         get {
  74                                 return output.GetStringBuilder ().ToString ();
  75                         }
  76                 }
  77
  78                 public bool Invoke(string[] args)
  79                 {
  80                         output = new StringWriter ();
  81                         method_arg [0] = args;
  82                         method_arg [1] = output;
  83                         return (bool)ep.Invoke (null, method_arg);
  84                 }
  85
  86                 public bool IsWarning (int warningNumber)
  87                 {
  88                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  89                 }
  90         }
  91
  92 #if !NET_2_1
  93         class ProcessTester: ITester
  94         {
  95                 ProcessStartInfo pi;
  96                 string output;
  97
  98                 public ProcessTester (string p_path)
  99                 {
 100                         pi = new ProcessStartInfo ();
 101                         pi.FileName = p_path;
 102                         pi.CreateNoWindow = true;
 103                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 104                         pi.RedirectStandardOutput = true;
 105                         pi.RedirectStandardError = true;
 106                         pi.UseShellExecute = false;
 107                 }
 108
 109                 public string Output {
 110                         get {
 111                                 return output;
 112                         }
 113                 }
 114
 115                 public bool Invoke(string[] args)
 116                 {
 117                         StringBuilder sb = new StringBuilder ("/nologo ");
 118                         foreach (string s in args) {
 119                                 sb.Append (s);
 120                                 sb.Append (" ");
 121                         }
 122                         pi.Arguments = sb.ToString ();
 123                         Process p = Process.Start (pi);
 124                         output = p.StandardError.ReadToEnd ();
 125                         if (output.Length == 0)
 126                             output = p.StandardOutput.ReadToEnd ();
 127                         p.WaitForExit ();
 128                         return p.ExitCode == 0;
 129                 }
 130
 131                 public bool IsWarning (int warningNumber)
 132                 {
 133                         throw new NotImplementedException ();
 134                 }
 135         }
 136 #endif
 137
 138         class TestCase : MarshalByRefObject
 139         {
 140                 public readonly string FileName;
 141                 public readonly string[] CompilerOptions;
 142                 public readonly string[] Dependencies;
 143
 144                 public TestCase (string filename, string[] options, string[] deps)
 145                 {
 146                         this.FileName = filename;
 147                         this.CompilerOptions = options;
 148                         this.Dependencies = deps;
 149                 }
 150         }
 151
 152         class PositiveTestCase : TestCase
 153         {
 154                 public class VerificationData : MarshalByRefObject
 155                 {
 156                         public class MethodData : MarshalByRefObject
 157                         {
 158                                 public MethodData (MethodBase mi, int il_size)
 159                                 {
 160                                         this.Type = mi.DeclaringType.ToString ();
 161                                         this.MethodName = mi.ToString ();
 162                                         this.ILSize = il_size;
 163                                 }
 164
 165                                 public MethodData (string type_name, string method_name, int il_size)
 166                                 {
 167                                         this.Type = type_name;
 168                                         this.MethodName = method_name;
 169                                         this.ILSize = il_size;
 170                                 }
 171
 172                                 public string Type;
 173                                 public string MethodName;
 174                                 public int ILSize;
 175                                 public bool Checked;
 176                         }
 177
 178                         ArrayList methods;
 179                         public bool IsNewSet;
 180
 181                         public VerificationData (string test_file)
 182                         {
 183 #if NET_2_0
 184                                 this.test_file = test_file;
 185 #endif
 186                         }
 187
 188 #if NET_2_0
 189                         string test_file;
 190
 191                         public static VerificationData FromFile (string name, XmlReader r)
 192                         {
 193                                 VerificationData tc = new VerificationData (name);
 194                                 ArrayList methods = new ArrayList ();
 195                                 r.Read ();
 196                                 while (r.ReadToNextSibling ("type")) {
 197                                         string type_name = r ["name"];
 198                                         r.Read ();
 199                                         while (r.ReadToNextSibling ("method")) {
 200                                                 string m_name = r ["name"];
 201
 202                                                 r.ReadToDescendant ("size");
 203                                                 int il_size = r.ReadElementContentAsInt ();
 204                                                 methods.Add (new MethodData (type_name, m_name, il_size));
 205                                                 r.Read ();
 206                                         }
 207                                         r.Read ();
 208                                 }
 209
 210                                 tc.methods = methods;
 211                                 return tc;
 212                         }
 213
 214                         public void WriteCodeInfoTo (XmlWriter w)
 215                         {
 216                                 w.WriteStartElement ("test");
 217                                 w.WriteAttributeString ("name", test_file);
 218
 219                                 string type = null;
 220                                 foreach (MethodData data in methods) {
 221                                         if (!data.Checked)
 222                                                 continue;
 223
 224                                         if (type != data.Type) {
 225                                                 if (type != null)
 226                                                         w.WriteEndElement ();
 227
 228                                                 type = data.Type;
 229                                                 w.WriteStartElement ("type");
 230                                                 w.WriteAttributeString ("name", type);
 231                                         }
 232
 233                                         w.WriteStartElement ("method");
 234                                         w.WriteAttributeString ("name", data.MethodName);
 235                                         w.WriteStartElement ("size");
 236                                         w.WriteValue (data.ILSize);
 237                                         w.WriteEndElement ();
 238                                         w.WriteEndElement ();
 239                                 }
 240
 241                                 if (type != null)
 242                                         w.WriteEndElement ();
 243
 244                                 w.WriteEndElement ();
 245                         }
 246 #endif
 247
 248                         public MethodData FindMethodData (string method_name, string declaring_type)
 249                         {
 250                                 if (methods == null)
 251                                         return null;
 252
 253                                 foreach (MethodData md in methods) {
 254                                         if (md.MethodName == method_name && md.Type == declaring_type)
 255                                                 return md;
 256                                 }
 257
 258                                 return null;
 259                         }
 260
 261                         public void AddNewMethod (MethodBase mb, int il_size)
 262                         {
 263                                 if (methods == null)
 264                                         methods = new ArrayList ();
 265
 266                                 MethodData md = new MethodData (mb, il_size);
 267                                 md.Checked = true;
 268                                 methods.Add (md);
 269                         }
 270                 }
 271
 272                 VerificationData verif_data;
 273
 274                 public PositiveTestCase (string filename, string [] options, string [] deps)
 275                         : base (filename, options, deps)
 276                 {
 277                 }
 278
 279                 public void CreateNewTest ()
 280                 {
 281                         verif_data = new VerificationData (FileName);
 282                         verif_data.IsNewSet = true;
 283                 }
 284
 285                 public VerificationData VerificationProvider {
 286                         set {
 287                                 verif_data = value;
 288                         }
 289                         get {
 290                                 return verif_data;
 291                         }
 292                 }
 293         }
 294
 295         class Checker: MarshalByRefObject, IDisposable
 296         {
 297                 protected ITester tester;
 298                 protected int success;
 299                 protected int total;
 300                 protected int ignored;
 301                 protected int syntax_errors;
 302                 string issue_file;
 303                 StreamWriter log_file;
 304                 protected string[] extra_compiler_options;
 305                 // protected string[] compiler_options;
 306                 // protected string[] dependencies;
 307
 308                 protected ArrayList tests = new ArrayList ();
 309                 protected Hashtable test_hash = new Hashtable ();
 310                 protected ArrayList regression = new ArrayList ();
 311                 protected ArrayList know_issues = new ArrayList ();
 312                 protected ArrayList ignore_list = new ArrayList ();
 313                 protected ArrayList no_error_list = new ArrayList ();
 314
 315                 protected bool verbose;
 316                 protected bool safe_execution;
 317
 318                 int total_known_issues;
 319
 320                 protected Checker (ITester tester)
 321                 {
 322                         this.tester = tester;
 323                 }
 324
 325                 public string IssueFile {
 326                         set {
 327                                 this.issue_file = value;
 328                                 ReadWrongErrors (issue_file);
 329                         }
 330                 }
 331
 332                 public string LogFile {
 333                         set {
 334                                 this.log_file = new StreamWriter (value, false);
 335                         }
 336                 }
 337
 338                 public bool Verbose {
 339                         set {
 340                                 verbose = value;
 341                         }
 342                 }
 343
 344                 public bool SafeExecution {
 345                         set {
 346                                 safe_execution = value;
 347                         }
 348                 }
 349
 350                 public string[] ExtraCompilerOptions {
 351                         set {
 352                                 extra_compiler_options = value;
 353                         }
 354                 }
 355
 356                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 357                                                         out string[] dependencies)
 358                 {
 359                         int row = 0;
 360                         compiler_options = null;
 361                         dependencies = null;
 362                         try {
 363                                 using (StreamReader sr = new StreamReader (file)) {
 364                                         String line;
 365                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 366                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 367                                                                       ref dependencies))
 368                                                         return false;
 369                                         }
 370                                 }
 371                         } catch {
 372                                 return false;
 373                         }
 374                         return true;
 375                 }
 376
 377                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 378                                                         ref string[] compiler_options,
 379                                                         ref string[] dependencies)
 380                 {
 381                         const string options = "// Compiler options:";
 382                         const string depends = "// Dependencies:";
 383
 384                         if (row == 1) {
 385                                 compiler_options = null;
 386                                 dependencies = null;
 387                         }
 388
 389                         int index = line.IndexOf (options);
 390                         if (index != -1) {
 391                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 392                                 for (int i = 0; i < compiler_options.Length; i++)
 393                                         compiler_options[i] = compiler_options[i].TrimStart ();
 394                         }
 395                         index = line.IndexOf (depends);
 396                         if (index != -1) {
 397                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 398                                 for (int i = 0; i < dependencies.Length; i++)
 399                                         dependencies[i] = dependencies[i].TrimStart ();
 400                         }
 401
 402                         return true;
 403                 }
 404
 405                 public bool Do (string filename)
 406                 {
 407                         if (test_hash.Contains (filename))
 408                                 return true;
 409
 410                         if (verbose)
 411                                 Log (filename + "...\t");
 412
 413                         if (ignore_list.Contains (filename)) {
 414                                 ++ignored;
 415                                 LogFileLine (filename, "NOT TESTED");
 416                                 return false;
 417                         }
 418
 419                         string[] compiler_options, dependencies;
 420                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 421                                 LogFileLine (filename, "ERROR");
 422                                 return false;
 423                         }
 424
 425                         if (extra_compiler_options != null) {
 426                                 if (compiler_options == null)
 427                                         compiler_options = extra_compiler_options;
 428                                 else {
 429                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 430                                         extra_compiler_options.CopyTo (new_options, 0);
 431                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 432                                         compiler_options = new_options;
 433                                 }
 434                         }
 435
 436                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 437                         test_hash.Add (filename, test);
 438
 439                         ++total;
 440                         if (dependencies != null) {
 441                                 foreach (string dependency in dependencies) {
 442                                         if (!Do (dependency)) {
 443                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 444                                                 return false;
 445                                         }
 446                                 }
 447                         }
 448
 449                         tests.Add (test);
 450
 451                         return Check (test);
 452                 }
 453
 454                 protected virtual bool Check (TestCase test)
 455                 {
 456                         string[] test_args;
 457
 458                         if (test.CompilerOptions != null) {
 459                                 test_args = new string [2 + test.CompilerOptions.Length];
 460                                 test.CompilerOptions.CopyTo (test_args, 0);
 461                         } else {
 462                                 test_args = new string [2];
 463                         }
 464                         test_args [test_args.Length - 2] = test.FileName;
 465                         test_args [test_args.Length - 1] = "-debug";
 466
 467                         return tester.Invoke (test_args);
 468                 }
 469
 470                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 471                 {
 472                         return new TestCase (filename, options, deps);
 473                 }
 474
 475                 void ReadWrongErrors (string file)
 476                 {
 477                         const string ignored = "IGNORE";
 478                         const string no_error = "NO ERROR";
 479
 480                         using (StreamReader sr = new StreamReader (file)) {
 481                                 string line;
 482                                 while ((line = sr.ReadLine()) != null) {
 483                                         if (line.StartsWith ("#"))
 484                                                 continue;
 485
 486                                         ArrayList active_cont = know_issues;
 487
 488                                         if (line.IndexOf (ignored) > 0)
 489                                                 active_cont = ignore_list;
 490                                         else if (line.IndexOf (no_error) > 0)
 491                                                 active_cont = no_error_list;
 492
 493                                         string file_name = line.Split (' ')[0];
 494                                         if (file_name.Length == 0)
 495                                                 continue;
 496
 497                                         active_cont.Add (file_name);
 498                                 }
 499                         }
 500                         total_known_issues = know_issues.Count;
 501                 }
 502
 503                 protected virtual void PrintSummary ()
 504                 {
 505                         LogLine ("Done" + Environment.NewLine);
 506                         float rate = 0;
 507                         if (total > 0)
 508                                 rate = (float) (success) / (float)total;
 509                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 510
 511                         if (syntax_errors > 0)
 512                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 513
 514                         if (ignored > 0)
 515                                 LogLine ("{0} test(s) ignored", ignored);
 516
 517                         if (total_known_issues - know_issues.Count > 0)
 518                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 519
 520                         know_issues.AddRange (no_error_list);
 521                         if (know_issues.Count > 0) {
 522                                 LogLine ("");
 523                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 524                                 foreach (string s in know_issues)
 525                                         LogLine (s);
 526                         }
 527                         if (regression.Count > 0) {
 528                                 LogLine ("");
 529                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 530                                 foreach (string s in regression)
 531                                         LogLine (s);
 532                         }
 533                 }
 534
 535                 public int ResultCode
 536                 {
 537                         get {
 538                                 return regression.Count == 0 ? 0 : 1;
 539                         }
 540                 }
 541
 542                 protected void Log (string msg, params object [] rest)
 543                 {
 544                         Console.Write (msg, rest);
 545                         if (log_file != null)
 546                                 log_file.Write (msg, rest);
 547                 }
 548
 549                 protected void LogLine (string msg)
 550                 {
 551                         Console.WriteLine (msg);
 552                         if (log_file != null)
 553                                 log_file.WriteLine (msg);
 554                 }
 555
 556                 protected void LogLine (string msg, params object [] rest)
 557                 {
 558                         Console.WriteLine (msg, rest);
 559                         if (log_file != null)
 560                                 log_file.WriteLine (msg, rest);
 561                 }
 562
 563                 public void LogFileLine (string file, string msg, params object [] args)
 564                 {
 565                         string s = verbose ?
 566                                 string.Format (msg, args) :
 567                                 file + "...\t" + string.Format (msg, args);
 568
 569                         Console.WriteLine (s);
 570                         if (log_file != null)
 571                                 log_file.WriteLine (s);
 572                 }
 573
 574                 #region IDisposable Members
 575
 576                 public void Dispose()
 577                 {
 578                         if (log_file != null)
 579                                 log_file.Close ();
 580                 }
 581
 582                 #endregion
 583
 584                 public virtual void Initialize ()
 585                 {
 586                 }
 587
 588                 public virtual void CleanUp ()
 589                 {
 590                         PrintSummary ();
 591                 }
 592         }
 593
 594         class PositiveChecker: Checker
 595         {
 596                 readonly string files_folder;
 597                 readonly static object[] default_args = new object[1] { new string[] {} };
 598                 string doc_output;
 599                 string verif_file;
 600                 bool update_verif_file;
 601                 Hashtable verif_data;
 602
 603 #if !NET_2_1
 604                 ProcessStartInfo pi;
 605 #endif
 606                 readonly string mono;
 607
 608                 public enum TestResult {
 609                         CompileError,
 610                         ExecError,
 611                         LoadError,
 612                         XmlError,
 613                         Success,
 614                         ILError
 615                 }
 616
 617                 public PositiveChecker (ITester tester, string verif_file):
 618                         base (tester)
 619                 {
 620                         files_folder = Directory.GetCurrentDirectory ();
 621                         this.verif_file = verif_file;
 622
 623 #if !NET_2_1
 624                         pi = new ProcessStartInfo ();
 625                         pi.CreateNoWindow = true;
 626                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 627                         pi.RedirectStandardOutput = true;
 628                         pi.RedirectStandardError = true;
 629                         pi.UseShellExecute = false;
 630
 631                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 632                         if (mono != null) {
 633                                 pi.FileName = mono;
 634                         }
 635 #endif
 636                 }
 637
 638                 public bool UpdateVerificationDataFile {
 639                         set {
 640                                 update_verif_file = value;
 641                         }
 642                         get {
 643                                 return update_verif_file;
 644                         }
 645                 }
 646
 647                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 648                                                         out string[] dependencies) {
 649                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 650                                 return false;
 651
 652                         doc_output = null;
 653                         if (compiler_options == null)
 654                                 return true;
 655
 656                         foreach (string one_opt in compiler_options) {
 657                                 if (one_opt.StartsWith ("-doc:")) {
 658                                         doc_output = one_opt.Split (':', '/')[1];
 659                                 }
 660                         }
 661                         return true;
 662                 }
 663
 664                 class DomainTester : MarshalByRefObject
 665                 {
 666                         public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
 667                         {
 668                                 Assembly assembly = Assembly.LoadFile (file);
 669
 670                                 bool success = true;
 671                                 Type[] types = assembly.GetTypes ();
 672                                 foreach (Type t in types) {
 673
 674                                         // Skip interfaces
 675                                         if (!t.IsClass && !t.IsValueType)
 676                                                 continue;
 677
 678                                         if (test.VerificationProvider == null) {
 679                                                 if (!checker.UpdateVerificationDataFile)
 680                                                         checker.LogFileLine (test.FileName, "Missing IL verification data");
 681                                                 test.CreateNewTest ();
 682                                         }
 683
 684                                         foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 685                                                 MethodBase mi = m as MethodBase;
 686                                                 if (mi == null)
 687                                                         continue;
 688
 689                                                 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 690                                                         continue;
 691
 692                                                 success &= CompareIL (mi, test, checker);
 693                                         }
 694                                 }
 695
 696                                 return success;
 697                         }
 698
 699                         bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
 700                         {
 701                                 string m_name = mi.ToString ();
 702                                 string decl_type = mi.DeclaringType.ToString ();
 703                                 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
 704
 705                                 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
 706                                 if (md == null) {
 707                                         data_provider.AddNewMethod (mi, GetILSize (mi));
 708                                         if (!data_provider.IsNewSet) {
 709                                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 710                                                 return false;
 711                                         }
 712
 713                                         return true;
 714                                 }
 715
 716                                 if (md.Checked) {
 717                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 718                                         return false;
 719                                 }
 720
 721                                 md.Checked = true;
 722
 723                                 int il_size = GetILSize (mi);
 724                                 if (md.ILSize == il_size)
 725                                         return true;
 726
 727                                 if (md.ILSize > il_size) {
 728                                         checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", m_name, md.ILSize, il_size);
 729                                         md.ILSize = il_size;
 730                                         return true;
 731                                 }
 732
 733                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
 734                                         string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
 735
 736                                 md.ILSize = il_size;
 737
 738                                 return false;
 739                         }
 740
 741                         static int GetILSize (MethodBase mi)
 742                         {
 743 #if NET_2_0
 744                                 MethodBody body = mi.GetMethodBody ();
 745                                 if (body != null)
 746                                         return body.GetILAsByteArray ().Length;
 747 #endif
 748                                 return 0;
 749                         }
 750
 751                         bool ExecuteFile (MethodInfo entry_point, string filename)
 752                         {
 753                                 TextWriter stdout = Console.Out;
 754                                 TextWriter stderr = Console.Error;
 755                                 Console.SetOut (TextWriter.Null);
 756                                 Console.SetError (TextWriter.Null);
 757                                 ParameterInfo[] pi = entry_point.GetParameters ();
 758                                 object[] args = pi.Length == 0 ? null : default_args;
 759
 760                                 object result = null;
 761                                 try {
 762                                         try {
 763                                                 result = entry_point.Invoke (null, args);
 764                                         } finally {
 765                                                 Console.SetOut (stdout);
 766                                                 Console.SetError (stderr);
 767                                         }
 768                                 } catch (Exception e) {
 769                                         throw new ApplicationException (e.ToString ());
 770                                 }
 771
 772                                 if (result is int && (int) result != 0)
 773                                         throw new ApplicationException ("Wrong return code: " + result.ToString ());
 774
 775                                 return true;
 776                         }
 777
 778                         public bool Test (string file)
 779                         {
 780                                 Assembly assembly = Assembly.LoadFile (file);
 781                                 return ExecuteFile (assembly.EntryPoint, file);
 782                         }
 783                 }
 784
 785                 protected override bool Check(TestCase test)
 786                 {
 787                         string filename = test.FileName;
 788                         try {
 789                                 if (!base.Check (test)) {
 790                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 791                                         return false;
 792                                 }
 793                         }
 794                         catch (Exception e) {
 795                                 if (e.InnerException != null)
 796                                         e = e.InnerException;
 797
 798                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 799                                 return false;
 800                         }
 801
 802                         // Test setup
 803                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 804                                 if (verbose)
 805                                         LogFileLine (filename, "OK");
 806                                 --total;
 807                                 return true;
 808                         }
 809
 810                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 811
 812                         // Enable .dll only tests (no execution required)
 813                         if (!File.Exists(file)) {
 814                                 HandleFailure (filename, TestResult.Success, null);
 815                                 return true;
 816                         }
 817
 818                         AppDomain domain = null;
 819                         if (safe_execution) {
 820                                 // Create a new AppDomain, with the current directory as the base.
 821                                 AppDomainSetup setupInfo = new AppDomainSetup ();
 822                                 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
 823                                 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
 824                                 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
 825                         }
 826
 827                         try {
 828                                 DomainTester tester;
 829                                 try {
 830                                         if (domain != null)
 831                                                 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
 832                                         else
 833                                                 tester = new DomainTester ();
 834
 835                                         if (!tester.Test (file))
 836                                                 return false;
 837
 838                                 } catch (ApplicationException e) {
 839                                         HandleFailure (filename, TestResult.ExecError, e.Message);
 840                                         return false;
 841                                 } catch (Exception e) {
 842                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 843                                         return false;
 844                                 }
 845
 846                                 if (doc_output != null) {
 847                                         string ref_file = filename.Replace (".cs", "-ref.xml");
 848                                         try {
 849 #if !NET_2_1
 850                                                 XmlComparer.Compare (ref_file, doc_output);
 851 #endif
 852                                         } catch (Exception e) {
 853                                                 HandleFailure (filename, TestResult.XmlError, e.Message);
 854                                                 return false;
 855                                         }
 856                                 } else {
 857                                         if (verif_file != null) {
 858                                                 PositiveTestCase pt = (PositiveTestCase) test;
 859                                                 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
 860
 861                                                 if (!tester.CheckILSize (pt, this, file))
 862                                                         return false;
 863                                         }
 864                                 }
 865                         } finally {
 866                                 if (domain != null)
 867                                         AppDomain.Unload (domain);
 868                         }
 869
 870                         HandleFailure (filename, TestResult.Success, null);
 871                         return true;
 872                 }
 873
 874                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
 875                 {
 876                         return new PositiveTestCase (filename, options, deps);
 877                 }
 878
 879                 public void HandleFailure (string file, TestResult status, string extra)
 880                 {
 881                         switch (status) {
 882                                 case TestResult.Success:
 883                                         success++;
 884                                         if (know_issues.Contains (file)) {
 885                                                 LogFileLine (file, "FIXED ISSUE");
 886                                                 return;
 887                                         }
 888                                         if (verbose)
 889                                                 LogFileLine (file, "OK");
 890                                         return;
 891
 892                                 case TestResult.CompileError:
 893                                         if (know_issues.Contains (file)) {
 894                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 895                                                 know_issues.Remove (file);
 896                                                 return;
 897                                         }
 898                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 899                                         break;
 900
 901                                 case TestResult.ExecError:
 902                                         if (know_issues.Contains (file)) {
 903                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 904                                                 know_issues.Remove (file);
 905                                                 return;
 906                                         }
 907                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 908                                         break;
 909
 910                                 case TestResult.XmlError:
 911                                         if (know_issues.Contains (file)) {
 912                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
 913                                                 know_issues.Remove (file);
 914                                                 return;
 915                                         }
 916                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 917                                         break;
 918
 919                                 case TestResult.LoadError:
 920                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
 921                                         break;
 922
 923                                 case TestResult.ILError:
 924                                         if (!update_verif_file) {
 925                                                 LogFileLine (file, "IL REGRESSION: " + extra);
 926                                         }
 927                                         extra = null;
 928                                         break;
 929                         }
 930
 931                         if (extra != null)
 932                                 LogLine ("{0}", extra);
 933
 934                         if (!regression.Contains (file))
 935                                 regression.Add (file);
 936                 }
 937
 938                 public override void Initialize ()
 939                 {
 940                         if (verif_file != null) {
 941 #if NET_2_0
 942                                 LoadVerificationData (verif_file);
 943 #else
 944                                 throw new NotSupportedException ();
 945 #endif
 946                         }
 947
 948                         base.Initialize ();
 949                 }
 950
 951                 public override void CleanUp ()
 952                 {
 953                         base.CleanUp ();
 954
 955                         if (update_verif_file) {
 956 #if NET_2_0
 957                                 UpdateVerificationData (verif_file);
 958 #else
 959                                 throw new NotSupportedException ();
 960 #endif
 961                         }
 962                 }
 963
 964 #if NET_2_0
 965                 void LoadVerificationData (string file)
 966                 {
 967                         LogLine ("Loading verification data from `{0}' ...", file);
 968
 969                         using (XmlReader r = XmlReader.Create (file)) {
 970                                 r.ReadStartElement ("tests");
 971                                 verif_data = new Hashtable ();
 972
 973                                 while (r.Read ()) {
 974                                         if (r.Name != "test")
 975                                                 continue;
 976
 977                                         string name = r.GetAttribute ("name");
 978                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
 979                                         verif_data.Add (name, tc);
 980                                 }
 981                         }
 982                 }
 983
 984                 void UpdateVerificationData (string file)
 985                 {
 986                         LogLine ("Updating verification data `{0}' ...", file);
 987
 988                         XmlWriterSettings s = new XmlWriterSettings ();
 989                         s.Indent = true;
 990                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
 991                                 w.WriteStartDocument ();
 992                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
 993                                 w.WriteStartElement ("tests");
 994                                 foreach (PositiveTestCase tc in tests) {
 995                                         if (tc.VerificationProvider != null)
 996                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
 997                                 }
 998                                 w.WriteEndElement ();
 999                         }
1000                 }
1001 #endif
1002         }
1003
1004         class NegativeChecker: Checker
1005         {
1006                 string expected_message;
1007                 string error_message;
1008                 bool check_msg;
1009                 bool check_error_line;
1010                 bool is_warning;
1011                 IDictionary wrong_warning;
1012
1013                 protected enum CompilerError {
1014                         Expected,
1015                         Wrong,
1016                         Missing,
1017                         WrongMessage,
1018                         MissingLocation,
1019                         Duplicate
1020                 }
1021
1022                 public NegativeChecker (ITester tester, bool check_msg):
1023                         base (tester)
1024                 {
1025                         this.check_msg = check_msg;
1026                         wrong_warning = new Hashtable ();
1027                 }
1028
1029                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1030                                                         ref string[] compiler_options,
1031                                                         ref string[] dependencies)
1032                 {
1033                         if (row == 1) {
1034                                 expected_message = null;
1035
1036                                 int index = line.IndexOf (':');
1037                                 if (index == -1 || index > 15) {
1038                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1039                                         ++syntax_errors;
1040                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1041                                                               ref dependencies);
1042                                         return false;
1043                                 }
1044
1045                                 expected_message = line.Substring (index + 1).Trim ();
1046                         }
1047
1048                         if (row == 2) {
1049                                 string filtered = line.Replace(" ", "");
1050
1051                                 // Some error tests require to have different error text for different runtimes.
1052                                 if (filtered.StartsWith ("//GMCS")) {
1053                                         row = 1;
1054 #if !NET_2_0
1055                                         return true;
1056 #else
1057                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1058 #endif
1059                                 }
1060
1061                                 check_error_line = !filtered.StartsWith ("//Line:0");
1062
1063                                 if (!filtered.StartsWith ("//Line:")) {
1064                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1065                                         ++syntax_errors;
1066                                         return false;
1067                                 }
1068                         }
1069
1070                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1071                                 return false;
1072
1073                         is_warning = false;
1074                         if (compiler_options != null) {
1075                                 foreach (string s in compiler_options) {
1076                                         if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1077                                                 is_warning = true;
1078                                 }
1079                         }
1080                         return true;
1081                 }
1082
1083
1084                 protected override bool Check (TestCase test)
1085                 {
1086                         string filename = test.FileName;
1087
1088                         int start_char = 0;
1089                         while (Char.IsLetter (filename, start_char))
1090                                 ++start_char;
1091
1092                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1093                         string expected = filename.Substring (start_char, end_char - start_char);
1094
1095                         try {
1096                                 if (base.Check (test)) {
1097                                         HandleFailure (filename, CompilerError.Missing);
1098                                         return false;
1099                                 }
1100                         }
1101                         catch (Exception e) {
1102                                 HandleFailure (filename, CompilerError.Missing);
1103                                 if (e.InnerException != null)
1104                                         e = e.InnerException;
1105
1106                                 Log (e.ToString ());
1107                                 return false;
1108                         }
1109
1110                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1111                         if (tester.IsWarning (err_id)) {
1112                                 if (!is_warning)
1113                                         wrong_warning [err_id] = true;
1114                         } else {
1115                                 if (is_warning)
1116                                         wrong_warning [err_id] = false;
1117                         }
1118
1119                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1120                         if (HandleFailure (filename, result_code)) {
1121                                 success++;
1122                                 return true;
1123                         }
1124
1125                         if (result_code == CompilerError.Wrong)
1126                                 LogLine (tester.Output);
1127
1128                         return false;
1129                 }
1130
1131                 CompilerError GetCompilerError (string expected, string buffer)
1132                 {
1133                         const string error_prefix = "CS";
1134                         const string ignored_error = "error CS5001";
1135                         string tested_text = "error " + error_prefix + expected;
1136                         StringReader sr = new StringReader (buffer);
1137                         string line = sr.ReadLine ();
1138                         ArrayList ld = new ArrayList ();
1139                         CompilerError result = CompilerError.Missing;
1140                         while (line != null) {
1141                                 if (ld.Contains (line)) {
1142                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1143                                                 return CompilerError.Duplicate;
1144                                 }
1145                                 ld.Add (line);
1146
1147                                 if (result != CompilerError.Expected) {
1148                                         if (line.IndexOf (tested_text) != -1) {
1149                                                 if (check_msg) {
1150                                                         int first = line.IndexOf (':');
1151                                                         int second = line.IndexOf (':', first + 1);
1152                                                         if (second == -1 || !check_error_line)
1153                                                                 second = first;
1154
1155                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1156                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1157                                                                 error_message = msg;
1158                                                                 return CompilerError.WrongMessage;
1159                                                         }
1160
1161                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1162                                                                 return CompilerError.MissingLocation;
1163                                                 }
1164                                                 result = CompilerError.Expected;
1165                                         } else if (line.IndexOf (error_prefix) != -1 &&
1166                                                 line.IndexOf (ignored_error) == -1)
1167                                                 result = CompilerError.Wrong;
1168                                 }
1169
1170                                 line = sr.ReadLine ();
1171                         }
1172
1173                         return result;
1174                 }
1175
1176                 bool HandleFailure (string file, CompilerError status)
1177                 {
1178                         switch (status) {
1179                                 case CompilerError.Expected:
1180                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1181                                                 LogFileLine (file, "FIXED ISSUE");
1182                                                 return true;
1183                                         }
1184
1185                                         if (verbose)
1186                                                 LogFileLine (file, "OK");
1187                                         return true;
1188
1189                                 case CompilerError.Wrong:
1190                                         if (know_issues.Contains (file)) {
1191                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1192                                                 know_issues.Remove (file);
1193                                                 return false;
1194                                         }
1195                                         if (no_error_list.Contains (file)) {
1196                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1197                                                 no_error_list.Remove (file);
1198                                         }
1199                                         else {
1200                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1201                                         }
1202                                         break;
1203
1204                                 case CompilerError.WrongMessage:
1205                                         if (know_issues.Contains (file)) {
1206                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1207                                                 know_issues.Remove (file);
1208                                                 return false;
1209                                         }
1210                                         if (no_error_list.Contains (file)) {
1211                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1212                                                 no_error_list.Remove (file);
1213                                         }
1214                                         else {
1215                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1216                                                 LogLine ("Exp: {0}", expected_message);
1217                                                 LogLine ("Was: {0}", error_message);
1218                                         }
1219                                         break;
1220
1221                                 case CompilerError.Missing:
1222                                         if (no_error_list.Contains (file)) {
1223                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1224                                                 no_error_list.Remove (file);
1225                                                 return false;
1226                                         }
1227
1228                                         if (know_issues.Contains (file)) {
1229                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1230                                                 know_issues.Remove (file);
1231                                         }
1232                                         else {
1233                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1234                                         }
1235
1236                                         break;
1237
1238                                 case CompilerError.MissingLocation:
1239                                         if (know_issues.Contains (file)) {
1240                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1241                                                 know_issues.Remove (file);
1242                                                 return false;
1243                                         }
1244                                         if (no_error_list.Contains (file)) {
1245                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1246                                                 no_error_list.Remove (file);
1247                                         }
1248                                         else {
1249                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1250                                         }
1251                                         break;
1252
1253                                 case CompilerError.Duplicate:
1254                                         // Will become an error soon
1255                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1256                                         return true;
1257                         }
1258
1259                         regression.Add (file);
1260                         return false;
1261                 }
1262
1263                 protected override void PrintSummary()
1264                 {
1265                         base.PrintSummary ();
1266
1267                         if (wrong_warning.Count > 0) {
1268                                 LogLine ("");
1269                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1270                                 LogLine ("");
1271                                 foreach (DictionaryEntry de in wrong_warning)
1272                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1273                         }
1274                 }
1275
1276         }
1277
1278         class Tester {
1279
1280                 static int Main(string[] args)
1281                 {
1282                         string temp;
1283
1284                         if (GetOption ("help", args, false, out temp)) {
1285                                 Usage ();
1286                                 return 0;
1287                         }
1288
1289                         string compiler;
1290                         if (!GetOption ("compiler", args, true, out compiler)) {
1291                                 Usage ();
1292                                 return 1;
1293                         }
1294
1295                         ITester tester;
1296                         try {
1297                                 Console.WriteLine ("Loading " + compiler + " ...");
1298                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1299                         }
1300                         catch (Exception) {
1301 #if NET_2_1
1302                                 throw;
1303 #else
1304                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1305                                 if (!File.Exists (compiler)) {
1306                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1307                                         return 1;
1308                                 }
1309                                 tester = new ProcessTester (compiler);
1310 #endif
1311                         }
1312
1313                         string mode;
1314                         if (!GetOption ("mode", args, true, out mode)) {
1315                                 Usage ();
1316                                 return 1;
1317                         }
1318
1319                         Checker checker;
1320                         switch (mode) {
1321                                 case "neg":
1322                                         checker = new NegativeChecker (tester, true);
1323                                         break;
1324                                 case "pos":
1325                                         string iltest;
1326                                         GetOption ("il", args, false, out iltest);
1327                                         checker = new PositiveChecker (tester, iltest);
1328
1329                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1330                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1331                                         }
1332
1333                                         break;
1334                                 default:
1335                                         Console.Error.WriteLine ("Invalid -mode argument");
1336                                         return 1;
1337                         }
1338
1339
1340                         if (GetOption ("issues", args, true, out temp))
1341                                 checker.IssueFile = temp;
1342                         if (GetOption ("log", args, true, out temp))
1343                                 checker.LogFile = temp;
1344                         if (GetOption ("verbose", args, false, out temp))
1345                                 checker.Verbose = true;
1346                         if (GetOption ("safe-execution", args, false, out temp))
1347                                 checker.SafeExecution = true;
1348                         if (GetOption ("compiler-options", args, true, out temp)) {
1349                                 string[] extra = temp.Split (' ');
1350                                 checker.ExtraCompilerOptions = extra;
1351                         }
1352
1353                         string test_pattern;
1354                         if (!GetOption ("files", args, true, out test_pattern)) {
1355                                 Usage ();
1356                                 return 1;
1357                         }
1358
1359                         ArrayList files = new ArrayList ();
1360                         switch (test_pattern) {
1361                         case "v1":
1362                                 files.AddRange (Directory.GetFiles (".", "test*.cs"));
1363                                 break;
1364                         case "v2":
1365                                 files.AddRange (Directory.GetFiles (".", "gtest*.cs"));
1366                                 goto case "v1";
1367                         case "v4":
1368                                 files.AddRange (Directory.GetFiles (".", "dtest*.cs"));
1369                                 goto case "v2";
1370                         default:
1371                                 files.AddRange (Directory.GetFiles (".", test_pattern));
1372                                 break;
1373                         }
1374
1375                         if (files.Count == 0) {
1376                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1377                                 return 2;
1378                         }
1379
1380                         checker.Initialize ();
1381
1382                         foreach (string s in files) {
1383                                 string filename = Path.GetFileName (s);
1384                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1385                                         continue;
1386                                 }
1387
1388                                 if (filename.EndsWith ("-p2.cs"))
1389                                         continue;
1390
1391                                 checker.Do (filename);
1392                         }
1393
1394                         checker.CleanUp ();
1395
1396                         checker.Dispose ();
1397
1398                         return checker.ResultCode;
1399                 }
1400
1401                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1402                 {
1403                         opt = "-" + opt;
1404                         foreach (string a in args) {
1405                                 if (a.StartsWith (opt)) {
1406                                         int sep = a.IndexOf (':');
1407                                         if (sep > 0) {
1408                                                 value = a.Substring (sep + 1);
1409                                         } else {
1410                                                 value = null;
1411                                                 if (req_arg) {
1412                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1413                                                         return false;
1414                                                 }
1415                                         }
1416
1417                                         return true;
1418                                 }
1419                         }
1420
1421                         value = null;
1422                         return false;
1423                 }
1424
1425                 static void Usage ()
1426                 {
1427                         Console.WriteLine (
1428                                 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1429                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1430                                 "   \n" +
1431                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1432                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1433                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1434                                 "   -issues:FILE     The list of expected failures\n" +
1435                                 "   -log:FILE        Writes any output also to the file\n" +
1436                                 "   -help            Lists all options\n" +
1437                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1438                                 "   -safe-execution  Runs compiled executables in separate app-domain\n" +
1439                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1440                                 "   -verbose         Prints more details during testing\n"
1441                                 );
1442                 }
1443         }
1444 }