mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
   8 // Copyright (C) 2012 Xamarin Inc (http://www.xamarin.com)
   9 //
  10 // Permission is hereby granted, free of charge, to any person obtaining
  11 // a copy of this software and associated documentation files (the
  12 // "Software"), to deal in the Software without restriction, including
  13 // without limitation the rights to use, copy, modify, merge, publish,
  14 // distribute, sublicense, and/or sell copies of the Software, and to
  15 // permit persons to whom the Software is furnished to do so, subject to
  16 // the following conditions:
  17 //
  18 // The above copyright notice and this permission notice shall be
  19 // included in all copies or substantial portions of the Software.
  20 //
  21 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  22 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  23 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  24 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  25 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  26 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  27 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  28 //
  29
  30 using System;
  31 using System.IO;
  32 using System.Diagnostics;
  33 using System.Reflection;
  34 using System.Text;
  35 using System.Collections;
  36 using System.Xml;
  37 using System.Collections.Generic;
  38 using Mono.CompilerServices.SymbolWriter;
  39 using System.Globalization;
  40
  41 namespace TestRunner {
  42
  43         interface ITester
  44         {
  45                 string Output { get; }
  46                 bool Invoke (string[] args);
  47                 bool IsWarning (int warningNumber);
  48         }
  49
  50         class ReflectionTester: ITester {
  51                 MethodInfo ep;
  52                 object[] method_arg;
  53                 StringWriter output;
  54                 int[] all_warnings;
  55
  56                 public ReflectionTester (Assembly a)
  57                 {
  58                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  59
  60                         if (t == null)
  61                                 Console.Error.WriteLine ("null, huh?");
  62
  63                         ep = t.GetMethod ("InvokeCompiler",
  64                                 BindingFlags.Static | BindingFlags.Public);
  65                         if (ep == null)
  66                                 throw new MissingMethodException ("static InvokeCompiler");
  67                         method_arg = new object [2];
  68
  69                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  70                         all_warnings = (int[])pi.GetValue (null, null);
  71                         Array.Sort (all_warnings);
  72                 }
  73
  74                 public string Output {
  75                         get {
  76                                 return output.GetStringBuilder ().ToString ();
  77                         }
  78                 }
  79
  80                 public bool Invoke(string[] args)
  81                 {
  82                         output = new StringWriter ();
  83                         method_arg [0] = args;
  84                         method_arg [1] = output;
  85                         return (bool)ep.Invoke (null, method_arg);
  86                 }
  87
  88                 public bool IsWarning (int warningNumber)
  89                 {
  90                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  91                 }
  92         }
  93
  94 #if !NET_2_1
  95         class ProcessTester: ITester
  96         {
  97                 ProcessStartInfo pi;
  98                 string output;
  99
 100                 public ProcessTester (string p_path)
 101                 {
 102                         pi = new ProcessStartInfo ();
 103                         pi.FileName = p_path;
 104                         pi.CreateNoWindow = true;
 105                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 106                         pi.RedirectStandardOutput = true;
 107                         pi.RedirectStandardError = true;
 108                         pi.UseShellExecute = false;
 109                 }
 110
 111                 public string Output {
 112                         get {
 113                                 return output;
 114                         }
 115                 }
 116
 117                 public bool Invoke(string[] args)
 118                 {
 119                         StringBuilder sb = new StringBuilder ("/nologo ");
 120                         foreach (string s in args) {
 121                                 sb.Append (s);
 122                                 sb.Append (" ");
 123                         }
 124                         pi.Arguments = sb.ToString ();
 125                         Process p = Process.Start (pi);
 126                         output = p.StandardError.ReadToEnd ();
 127                         if (output.Length == 0)
 128                             output = p.StandardOutput.ReadToEnd ();
 129                         p.WaitForExit ();
 130                         return p.ExitCode == 0;
 131                 }
 132
 133                 public bool IsWarning (int warningNumber)
 134                 {
 135                         throw new NotImplementedException ();
 136                 }
 137         }
 138 #endif
 139
 140         class TestCase : MarshalByRefObject
 141         {
 142                 public readonly string FileName;
 143                 public readonly string[] CompilerOptions;
 144                 public readonly string[] Dependencies;
 145
 146                 public TestCase (string filename, string[] options, string[] deps)
 147                 {
 148                         this.FileName = filename;
 149                         this.CompilerOptions = options;
 150                         this.Dependencies = deps;
 151                 }
 152         }
 153
 154         class NUnitChecker : PositiveChecker
 155         {
 156                 class TestCaseEntry
 157                 {
 158                         string name;
 159                         string referenceFile;
 160                         string executedMethod;
 161                         bool has_return;
 162
 163                         public TestCaseEntry (string name, string referenceFile, MethodInfo executedMethod)
 164                         {
 165                                 this.name = name.Replace ('-', '_');
 166                                 this.referenceFile = referenceFile;
 167                                 this.executedMethod = ConvertMethodInfoToText (executedMethod, out has_return);
 168                         }
 169
 170                         public string Name
 171                         {
 172                                 get
 173                                 {
 174                                         return name;
 175                                 }
 176                         }
 177
 178                         public string ReferenceFile {
 179                                 get {
 180                                         return referenceFile;
 181                                 }
 182                         }
 183
 184                         static string ConvertMethodInfoToText (MethodInfo mi, out bool hasReturn)
 185                         {
 186                                 hasReturn = mi.ReturnType != typeof (void);
 187                                 string declaring = mi.DeclaringType.FullName.Replace ('+', '.');
 188                                 var param = mi.GetParameters ();
 189                                 if (param.Length == 0)
 190                                         return declaring + "." + mi.Name + " ()";
 191
 192                                 return declaring + "." + mi.Name + " (new string[0])";
 193                         }
 194
 195                         public string GetTestFixture ()
 196                         {
 197                                 var call = name + "::" + executedMethod;
 198                                 if (!has_return)
 199                                         return call;
 200
 201                                 return string.Format ("Assert.AreEqual (0, {0})", call);
 202                         }
 203                 }
 204
 205                 List<TestCaseEntry> entries = new List<TestCaseEntry> ();
 206
 207                 public NUnitChecker (ITester tester)
 208                         : base (tester, null)
 209                 {
 210                 }
 211
 212                 public override void CleanUp ()
 213                 {
 214                         base.CleanUp ();
 215
 216                         StringBuilder aliases = new StringBuilder ();
 217                         var src_dir = Path.Combine ("projects", "MonoTouch");
 218                         string src_file = Path.Combine (src_dir, "tests.cs");
 219
 220                         using (var file = new StreamWriter (src_file, false)) {
 221                                 foreach (var e in entries) {
 222                                         file.WriteLine ("extern alias {0};", e.Name);
 223                                         aliases.AppendFormat ("    <Reference Include=\"{0}\">", Path.GetFileNameWithoutExtension (e.ReferenceFile));
 224                                         aliases.Append (Environment.NewLine);
 225                                         aliases.AppendFormat ("      <Aliases>{0}</Aliases>", e.Name);
 226                                         aliases.Append (Environment.NewLine);
 227                                         aliases.AppendFormat ("      <HintPath>..\\..\\{0}</HintPath>", Path.GetFileName (e.ReferenceFile));
 228                                         aliases.Append (Environment.NewLine);
 229                                         aliases.AppendLine ("    </Reference>");
 230                                 }
 231
 232                                 file.WriteLine ();
 233                                 file.WriteLine ("using NUnit.Framework;");
 234                                 file.WriteLine ();
 235                                 file.WriteLine ("[TestFixture]");
 236                                 file.WriteLine ("public class Tests {");
 237
 238                                 foreach (var e in entries) {
 239                                         file.WriteLine ("\t[Test]");
 240                                         file.WriteLine ("\tpublic void TestFile_{0} ()", e.Name);
 241                                         file.WriteLine ("\t{");
 242                                         file.WriteLine ("\t\t{0};", e.GetTestFixture ());
 243                                         file.WriteLine ("\t}");
 244                                         file.WriteLine ();
 245                                 }
 246
 247                                 file.WriteLine ("}");
 248                         }
 249
 250                         var input = File.ReadAllText (Path.Combine (src_dir, "MonoTouch.csproj.template"));
 251                         input = input.Replace ("@GENERATED_REFERENCES", aliases.ToString ());
 252                         input = input.Replace ("@TEST_SOURCEFILE", Path.GetFileName (src_file));
 253
 254                         File.WriteAllText (Path.Combine (src_dir, "MonoTouch.csproj"), input);
 255                         return;
 256                 }
 257
 258                 protected override bool ExecuteTestFile (TestCase test, string binaryFileName)
 259                 {
 260                         Assembly assembly = Assembly.LoadFile (binaryFileName);
 261                         var ep = assembly.EntryPoint;
 262                         if (!ep.IsPublic) {
 263                                 HandleFailure (test.FileName, TestResult.LoadError, "Entry method is private");
 264                                 return false;
 265                         }
 266
 267                         if (ep.DeclaringType.IsNestedPrivate || ep.DeclaringType.IsNestedFamily) {
 268                                 HandleFailure (test.FileName, TestResult.LoadError, "Entry method in hidden nested type");
 269                                 return false;
 270                         }
 271
 272                         entries.Add (new TestCaseEntry (Path.GetFileNameWithoutExtension (test.FileName), binaryFileName, ep));
 273                         HandleFailure (test.FileName, TestResult.Success, null);
 274                         return true;
 275                 }
 276         }
 277
 278         class PositiveTestCase : TestCase
 279         {
 280                 public class VerificationData : MarshalByRefObject
 281                 {
 282                         public class MethodData : MarshalByRefObject
 283                         {
 284                                 public MethodData (MethodBase mi, int il_size)
 285                                 {
 286                                         this.Type = mi.DeclaringType.ToString ();
 287                                         this.MethodName = mi.ToString ();
 288                                         this.MethodAttributes = (int) mi.Attributes;
 289                                         this.ILSize = il_size;
 290                                 }
 291
 292                                 public MethodData (string type_name, string method_name, int method_attributes, int il_size)
 293                                 {
 294                                         this.Type = type_name;
 295                                         this.MethodName = method_name;
 296                                         this.MethodAttributes = method_attributes;
 297                                         this.ILSize = il_size;
 298                                 }
 299
 300                                 public string Type;
 301                                 public string MethodName;
 302                                 public int ILSize;
 303                                 public bool Checked;
 304                                 public int MethodAttributes;
 305                         }
 306
 307                         ArrayList methods;
 308                         public bool IsNewSet;
 309
 310                         public VerificationData (string test_file)
 311                         {
 312                                 this.test_file = test_file;
 313                         }
 314
 315                         string test_file;
 316
 317                         public static VerificationData FromFile (string name, XmlReader r)
 318                         {
 319                                 VerificationData tc = new VerificationData (name);
 320                                 ArrayList methods = new ArrayList ();
 321                                 r.Read ();
 322                                 while (r.ReadToNextSibling ("type")) {
 323                                         string type_name = r ["name"];
 324                                         r.Read ();
 325                                         while (r.ReadToNextSibling ("method")) {
 326                                                 string m_name = r ["name"];
 327                                                 int method_attrs = int.Parse (r["attrs"]);
 328
 329                                                 r.ReadToDescendant ("size");
 330                                                 int il_size = r.ReadElementContentAsInt ();
 331                                                 methods.Add (new MethodData (type_name, m_name, method_attrs, il_size));
 332                                                 r.Read ();
 333                                         }
 334                                         r.Read ();
 335                                 }
 336
 337                                 tc.methods = methods;
 338                                 return tc;
 339                         }
 340
 341                         public void WriteCodeInfoTo (XmlWriter w)
 342                         {
 343                                 w.WriteStartElement ("test");
 344                                 w.WriteAttributeString ("name", test_file);
 345
 346                                 string type = null;
 347                                 foreach (MethodData data in methods) {
 348                                         if (!data.Checked)
 349                                                 continue;
 350
 351                                         if (type != data.Type) {
 352                                                 if (type != null)
 353                                                         w.WriteEndElement ();
 354
 355                                                 type = data.Type;
 356                                                 w.WriteStartElement ("type");
 357                                                 w.WriteAttributeString ("name", type);
 358                                         }
 359
 360                                         w.WriteStartElement ("method");
 361                                         w.WriteAttributeString ("name", data.MethodName);
 362                                         int v = data.MethodAttributes;
 363                                         w.WriteAttributeString ("attrs", v.ToString ());
 364                                         w.WriteStartElement ("size");
 365                                         w.WriteValue (data.ILSize);
 366                                         w.WriteEndElement ();
 367                                         w.WriteEndElement ();
 368                                 }
 369
 370                                 if (type != null)
 371                                         w.WriteEndElement ();
 372
 373                                 w.WriteEndElement ();
 374                         }
 375
 376                         public MethodData FindMethodData (string method_name, string declaring_type)
 377                         {
 378                                 if (methods == null)
 379                                         return null;
 380
 381                                 foreach (MethodData md in methods) {
 382                                         if (md.MethodName == method_name && md.Type == declaring_type)
 383                                                 return md;
 384                                 }
 385
 386                                 return null;
 387                         }
 388
 389                         public void AddNewMethod (MethodBase mb, int il_size)
 390                         {
 391                                 if (methods == null)
 392                                         methods = new ArrayList ();
 393
 394                                 MethodData md = new MethodData (mb, il_size);
 395                                 md.Checked = true;
 396                                 methods.Add (md);
 397                         }
 398                 }
 399
 400                 VerificationData verif_data;
 401
 402                 public PositiveTestCase (string filename, string [] options, string [] deps)
 403                         : base (filename, options, deps)
 404                 {
 405                 }
 406
 407                 public void CreateNewTest ()
 408                 {
 409                         verif_data = new VerificationData (FileName);
 410                         verif_data.IsNewSet = true;
 411                 }
 412
 413                 public VerificationData VerificationProvider {
 414                         set {
 415                                 verif_data = value;
 416                         }
 417                         get {
 418                                 return verif_data;
 419                         }
 420                 }
 421         }
 422
 423         class Checker: MarshalByRefObject, IDisposable
 424         {
 425                 protected ITester tester;
 426                 protected int success;
 427                 protected int total;
 428                 protected int ignored;
 429                 protected int syntax_errors;
 430                 string issue_file;
 431                 StreamWriter log_file;
 432                 protected string[] extra_compiler_options;
 433                 // protected string[] compiler_options;
 434                 // protected string[] dependencies;
 435
 436                 protected ArrayList tests = new ArrayList ();
 437                 protected Hashtable test_hash = new Hashtable ();
 438                 protected ArrayList regression = new ArrayList ();
 439                 protected ArrayList know_issues = new ArrayList ();
 440                 protected ArrayList ignore_list = new ArrayList ();
 441                 protected ArrayList no_error_list = new ArrayList ();
 442                 ArrayList skip = new ArrayList ();
 443
 444                 protected bool verbose;
 445                 protected bool safe_execution;
 446
 447                 int total_known_issues;
 448
 449                 protected Checker (ITester tester)
 450                 {
 451                         this.tester = tester;
 452                 }
 453
 454                 public string IssueFile {
 455                         set {
 456                                 this.issue_file = value;
 457                                 ReadWrongErrors (issue_file);
 458                         }
 459                 }
 460
 461                 public string LogFile {
 462                         set {
 463                                 this.log_file = new StreamWriter (value, false);
 464                         }
 465                 }
 466
 467                 public bool Verbose {
 468                         set {
 469                                 verbose = value;
 470                         }
 471                 }
 472
 473                 public bool SafeExecution {
 474                         set {
 475                                 safe_execution = value;
 476                         }
 477                 }
 478
 479                 public string[] ExtraCompilerOptions {
 480                         set {
 481                                 extra_compiler_options = value;
 482                         }
 483                 }
 484
 485                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 486                                                         out string[] dependencies)
 487                 {
 488                         int row = 0;
 489                         compiler_options = null;
 490                         dependencies = null;
 491                         try {
 492                                 using (StreamReader sr = new StreamReader (file)) {
 493                                         String line;
 494                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 495                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 496                                                                       ref dependencies))
 497                                                         return false;
 498                                         }
 499                                 }
 500                         } catch {
 501                                 return false;
 502                         }
 503                         return true;
 504                 }
 505
 506                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 507                                                         ref string[] compiler_options,
 508                                                         ref string[] dependencies)
 509                 {
 510                         const string options = "// Compiler options:";
 511                         const string depends = "// Dependencies:";
 512
 513                         if (row == 1) {
 514                                 compiler_options = null;
 515                                 dependencies = null;
 516                         }
 517
 518                         int index = line.IndexOf (options);
 519                         if (index != -1) {
 520                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 521                                 for (int i = 0; i < compiler_options.Length; i++)
 522                                         compiler_options[i] = compiler_options[i].TrimStart ();
 523                         }
 524                         index = line.IndexOf (depends);
 525                         if (index != -1) {
 526                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 527                                 for (int i = 0; i < dependencies.Length; i++)
 528                                         dependencies[i] = dependencies[i].TrimStart ();
 529                         }
 530
 531                         return true;
 532                 }
 533
 534                 public bool Do (string filename)
 535                 {
 536                         if (test_hash.Contains (filename))
 537                                 return true;
 538
 539                         if (verbose)
 540                                 Log (filename + "...\t");
 541
 542                         if (skip.Contains (filename)) {
 543                                 return false;
 544                         }
 545
 546                         if (ignore_list.Contains (filename)) {
 547                                 ++ignored;
 548                                 LogFileLine (filename, "NOT TESTED");
 549                                 return false;
 550                         }
 551
 552                         string[] compiler_options, dependencies;
 553                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 554                                 LogFileLine (filename, "ERROR");
 555                                 return false;
 556                         }
 557
 558                         if (extra_compiler_options != null) {
 559                                 if (compiler_options == null)
 560                                         compiler_options = extra_compiler_options;
 561                                 else {
 562                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 563                                         extra_compiler_options.CopyTo (new_options, 0);
 564                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 565                                         compiler_options = new_options;
 566                                 }
 567                         }
 568
 569                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 570                         test_hash.Add (filename, test);
 571
 572                         ++total;
 573                         if (dependencies != null) {
 574                                 foreach (string dependency in dependencies) {
 575                                         if (!Do (dependency)) {
 576                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 577                                                 return false;
 578                                         }
 579                                 }
 580                         }
 581
 582                         tests.Add (test);
 583
 584                         return Check (test);
 585                 }
 586
 587                 protected virtual bool Check (TestCase test)
 588                 {
 589                         string[] test_args;
 590
 591                         if (test.CompilerOptions != null) {
 592                                 test_args = new string[1 + test.CompilerOptions.Length];
 593                                 test.CompilerOptions.CopyTo (test_args, 0);
 594                         } else {
 595                                 test_args = new string[1];
 596                         }
 597                         test_args[test_args.Length - 1] = test_args[0];
 598                         test_args[0] = test.FileName;
 599
 600                         return tester.Invoke (test_args);
 601                 }
 602
 603                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 604                 {
 605                         return new TestCase (filename, options, deps);
 606                 }
 607
 608                 void ReadWrongErrors (string file)
 609                 {
 610                         const string ignored = "IGNORE";
 611                         const string no_error = "NO ERROR";
 612                         const string skip_tag = "SKIP";
 613
 614                         using (StreamReader sr = new StreamReader (file)) {
 615                                 string line;
 616                                 while ((line = sr.ReadLine()) != null) {
 617                                         if (line.StartsWith ("#"))
 618                                                 continue;
 619
 620                                         ArrayList active_cont = know_issues;
 621
 622                                         if (line.IndexOf (ignored) > 0)
 623                                                 active_cont = ignore_list;
 624                                         else if (line.IndexOf (no_error) > 0)
 625                                                 active_cont = no_error_list;
 626                                         else if (line.Contains (skip_tag))
 627                                                 active_cont = skip;
 628
 629                                         string file_name = line.Split (' ')[0];
 630                                         if (file_name.Length == 0)
 631                                                 continue;
 632
 633                                         active_cont.Add (file_name);
 634                                 }
 635                         }
 636                         total_known_issues = know_issues.Count;
 637                 }
 638
 639                 protected virtual void PrintSummary ()
 640                 {
 641                         LogLine ("Done" + Environment.NewLine);
 642                         float rate = 0;
 643                         if (total > 0)
 644                                 rate = (float) (success) / (float)total;
 645                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 646
 647                         if (syntax_errors > 0)
 648                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 649
 650                         if (ignored > 0)
 651                                 LogLine ("{0} test(s) ignored", ignored);
 652
 653                         if (total_known_issues - know_issues.Count > 0)
 654                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 655
 656                         know_issues.AddRange (no_error_list);
 657                         if (know_issues.Count > 0) {
 658                                 LogLine ("");
 659                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 660                                 foreach (string s in know_issues)
 661                                         LogLine (s);
 662                         }
 663                         if (regression.Count > 0) {
 664                                 LogLine ("");
 665                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 666                                 foreach (string s in regression)
 667                                         LogLine (s);
 668                         }
 669                 }
 670
 671                 public int ResultCode
 672                 {
 673                         get {
 674                                 return regression.Count == 0 ? 0 : 1;
 675                         }
 676                 }
 677
 678                 protected void Log (string msg, params object [] rest)
 679                 {
 680                         Console.Write (msg, rest);
 681                         if (log_file != null)
 682                                 log_file.Write (msg, rest);
 683                 }
 684
 685                 protected void LogLine (string msg)
 686                 {
 687                         Console.WriteLine (msg);
 688                         if (log_file != null)
 689                                 log_file.WriteLine (msg);
 690                 }
 691
 692                 protected void LogLine (string msg, params object [] rest)
 693                 {
 694                         Console.WriteLine (msg, rest);
 695                         if (log_file != null)
 696                                 log_file.WriteLine (msg, rest);
 697                 }
 698
 699                 public void LogFileLine (string file, string msg)
 700                 {
 701                         string s = verbose ? msg : file + "...\t" + msg;
 702
 703                         Console.WriteLine (s);
 704                         if (log_file != null)
 705                                 log_file.WriteLine (s);
 706                 }
 707
 708                 #region IDisposable Members
 709
 710                 public void Dispose()
 711                 {
 712                         if (log_file != null)
 713                                 log_file.Close ();
 714                 }
 715
 716                 #endregion
 717
 718                 public virtual void Initialize ()
 719                 {
 720                 }
 721
 722                 public virtual void CleanUp ()
 723                 {
 724                         PrintSummary ();
 725                 }
 726         }
 727
 728         class PositiveChecker: Checker
 729         {
 730                 readonly string files_folder;
 731                 readonly static object[] default_args = new object[1] { new string[] {} };
 732                 string doc_output;
 733                 string verif_file;
 734                 bool update_verif_file;
 735                 Hashtable verif_data;
 736
 737 #if !NET_2_1
 738                 ProcessStartInfo pi;
 739 #endif
 740                 readonly string mono;
 741
 742                 public enum TestResult {
 743                         CompileError,
 744                         ExecError,
 745                         LoadError,
 746                         XmlError,
 747                         Success,
 748                         ILError,
 749                         DebugError,
 750                         MethodAttributesError
 751                 }
 752
 753                 public PositiveChecker (ITester tester, string verif_file):
 754                         base (tester)
 755                 {
 756                         files_folder = Directory.GetCurrentDirectory ();
 757                         this.verif_file = verif_file;
 758
 759 #if !NET_2_1
 760                         pi = new ProcessStartInfo ();
 761                         pi.CreateNoWindow = true;
 762                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 763                         pi.RedirectStandardOutput = true;
 764                         pi.RedirectStandardError = true;
 765                         pi.UseShellExecute = false;
 766
 767                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 768                         if (mono != null) {
 769                                 pi.FileName = mono;
 770                         }
 771 #endif
 772                 }
 773
 774                 public bool UpdateVerificationDataFile {
 775                         set {
 776                                 update_verif_file = value;
 777                         }
 778                         get {
 779                                 return update_verif_file;
 780                         }
 781                 }
 782
 783                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 784                                                         out string[] dependencies) {
 785                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 786                                 return false;
 787
 788                         doc_output = null;
 789                         if (compiler_options == null)
 790                                 return true;
 791
 792                         foreach (string one_opt in compiler_options) {
 793                                 if (one_opt.StartsWith ("-doc:")) {
 794                                         doc_output = one_opt.Split (':', '/')[1];
 795                                 }
 796                         }
 797                         return true;
 798                 }
 799
 800                 class DomainTester : MarshalByRefObject
 801                 {
 802                         public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
 803                         {
 804                                 Assembly assembly = Assembly.LoadFile (file);
 805
 806                                 bool success = true;
 807                                 Type[] types = assembly.GetTypes ();
 808                                 foreach (Type t in types) {
 809
 810                                         // Skip interfaces
 811                                         if (!t.IsClass && !t.IsValueType)
 812                                                 continue;
 813
 814                                         if (test.VerificationProvider == null) {
 815                                                 if (!checker.UpdateVerificationDataFile)
 816                                                         checker.LogFileLine (test.FileName, "Missing IL verification data");
 817                                                 test.CreateNewTest ();
 818                                         }
 819
 820                                         foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 821                                                 MethodBase mi = m as MethodBase;
 822                                                 if (mi == null)
 823                                                         continue;
 824
 825                                                 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 826                                                         continue;
 827
 828                                                 success &= CompareIL (mi, test, checker);
 829                                         }
 830                                 }
 831
 832                                 return success;
 833                         }
 834
 835                         bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
 836                         {
 837                                 string m_name = mi.ToString ();
 838                                 string decl_type = mi.DeclaringType.ToString ();
 839                                 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
 840
 841                                 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
 842                                 if (md == null) {
 843                                         data_provider.AddNewMethod (mi, GetILSize (mi));
 844                                         if (!data_provider.IsNewSet) {
 845                                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 846                                                 return false;
 847                                         }
 848
 849                                         return true;
 850                                 }
 851
 852                                 if (md.Checked) {
 853                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 854                                         return false;
 855                                 }
 856
 857                                 md.Checked = true;
 858
 859                                 if (md.MethodAttributes != (int) mi.Attributes) {
 860                                         checker.HandleFailure (test.FileName, PositiveChecker.TestResult.MethodAttributesError,
 861                                                 string.Format ("{0} ({1} -> {2})", decl_type + ": " + m_name, (MethodAttributes) md.MethodAttributes, mi.Attributes));
 862                                 }
 863
 864                                 md.MethodAttributes = (int) mi.Attributes;
 865
 866                                 int il_size = GetILSize (mi);
 867                                 if (md.ILSize == il_size)
 868                                         return true;
 869
 870                                 if (md.ILSize > il_size) {
 871                                         checker.LogFileLine (test.FileName, string.Format ("{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
 872                                         md.ILSize = il_size;
 873                                         return true;
 874                                 }
 875
 876                                 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
 877                                         string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
 878
 879                                 md.ILSize = il_size;
 880
 881                                 return false;
 882                         }
 883
 884                         static int GetILSize (MethodBase mi)
 885                         {
 886                                 MethodBody body = mi.GetMethodBody ();
 887                                 if (body != null)
 888                                         return body.GetILAsByteArray ().Length;
 889
 890                                 return 0;
 891                         }
 892
 893                         bool ExecuteFile (MethodInfo entry_point, string filename)
 894                         {
 895                                 TextWriter stdout = Console.Out;
 896                                 TextWriter stderr = Console.Error;
 897                                 Console.SetOut (TextWriter.Null);
 898                                 Console.SetError (TextWriter.Null);
 899                                 ParameterInfo[] pi = entry_point.GetParameters ();
 900                                 object[] args = pi.Length == 0 ? null : default_args;
 901
 902                                 object result = null;
 903                                 try {
 904                                         try {
 905                                                 result = entry_point.Invoke (null, args);
 906                                         } finally {
 907                                                 Console.SetOut (stdout);
 908                                                 Console.SetError (stderr);
 909                                         }
 910                                 } catch (Exception e) {
 911                                         throw new ApplicationException (e.ToString ());
 912                                 }
 913
 914                                 if (result is int && (int) result != 0)
 915                                         throw new ApplicationException ("Wrong return code: " + result.ToString ());
 916
 917                                 return true;
 918                         }
 919
 920                         public bool Test (string file)
 921                         {
 922                                 Assembly assembly = Assembly.LoadFile (file);
 923                                 return ExecuteFile (assembly.EntryPoint, file);
 924                         }
 925                 }
 926
 927                 protected override bool Check(TestCase test)
 928                 {
 929                         string filename = test.FileName;
 930                         try {
 931                                 if (!base.Check (test)) {
 932                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 933                                         return false;
 934                                 }
 935                         }
 936                         catch (Exception e) {
 937                                 if (e.InnerException != null)
 938                                         e = e.InnerException;
 939
 940                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 941                                 return false;
 942                         }
 943
 944                         // Test setup
 945                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 946                                 if (verbose)
 947                                         LogFileLine (filename, "OK");
 948                                 --total;
 949                                 return true;
 950                         }
 951
 952                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 953
 954                         // Enable .dll only tests (no execution required)
 955                         if (!File.Exists(file)) {
 956                                 HandleFailure (filename, TestResult.Success, null);
 957                                 return true;
 958                         }
 959
 960                         return ExecuteTestFile (test, file);
 961                 }
 962
 963                 protected virtual bool ExecuteTestFile (TestCase test, string binaryFileName)
 964                 {
 965                         string filename = test.FileName;
 966
 967                         AppDomain domain = null;
 968 #if !NET_2_1
 969                         if (safe_execution) {
 970                                 // Create a new AppDomain, with the current directory as the base.
 971                                 AppDomainSetup setupInfo = new AppDomainSetup ();
 972                                 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
 973                                 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
 974                                 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (binaryFileName), null, setupInfo);
 975                         }
 976 #endif
 977                         try {
 978                                 DomainTester tester;
 979                                 try {
 980 #if !NET_2_1
 981                                         if (domain != null)
 982                                                 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
 983                                         else
 984 #endif
 985                                                 tester = new DomainTester ();
 986
 987                                         if (!tester.Test (binaryFileName))
 988                                                 return false;
 989
 990                                 } catch (ApplicationException e) {
 991                                         HandleFailure (filename, TestResult.ExecError, e.Message);
 992                                         return false;
 993                                 } catch (Exception e) {
 994                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 995                                         return false;
 996                                 }
 997
 998                                 if (doc_output != null) {
 999                                         string ref_file = filename.Replace (".cs", "-ref.xml");
1000                                         try {
1001 #if !NET_2_1
1002                                                 new XmlComparer ("doc").Compare (ref_file, doc_output);
1003 #endif
1004                                         } catch (Exception e) {
1005                                                 HandleFailure (filename, TestResult.XmlError, e.Message);
1006                                                 return false;
1007                                         }
1008                                 } else {
1009                                         if (verif_file != null) {
1010                                                 PositiveTestCase pt = (PositiveTestCase) test;
1011                                                 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
1012
1013                                                 if (!tester.CheckILSize (pt, this, binaryFileName))
1014                                                         return false;
1015                                         }
1016
1017                                         if (filename.StartsWith ("test-debug", StringComparison.OrdinalIgnoreCase)) {
1018                                                 var mdb_file_name = binaryFileName + ".mdb";
1019                                                 MonoSymbolFile mdb_file = MonoSymbolFile.ReadSymbolFile (mdb_file_name);
1020                                                 var mdb_xml_file = mdb_file_name + ".xml";
1021                                                 ConvertSymbolFileToXml (mdb_file, mdb_xml_file);
1022
1023                                                 var ref_file = filename.Replace(".cs", "-ref.xml");
1024                                                 try {
1025                                                         new XmlComparer ("symbols").Compare (ref_file, mdb_xml_file);
1026                                                 } catch (Exception e) {
1027                                                         HandleFailure (filename, TestResult.DebugError, e.Message);
1028                                                         return false;
1029                                                 }
1030                                         }
1031
1032                                 }
1033                         } finally {
1034                                 if (domain != null)
1035                                         AppDomain.Unload (domain);
1036                         }
1037
1038                         HandleFailure (filename, TestResult.Success, null);
1039                         return true;
1040                 }
1041
1042                 static void ConvertSymbolFileToXml (MonoSymbolFile symbolFile, string xmlFile)
1043                 {
1044                         using (XmlTextWriter writer = new XmlTextWriter (xmlFile, Encoding.UTF8)) {
1045                                 writer.Formatting = Formatting.Indented;
1046
1047                                 writer.WriteStartDocument ();
1048
1049                                 writer.WriteStartElement ("symbols");
1050
1051                                 writer.WriteStartElement ("files");
1052                                 foreach (var file in symbolFile.Sources) {
1053                                         writer.WriteStartElement ("file");
1054                                         writer.WriteAttributeString ("id", file.Index.ToString ());
1055                                         writer.WriteAttributeString ("name", Path.GetFileName (file.FileName));
1056                                         var checksum = file.Checksum;
1057                                         if (checksum != null)
1058                                                 writer.WriteAttributeString ("checksum", ChecksumToString (checksum));
1059
1060                                         writer.WriteEndElement ();
1061                                 }
1062                                 writer.WriteEndElement ();
1063
1064                                 writer.WriteStartElement ("methods");
1065                                 foreach (var method in symbolFile.Methods) {
1066                                         writer.WriteStartElement ("method");
1067                                         writer.WriteAttributeString ("token", IntToHex (method.Token));
1068
1069                                         var il_entries = method.GetLineNumberTable ();
1070                                         writer.WriteStartElement ("sequencepoints");
1071                                         foreach (var entry in il_entries.LineNumbers) {
1072                                                 writer.WriteStartElement ("entry");
1073                                                 writer.WriteAttributeString ("il", IntToHex (entry.Offset));
1074                                                 writer.WriteAttributeString ("row", entry.Row.ToString ());
1075                                                 writer.WriteAttributeString ("col", entry.Column.ToString ());
1076                                                 writer.WriteAttributeString ("file_ref", entry.File.ToString ());
1077                                                 writer.WriteAttributeString ("hidden", BoolToString (entry.IsHidden));
1078                                                 writer.WriteEndElement ();
1079                                         }
1080                                         writer.WriteEndElement ();
1081
1082                                         writer.WriteStartElement ("locals");
1083                                         foreach (var local in method.GetLocals ()) {
1084                                                 writer.WriteStartElement ("entry");
1085                                                 writer.WriteAttributeString ("name", local.Name);
1086                                                 writer.WriteAttributeString ("il_index", local.Index.ToString ());
1087                                                 writer.WriteAttributeString ("scope_ref", local.BlockIndex.ToString ());
1088                                                 writer.WriteEndElement ();
1089                                         }
1090                                         writer.WriteEndElement ();
1091
1092                                         writer.WriteStartElement ("scopes");
1093                                         foreach (var scope in method.GetCodeBlocks ()) {
1094                                                 writer.WriteStartElement ("entry");
1095                                                 writer.WriteAttributeString ("index", scope.Index.ToString ());
1096                                                 writer.WriteAttributeString ("start", IntToHex (scope.StartOffset));
1097                                                 writer.WriteAttributeString ("end", IntToHex (scope.EndOffset));
1098                                                 writer.WriteEndElement ();
1099                                         }
1100                                         writer.WriteEndElement ();
1101
1102                                         writer.WriteEndElement ();
1103                                 }
1104                                 writer.WriteEndElement ();
1105
1106                                 writer.WriteEndElement ();
1107                                 writer.WriteEndDocument ();
1108                         }
1109                 }
1110
1111                 static string ChecksumToString (byte[] checksum)
1112                 {
1113                         var sb = new StringBuilder (checksum.Length * 2);
1114                         for (int i = 0; i < checksum.Length; i++) {
1115                                 sb.Append ("0123456789abcdef"[checksum[i] >> 4]);
1116                                 sb.Append ("0123456789abcdef"[checksum[i] & 0x0F]);
1117                         }
1118
1119                         return sb.ToString ();
1120                 }
1121
1122                 static string IntToHex (int value)
1123                 {
1124                         return "0x" + value.ToString ("x", CultureInfo.InvariantCulture);
1125                 }
1126
1127                 static string BoolToString (bool value)
1128                 {
1129                         return value ? "true" : "false";
1130                 }
1131
1132                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
1133                 {
1134                         return new PositiveTestCase (filename, options, deps);
1135                 }
1136
1137                 public void HandleFailure (string file, TestResult status, string extra)
1138                 {
1139                         switch (status) {
1140                                 case TestResult.Success:
1141                                         success++;
1142                                         if (know_issues.Contains (file)) {
1143                                                 LogFileLine (file, "FIXED ISSUE");
1144                                                 return;
1145                                         }
1146                                         if (verbose)
1147                                                 LogFileLine (file, "OK");
1148                                         return;
1149
1150                                 case TestResult.CompileError:
1151                                         if (know_issues.Contains (file)) {
1152                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
1153                                                 know_issues.Remove (file);
1154                                                 return;
1155                                         }
1156                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
1157                                         break;
1158
1159                                 case TestResult.ExecError:
1160                                         if (know_issues.Contains (file)) {
1161                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
1162                                                 know_issues.Remove (file);
1163                                                 return;
1164                                         }
1165                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
1166                                         break;
1167
1168                                 case TestResult.XmlError:
1169                                         if (know_issues.Contains (file)) {
1170                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
1171                                                 know_issues.Remove (file);
1172                                                 return;
1173                                         }
1174                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
1175                                         break;
1176
1177                                 case TestResult.LoadError:
1178                                         if (extra != null)
1179                                                 extra = ": " + extra;
1180
1181                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)" + extra);
1182                                         extra = null;
1183                                         break;
1184
1185                                 case TestResult.MethodAttributesError:
1186                                 case TestResult.ILError:
1187                                         if (!update_verif_file) {
1188                                                 LogFileLine (file, "IL REGRESSION: " + extra);
1189                                         }
1190                                         extra = null;
1191                                         break;
1192
1193                                 case TestResult.DebugError:
1194                                         LogFileLine (file, "REGRESSION (SUCCESS -> SYMBOL FILE ERROR)");
1195                                         break;
1196                         }
1197
1198                         if (extra != null)
1199                                 LogLine ("{0}", extra);
1200
1201                         if (!regression.Contains (file))
1202                                 regression.Add (file);
1203                 }
1204
1205                 public override void Initialize ()
1206                 {
1207                         if (verif_file != null) {
1208                                 LoadVerificationData (verif_file);
1209                         }
1210
1211                         base.Initialize ();
1212                 }
1213
1214                 public override void CleanUp ()
1215                 {
1216                         base.CleanUp ();
1217
1218                         if (update_verif_file) {
1219                                 UpdateVerificationData (verif_file);
1220                         }
1221                 }
1222
1223                 void LoadVerificationData (string file)
1224                 {
1225                         verif_data = new Hashtable ();
1226
1227                         if (!File.Exists (file)) {
1228                                 LogLine ("Writing verification data to `{0}' ...", file);
1229                                 return;
1230                         }
1231
1232                         LogLine ("Loading verification data from `{0}' ...", file);
1233
1234                         using (XmlReader r = XmlReader.Create (file)) {
1235                                 r.ReadStartElement ("tests");
1236
1237                                 while (r.Read ()) {
1238                                         if (r.Name != "test")
1239                                                 continue;
1240
1241                                         string name = r.GetAttribute ("name");
1242                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
1243                                         verif_data.Add (name, tc);
1244                                 }
1245                         }
1246                 }
1247
1248                 void UpdateVerificationData (string file)
1249                 {
1250                         LogLine ("Updating verification data `{0}' ...", file);
1251
1252                         XmlWriterSettings s = new XmlWriterSettings ();
1253                         s.Indent = true;
1254                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
1255                                 w.WriteStartDocument ();
1256                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
1257                                 w.WriteStartElement ("tests");
1258                                 foreach (PositiveTestCase tc in tests) {
1259                                         if (tc.VerificationProvider != null)
1260                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
1261                                 }
1262                                 w.WriteEndElement ();
1263                         }
1264                 }
1265         }
1266
1267         class NegativeChecker: Checker
1268         {
1269                 string expected_message;
1270                 string error_message;
1271                 bool check_msg;
1272                 bool check_error_line;
1273                 bool is_warning;
1274                 IDictionary wrong_warning;
1275
1276                 protected enum CompilerError {
1277                         Expected,
1278                         Wrong,
1279                         Missing,
1280                         WrongMessage,
1281                         MissingLocation,
1282                         Duplicate
1283                 }
1284
1285                 public NegativeChecker (ITester tester, bool check_msg):
1286                         base (tester)
1287                 {
1288                         this.check_msg = check_msg;
1289                         wrong_warning = new Hashtable ();
1290                 }
1291
1292                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1293                                                         ref string[] compiler_options,
1294                                                         ref string[] dependencies)
1295                 {
1296                         if (row == 1) {
1297                                 expected_message = null;
1298
1299                                 int index = line.IndexOf (':');
1300                                 if (index == -1 || index > 15) {
1301                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1302                                         ++syntax_errors;
1303                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1304                                                               ref dependencies);
1305                                         return false;
1306                                 }
1307
1308                                 expected_message = line.Substring (index + 1).Trim ();
1309                         }
1310
1311                         if (row == 2) {
1312                                 string filtered = line.Replace(" ", "");
1313
1314                                 // Some error tests require to have different error text for different runtimes.
1315                                 if (filtered.StartsWith ("//GMCS")) {
1316                                         row = 1;
1317                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1318                                 }
1319
1320                                 check_error_line = !filtered.StartsWith ("//Line:0");
1321
1322                                 if (!filtered.StartsWith ("//Line:")) {
1323                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1324                                         ++syntax_errors;
1325                                         return false;
1326                                 }
1327                         }
1328
1329                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1330                                 return false;
1331
1332                         is_warning = false;
1333                         if (compiler_options != null) {
1334                                 foreach (string s in compiler_options) {
1335                                         if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1336                                                 is_warning = true;
1337                                 }
1338                         }
1339                         return true;
1340                 }
1341
1342
1343                 protected override bool Check (TestCase test)
1344                 {
1345                         string filename = test.FileName;
1346
1347                         int start_char = 0;
1348                         while (Char.IsLetter (filename, start_char))
1349                                 ++start_char;
1350
1351                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1352                         string expected = filename.Substring (start_char, end_char - start_char);
1353
1354                         try {
1355                                 if (base.Check (test)) {
1356                                         HandleFailure (filename, CompilerError.Missing);
1357                                         return false;
1358                                 }
1359                         }
1360                         catch (Exception e) {
1361                                 HandleFailure (filename, CompilerError.Missing);
1362                                 if (e.InnerException != null)
1363                                         e = e.InnerException;
1364
1365                                 Log (e.ToString ());
1366                                 return false;
1367                         }
1368
1369                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1370                         if (tester.IsWarning (err_id)) {
1371                                 if (!is_warning)
1372                                         wrong_warning [err_id] = true;
1373                         } else {
1374                                 if (is_warning)
1375                                         wrong_warning [err_id] = false;
1376                         }
1377
1378                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1379                         if (HandleFailure (filename, result_code)) {
1380                                 success++;
1381                                 return true;
1382                         }
1383
1384                         if (result_code == CompilerError.Wrong)
1385                                 LogLine (tester.Output);
1386
1387                         return false;
1388                 }
1389
1390                 CompilerError GetCompilerError (string expected, string buffer)
1391                 {
1392                         const string error_prefix = "CS";
1393                         const string ignored_error = "error CS5001";
1394                         string tested_text = "error " + error_prefix + expected;
1395                         StringReader sr = new StringReader (buffer);
1396                         string line = sr.ReadLine ();
1397                         ArrayList ld = new ArrayList ();
1398                         CompilerError result = CompilerError.Missing;
1399                         while (line != null) {
1400                                 if (ld.Contains (line) && result == CompilerError.Expected) {
1401                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1402                                                 return CompilerError.Duplicate;
1403                                 }
1404                                 ld.Add (line);
1405
1406                                 if (result != CompilerError.Expected) {
1407                                         if (line.IndexOf (tested_text) != -1) {
1408                                                 if (check_msg) {
1409                                                         int first = line.IndexOf (':');
1410                                                         int second = line.IndexOf (':', first + 1);
1411                                                         if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
1412                                                                 if (check_error_line) {
1413                                                                         second = line.IndexOf (':', second + 1);
1414                                                                 }
1415                                                         } else if (second == -1 || !check_error_line) {
1416                                                                 second = first;
1417                                                         }
1418
1419                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1420                                                         if (msg != expected_message && !TryToMatchErrorMessage (msg, expected_message)) {
1421                                                                 error_message = msg;
1422                                                                 return CompilerError.WrongMessage;
1423                                                         }
1424
1425                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1426                                                                 return CompilerError.MissingLocation;
1427                                                 }
1428                                                 result = CompilerError.Expected;
1429                                         } else if (line.IndexOf (error_prefix) != -1 &&
1430                                                 line.IndexOf (ignored_error) == -1)
1431                                                 result = CompilerError.Wrong;
1432                                 }
1433
1434                                 line = sr.ReadLine ();
1435                         }
1436
1437                         return result;
1438                 }
1439
1440                 static bool TryToMatchErrorMessage (string actual, string expected)
1441                 {
1442                         actual = actual.Replace ("\\", "/");
1443                         var path_mask_start = expected.IndexOf ("*PATH*");
1444                         if (path_mask_start > 0 && actual.Length > path_mask_start) {
1445                                 var path_mask_continue = expected.Substring (path_mask_start + 6);
1446                                 var expected_continue = actual.IndexOf (path_mask_continue, path_mask_start);
1447                                 if (expected_continue > 0) {
1448                                         var path = actual.Substring (path_mask_start, expected_continue - path_mask_start);
1449                                         if (actual == expected.Replace ("*PATH*", path))
1450                                                 return true;
1451
1452                                         throw new ApplicationException (expected.Replace ("*PATH*", path));
1453                                 }
1454                         }
1455
1456                         return false;
1457                 }
1458
1459                 bool HandleFailure (string file, CompilerError status)
1460                 {
1461                         switch (status) {
1462                                 case CompilerError.Expected:
1463                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1464                                                 LogFileLine (file, "FIXED ISSUE");
1465                                                 return true;
1466                                         }
1467
1468                                         if (verbose)
1469                                                 LogFileLine (file, "OK");
1470                                         return true;
1471
1472                                 case CompilerError.Wrong:
1473                                         if (know_issues.Contains (file)) {
1474                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1475                                                 know_issues.Remove (file);
1476                                                 return false;
1477                                         }
1478                                         if (no_error_list.Contains (file)) {
1479                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1480                                                 no_error_list.Remove (file);
1481                                         }
1482                                         else {
1483                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1484                                         }
1485                                         break;
1486
1487                                 case CompilerError.WrongMessage:
1488                                         if (know_issues.Contains (file)) {
1489                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1490                                                 know_issues.Remove (file);
1491                                                 return false;
1492                                         }
1493                                         if (no_error_list.Contains (file)) {
1494                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1495                                                 no_error_list.Remove (file);
1496                                         }
1497                                         else {
1498                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1499                                                 LogLine ("Exp: {0}", expected_message);
1500                                                 LogLine ("Was: {0}", error_message);
1501                                         }
1502                                         break;
1503
1504                                 case CompilerError.Missing:
1505                                         if (no_error_list.Contains (file)) {
1506                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1507                                                 no_error_list.Remove (file);
1508                                                 return false;
1509                                         }
1510
1511                                         if (know_issues.Contains (file)) {
1512                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1513                                                 know_issues.Remove (file);
1514                                         }
1515                                         else {
1516                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1517                                         }
1518
1519                                         break;
1520
1521                                 case CompilerError.MissingLocation:
1522                                         if (know_issues.Contains (file)) {
1523                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1524                                                 know_issues.Remove (file);
1525                                                 return false;
1526                                         }
1527                                         if (no_error_list.Contains (file)) {
1528                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1529                                                 no_error_list.Remove (file);
1530                                         }
1531                                         else {
1532                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1533                                         }
1534                                         break;
1535
1536                                 case CompilerError.Duplicate:
1537                                         // Will become an error soon
1538                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1539                                         return true;
1540                         }
1541
1542                         regression.Add (file);
1543                         return false;
1544                 }
1545
1546                 protected override void PrintSummary()
1547                 {
1548                         base.PrintSummary ();
1549
1550                         if (wrong_warning.Count > 0) {
1551                                 LogLine ("");
1552                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1553                                 LogLine ("");
1554                                 foreach (DictionaryEntry de in wrong_warning)
1555                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1556                         }
1557                 }
1558
1559         }
1560
1561         class Tester {
1562
1563                 static int Main(string[] args)
1564                 {
1565                         string temp;
1566
1567                         if (GetOption ("help", args, false, out temp)) {
1568                                 Usage ();
1569                                 return 0;
1570                         }
1571
1572                         string compiler;
1573                         if (!GetOption ("compiler", args, true, out compiler)) {
1574                                 Usage ();
1575                                 return 1;
1576                         }
1577
1578                         ITester tester;
1579                         try {
1580                                 Console.WriteLine ("Loading " + compiler + " ...");
1581                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1582                         } catch (Exception) {
1583                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1584                                 if (!File.Exists (compiler)) {
1585                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1586                                         return 1;
1587                                 }
1588                                 tester = new ProcessTester (compiler);
1589                         }
1590
1591                         string mode;
1592                         if (!GetOption ("mode", args, true, out mode)) {
1593                                 Usage ();
1594                                 return 1;
1595                         }
1596
1597                         Checker checker;
1598                         bool positive;
1599                         switch (mode) {
1600                                 case "neg":
1601                                         checker = new NegativeChecker (tester, true);
1602                                         positive = false;
1603                                         break;
1604                                 case "pos":
1605                                         string iltest;
1606                                         GetOption ("il", args, false, out iltest);
1607                                         checker = new PositiveChecker (tester, iltest);
1608                                         positive = true;
1609
1610                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1611                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1612                                         }
1613
1614                                         break;
1615                                 case "nunit":
1616                                         positive = true;
1617                                         checker = new NUnitChecker (tester);
1618                                         break;
1619                                 default:
1620                                         Console.Error.WriteLine ("Invalid -mode argument");
1621                                         return 1;
1622                         }
1623
1624
1625                         if (GetOption ("issues", args, true, out temp))
1626                                 checker.IssueFile = temp;
1627                         if (GetOption ("log", args, true, out temp))
1628                                 checker.LogFile = temp;
1629                         if (GetOption ("verbose", args, false, out temp))
1630                                 checker.Verbose = true;
1631                         if (GetOption ("safe-execution", args, false, out temp))
1632                                 checker.SafeExecution = true;
1633                         if (GetOption ("compiler-options", args, true, out temp)) {
1634                                 string[] extra = temp.Split (' ');
1635                                 checker.ExtraCompilerOptions = extra;
1636                         }
1637
1638                         string test_pattern;
1639                         if (!GetOption ("files", args, true, out test_pattern)) {
1640                                 Usage ();
1641                                 return 1;
1642                         }
1643
1644                         var files = new List<string> ();
1645                         switch (test_pattern) {
1646                         case "v1":
1647                                 files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
1648                                 break;
1649                         case "v2":
1650                                 files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
1651                                 goto case "v1";
1652                         case "v4":
1653                                 files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
1654                                 goto case "v2";
1655                         default:
1656                                 files.AddRange (Directory.GetFiles (".", test_pattern));
1657                                 break;
1658                         }
1659
1660                         if (files.Count == 0) {
1661                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1662                                 return 2;
1663                         }
1664
1665                         checker.Initialize ();
1666 /*
1667                         files.Sort ((a, b) => {
1668                                 if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1669                                         if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
1670                                                 return -1;
1671                                 } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1672                                         if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
1673                                                 return 1;
1674                                 }
1675
1676                                 return a.CompareTo (b);
1677                         });
1678 */
1679                         foreach (string s in files) {
1680                                 string filename = Path.GetFileName (s);
1681                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1682                                         continue;
1683                                 }
1684
1685                                 if (filename.EndsWith ("-p2.cs"))
1686                                         continue;
1687
1688                                 checker.Do (filename);
1689                         }
1690
1691                         checker.CleanUp ();
1692
1693                         checker.Dispose ();
1694
1695                         return checker.ResultCode;
1696                 }
1697
1698                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1699                 {
1700                         opt = "-" + opt;
1701                         foreach (string a in args) {
1702                                 if (a.StartsWith (opt)) {
1703                                         int sep = a.IndexOf (':');
1704                                         if (sep > 0) {
1705                                                 value = a.Substring (sep + 1);
1706                                         } else {
1707                                                 value = null;
1708                                                 if (req_arg) {
1709                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1710                                                         return false;
1711                                                 }
1712                                         }
1713
1714                                         return true;
1715                                 }
1716                         }
1717
1718                         value = null;
1719                         return false;
1720                 }
1721
1722                 static void Usage ()
1723                 {
1724                         Console.WriteLine (
1725                                 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1726                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1727                                 "   \n" +
1728                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1729                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1730                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1731                                 "   -issues:FILE     The list of expected failures\n" +
1732                                 "   -log:FILE        Writes any output also to the file\n" +
1733                                 "   -help            Lists all options\n" +
1734                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1735                                 "   -safe-execution  Runs compiled executables in separate app-domain\n" +
1736                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1737                                 "   -verbose         Prints more details during testing\n"
1738                                 );
1739                 }
1740         }
1741 }