mcs/tools/compiler-tester/compiler-tester.cs

   1 //
   2 // compiler-tester.cs
   3 //
   4 // Author:
   5 //   Marek Safar (marek.safar@gmail.com)
   6 //
   7
   8 //
   9 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
  10 //
  11 // Permission is hereby granted, free of charge, to any person obtaining
  12 // a copy of this software and associated documentation files (the
  13 // "Software"), to deal in the Software without restriction, including
  14 // without limitation the rights to use, copy, modify, merge, publish,
  15 // distribute, sublicense, and/or sell copies of the Software, and to
  16 // permit persons to whom the Software is furnished to do so, subject to
  17 // the following conditions:
  18 //
  19 // The above copyright notice and this permission notice shall be
  20 // included in all copies or substantial portions of the Software.
  21 //
  22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29 //
  30
  31 using System;
  32 using System.IO;
  33 using System.Diagnostics;
  34 using System.Reflection;
  35 using System.Text;
  36 using System.Collections;
  37 using System.Xml;
  38
  39 namespace TestRunner {
  40
  41         interface ITester
  42         {
  43                 string Output { get; }
  44                 bool Invoke (string[] args);
  45                 bool IsWarning (int warningNumber);
  46         }
  47
  48         class ReflectionTester: ITester {
  49                 MethodInfo ep;
  50                 object[] method_arg;
  51                 StringWriter output;
  52                 int[] all_warnings;
  53
  54                 public ReflectionTester (Assembly a)
  55                 {
  56                         Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
  57
  58                         if (t == null)
  59                                 Console.Error.WriteLine ("null, huh?");
  60
  61                         ep = t.GetMethod ("InvokeCompiler",
  62                                 BindingFlags.Static | BindingFlags.Public);
  63                         if (ep == null)
  64                                 throw new MissingMethodException ("static InvokeCompiler");
  65                         method_arg = new object [2];
  66
  67                         PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
  68                         all_warnings = (int[])pi.GetValue (null, null);
  69                         Array.Sort (all_warnings);
  70                 }
  71
  72                 public string Output {
  73                         get {
  74                                 return output.GetStringBuilder ().ToString ();
  75                         }
  76                 }
  77
  78                 public bool Invoke(string[] args)
  79                 {
  80                         output = new StringWriter ();
  81                         method_arg [0] = args;
  82                         method_arg [1] = output;
  83                         return (bool)ep.Invoke (null, method_arg);
  84                 }
  85
  86                 public bool IsWarning (int warningNumber)
  87                 {
  88                         return Array.BinarySearch (all_warnings, warningNumber) >= 0;
  89                 }
  90         }
  91
  92 #if !NET_2_1
  93         class ProcessTester: ITester
  94         {
  95                 ProcessStartInfo pi;
  96                 string output;
  97
  98                 public ProcessTester (string p_path)
  99                 {
 100                         pi = new ProcessStartInfo ();
 101                         pi.FileName = p_path;
 102                         pi.CreateNoWindow = true;
 103                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 104                         pi.RedirectStandardOutput = true;
 105                         pi.RedirectStandardError = true;
 106                         pi.UseShellExecute = false;
 107                 }
 108
 109                 public string Output {
 110                         get {
 111                                 return output;
 112                         }
 113                 }
 114
 115                 public bool Invoke(string[] args)
 116                 {
 117                         StringBuilder sb = new StringBuilder ("/nologo ");
 118                         foreach (string s in args) {
 119                                 sb.Append (s);
 120                                 sb.Append (" ");
 121                         }
 122                         pi.Arguments = sb.ToString ();
 123                         Process p = Process.Start (pi);
 124                         output = p.StandardError.ReadToEnd ();
 125                         if (output.Length == 0)
 126                             output = p.StandardOutput.ReadToEnd ();
 127                         p.WaitForExit ();
 128                         return p.ExitCode == 0;
 129                 }
 130
 131                 public bool IsWarning (int warningNumber)
 132                 {
 133                         throw new NotImplementedException ();
 134                 }
 135         }
 136 #endif
 137
 138         class TestCase
 139         {
 140                 public readonly string FileName;
 141                 public readonly string[] CompilerOptions;
 142                 public readonly string[] Dependencies;
 143
 144                 public TestCase (string filename, string[] options, string[] deps)
 145                 {
 146                         this.FileName = filename;
 147                         this.CompilerOptions = options;
 148                         this.Dependencies = deps;
 149                 }
 150         }
 151
 152         class PositiveTestCase : TestCase
 153         {
 154                 public class VerificationData
 155                 {
 156                         public class MethodData
 157                         {
 158                                 public MethodData (MethodBase mi, int il_size)
 159                                 {
 160                                         this.Type = mi.DeclaringType.ToString ();
 161                                         this.MethodName = mi.ToString ();
 162                                         this.ILSize = il_size;
 163                                 }
 164
 165                                 public MethodData (string type_name, string method_name, int il_size)
 166                                 {
 167                                         this.Type = type_name;
 168                                         this.MethodName = method_name;
 169                                         this.ILSize = il_size;
 170                                 }
 171
 172                                 public string Type;
 173                                 public string MethodName;
 174                                 public int ILSize;
 175                                 public bool Checked;
 176                         }
 177
 178                         ArrayList methods;
 179                         public bool IsNewSet;
 180
 181                         public VerificationData (string test_file)
 182                         {
 183 #if NET_2_0
 184                                 this.test_file = test_file;
 185 #endif
 186                         }
 187
 188 #if NET_2_0
 189                         string test_file;
 190
 191                         public static VerificationData FromFile (string name, XmlReader r)
 192                         {
 193                                 VerificationData tc = new VerificationData (name);
 194                                 ArrayList methods = new ArrayList ();
 195                                 r.Read ();
 196                                 while (r.ReadToNextSibling ("type")) {
 197                                         string type_name = r ["name"];
 198                                         r.Read ();
 199                                         while (r.ReadToNextSibling ("method")) {
 200                                                 string m_name = r ["name"];
 201
 202                                                 r.ReadToDescendant ("size");
 203                                                 int il_size = r.ReadElementContentAsInt ();
 204                                                 methods.Add (new MethodData (type_name, m_name, il_size));
 205                                                 r.Read ();
 206                                         }
 207                                         r.Read ();
 208                                 }
 209
 210                                 tc.methods = methods;
 211                                 return tc;
 212                         }
 213
 214                         public void WriteCodeInfoTo (XmlWriter w)
 215                         {
 216                                 w.WriteStartElement ("test");
 217                                 w.WriteAttributeString ("name", test_file);
 218
 219                                 string type = null;
 220                                 foreach (MethodData data in methods) {
 221                                         if (!data.Checked)
 222                                                 continue;
 223
 224                                         if (type != data.Type) {
 225                                                 if (type != null)
 226                                                         w.WriteEndElement ();
 227
 228                                                 type = data.Type;
 229                                                 w.WriteStartElement ("type");
 230                                                 w.WriteAttributeString ("name", type);
 231                                         }
 232
 233                                         w.WriteStartElement ("method");
 234                                         w.WriteAttributeString ("name", data.MethodName);
 235                                         w.WriteStartElement ("size");
 236                                         w.WriteValue (data.ILSize);
 237                                         w.WriteEndElement ();
 238                                         w.WriteEndElement ();
 239                                 }
 240
 241                                 if (type != null)
 242                                         w.WriteEndElement ();
 243
 244                                 w.WriteEndElement ();
 245                         }
 246 #endif
 247
 248                         public MethodData FindMethodData (string method_name, string declaring_type)
 249                         {
 250                                 if (methods == null)
 251                                         return null;
 252
 253                                 foreach (MethodData md in methods) {
 254                                         if (md.MethodName == method_name && md.Type == declaring_type)
 255                                                 return md;
 256                                 }
 257
 258                                 return null;
 259                         }
 260
 261                         public void AddNewMethod (MethodBase mb, int il_size)
 262                         {
 263                                 if (methods == null)
 264                                         methods = new ArrayList ();
 265
 266                                 MethodData md = new MethodData (mb, il_size);
 267                                 md.Checked = true;
 268                                 methods.Add (md);
 269                         }
 270                 }
 271
 272                 VerificationData verif_data;
 273
 274                 public PositiveTestCase (string filename, string [] options, string [] deps)
 275                         : base (filename, options, deps)
 276                 {
 277                 }
 278
 279                 public void CreateNewTest ()
 280                 {
 281                         verif_data = new VerificationData (FileName);
 282                         verif_data.IsNewSet = true;
 283                 }
 284
 285                 public VerificationData VerificationProvider {
 286                         set {
 287                                 verif_data = value;
 288                         }
 289                         get {
 290                                 return verif_data;
 291                         }
 292                 }
 293
 294                 public bool CompareIL (MethodBase mi, PositiveChecker checker)
 295                 {
 296                         string m_name = mi.ToString ();
 297                         string decl_type = mi.DeclaringType.ToString ();
 298                         VerificationData.MethodData md = verif_data.FindMethodData (m_name, decl_type);
 299                         if (md == null) {
 300                                 verif_data.AddNewMethod (mi, GetILSize (mi));
 301                                 if (!verif_data.IsNewSet) {
 302                                         checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
 303                                         return false;
 304                                 }
 305
 306                                 return true;
 307                         }
 308
 309                         if (md.Checked) {
 310                                 checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
 311                                 return false;
 312                         }
 313
 314                         md.Checked = true;
 315
 316                         int il_size = GetILSize (mi);
 317                         if (md.ILSize == il_size)
 318                                 return true;
 319
 320                         if (md.ILSize > il_size) {
 321                                 checker.LogFileLine (FileName, "{0} (code size reduction {1} -> {2})", m_name, md.ILSize, il_size);
 322                                 md.ILSize = il_size;
 323                                 return true;
 324                         }
 325
 326                         checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError,
 327                                 string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
 328
 329                         md.ILSize = il_size;
 330
 331                         return false;
 332                 }
 333
 334                 static int GetILSize (MethodBase mi)
 335                 {
 336 #if NET_2_0
 337                         MethodBody body = mi.GetMethodBody ();
 338                         if (body != null)
 339                                 return body.GetILAsByteArray ().Length;
 340 #endif
 341                         return 0;
 342                 }
 343         }
 344
 345         class Checker: IDisposable
 346         {
 347                 protected ITester tester;
 348                 protected int success;
 349                 protected int total;
 350                 protected int ignored;
 351                 protected int syntax_errors;
 352                 string issue_file;
 353                 StreamWriter log_file;
 354                 protected string[] extra_compiler_options;
 355                 // protected string[] compiler_options;
 356                 // protected string[] dependencies;
 357
 358                 protected ArrayList tests = new ArrayList ();
 359                 protected Hashtable test_hash = new Hashtable ();
 360                 protected ArrayList regression = new ArrayList ();
 361                 protected ArrayList know_issues = new ArrayList ();
 362                 protected ArrayList ignore_list = new ArrayList ();
 363                 protected ArrayList no_error_list = new ArrayList ();
 364
 365                 protected bool verbose;
 366
 367                 int total_known_issues;
 368
 369                 protected Checker (ITester tester)
 370                 {
 371                         this.tester = tester;
 372                 }
 373
 374                 public string IssueFile {
 375                         set {
 376                                 this.issue_file = value;
 377                                 ReadWrongErrors (issue_file);
 378                         }
 379                 }
 380
 381                 public string LogFile {
 382                         set {
 383                                 this.log_file = new StreamWriter (value, false);
 384                         }
 385                 }
 386
 387                 public bool Verbose {
 388                         set {
 389                                 verbose = value;
 390                         }
 391                 }
 392
 393                 public string[] ExtraCompilerOptions {
 394                         set {
 395                                 extra_compiler_options = value;
 396                         }
 397                 }
 398
 399                 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
 400                                                         out string[] dependencies)
 401                 {
 402                         int row = 0;
 403                         compiler_options = null;
 404                         dependencies = null;
 405                         try {
 406                                 using (StreamReader sr = new StreamReader (file)) {
 407                                         String line;
 408                                         while (row++ < 3 && (line = sr.ReadLine()) != null) {
 409                                                 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
 410                                                                       ref dependencies))
 411                                                         return false;
 412                                         }
 413                                 }
 414                         } catch {
 415                                 return false;
 416                         }
 417                         return true;
 418                 }
 419
 420                 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
 421                                                         ref string[] compiler_options,
 422                                                         ref string[] dependencies)
 423                 {
 424                         const string options = "// Compiler options:";
 425                         const string depends = "// Dependencies:";
 426
 427                         if (row == 1) {
 428                                 compiler_options = null;
 429                                 dependencies = null;
 430                         }
 431
 432                         int index = line.IndexOf (options);
 433                         if (index != -1) {
 434                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 435                                 for (int i = 0; i < compiler_options.Length; i++)
 436                                         compiler_options[i] = compiler_options[i].TrimStart ();
 437                         }
 438                         index = line.IndexOf (depends);
 439                         if (index != -1) {
 440                                 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
 441                                 for (int i = 0; i < dependencies.Length; i++)
 442                                         dependencies[i] = dependencies[i].TrimStart ();
 443                         }
 444
 445                         return true;
 446                 }
 447
 448                 public bool Do (string filename)
 449                 {
 450                         if (test_hash.Contains (filename))
 451                                 return true;
 452
 453                         if (ignore_list.Contains (filename)) {
 454                                 ++ignored;
 455                                 LogFileLine (filename, "NOT TESTED");
 456                                 return false;
 457                         }
 458
 459                         string[] compiler_options, dependencies;
 460                         if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
 461                                 LogFileLine (filename, "ERROR");
 462                                 return false;
 463                         }
 464
 465                         if (extra_compiler_options != null) {
 466                                 if (compiler_options == null)
 467                                         compiler_options = extra_compiler_options;
 468                                 else {
 469                                         string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
 470                                         extra_compiler_options.CopyTo (new_options, 0);
 471                                         compiler_options.CopyTo (new_options, extra_compiler_options.Length);
 472                                         compiler_options = new_options;
 473                                 }
 474                         }
 475
 476                         TestCase test = CreateTestCase (filename, compiler_options, dependencies);
 477                         test_hash.Add (filename, test);
 478
 479                         ++total;
 480                         if (dependencies != null) {
 481                                 foreach (string dependency in dependencies) {
 482                                         if (!Do (dependency)) {
 483                                                 LogFileLine (filename, "DEPENDENCY FAILED");
 484                                                 return false;
 485                                         }
 486                                 }
 487                         }
 488
 489                         tests.Add (test);
 490
 491                         return Check (test);
 492                 }
 493
 494                 protected virtual bool Check (TestCase test)
 495                 {
 496                         string[] test_args;
 497
 498                         if (test.CompilerOptions != null) {
 499                                 test_args = new string [2 + test.CompilerOptions.Length];
 500                                 test.CompilerOptions.CopyTo (test_args, 0);
 501                         } else {
 502                                 test_args = new string [2];
 503                         }
 504                         test_args [test_args.Length - 2] = test.FileName;
 505                         test_args [test_args.Length - 1] = "-debug";
 506
 507                         return tester.Invoke (test_args);
 508                 }
 509
 510                 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
 511                 {
 512                         return new TestCase (filename, options, deps);
 513                 }
 514
 515                 void ReadWrongErrors (string file)
 516                 {
 517                         const string ignored = "IGNORE";
 518                         const string no_error = "NO ERROR";
 519
 520                         using (StreamReader sr = new StreamReader (file)) {
 521                                 string line;
 522                                 while ((line = sr.ReadLine()) != null) {
 523                                         if (line.StartsWith ("#"))
 524                                                 continue;
 525
 526                                         ArrayList active_cont = know_issues;
 527
 528                                         if (line.IndexOf (ignored) > 0)
 529                                                 active_cont = ignore_list;
 530                                         else if (line.IndexOf (no_error) > 0)
 531                                                 active_cont = no_error_list;
 532
 533                                         string file_name = line.Split (' ')[0];
 534                                         if (file_name.Length == 0)
 535                                                 continue;
 536
 537                                         active_cont.Add (file_name);
 538                                 }
 539                         }
 540                         total_known_issues = know_issues.Count;
 541                 }
 542
 543                 protected virtual void PrintSummary ()
 544                 {
 545                         LogLine ("Done" + Environment.NewLine);
 546                         float rate = 0;
 547                         if (total > 0)
 548                                 rate = (float) (success) / (float)total;
 549                         LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
 550
 551                         if (syntax_errors > 0)
 552                                 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
 553
 554                         if (ignored > 0)
 555                                 LogLine ("{0} test(s) ignored", ignored);
 556
 557                         if (total_known_issues - know_issues.Count > 0)
 558                                 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
 559
 560                         know_issues.AddRange (no_error_list);
 561                         if (know_issues.Count > 0) {
 562                                 LogLine ("");
 563                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 564                                 foreach (string s in know_issues)
 565                                         LogLine (s);
 566                         }
 567                         if (regression.Count > 0) {
 568                                 LogLine ("");
 569                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 570                                 foreach (string s in regression)
 571                                         LogLine (s);
 572                         }
 573                 }
 574
 575                 public int ResultCode
 576                 {
 577                         get {
 578                                 return regression.Count == 0 ? 0 : 1;
 579                         }
 580                 }
 581
 582                 protected void Log (string msg, params object [] rest)
 583                 {
 584                         Console.Write (msg, rest);
 585                         if (log_file != null)
 586                                 log_file.Write (msg, rest);
 587                 }
 588
 589                 protected void LogLine (string msg)
 590                 {
 591                         Console.WriteLine (msg);
 592                         if (log_file != null)
 593                                 log_file.WriteLine (msg);
 594                 }
 595
 596                 protected void LogLine (string msg, params object [] rest)
 597                 {
 598                         Console.WriteLine (msg, rest);
 599                         if (log_file != null)
 600                                 log_file.WriteLine (msg, rest);
 601                 }
 602
 603                 public void LogFileLine (string file, string msg, params object [] args)
 604                 {
 605                         string s = file + "...\t" + string.Format (msg, args);
 606                         Console.WriteLine (s);
 607                         if (log_file != null)
 608                                 log_file.WriteLine (s);
 609                 }
 610
 611                 #region IDisposable Members
 612
 613                 public void Dispose()
 614                 {
 615                         if (log_file != null)
 616                                 log_file.Close ();
 617                 }
 618
 619                 #endregion
 620
 621                 public virtual void Initialize ()
 622                 {
 623                 }
 624
 625                 public virtual void CleanUp ()
 626                 {
 627                         PrintSummary ();
 628                 }
 629         }
 630
 631         class PositiveChecker: Checker
 632         {
 633                 readonly string files_folder;
 634                 readonly static object[] default_args = new object[1] { new string[] {} };
 635                 string doc_output;
 636                 string verif_file;
 637                 bool update_verif_file;
 638                 Hashtable verif_data;
 639
 640 #if !NET_2_1
 641                 ProcessStartInfo pi;
 642 #endif
 643                 readonly string mono;
 644
 645                 public enum TestResult {
 646                         CompileError,
 647                         ExecError,
 648                         LoadError,
 649                         XmlError,
 650                         Success,
 651                         ILError
 652                 }
 653
 654                 public PositiveChecker (ITester tester, string verif_file):
 655                         base (tester)
 656                 {
 657                         files_folder = Directory.GetCurrentDirectory ();
 658                         this.verif_file = verif_file;
 659
 660 #if !NET_2_1
 661                         pi = new ProcessStartInfo ();
 662                         pi.CreateNoWindow = true;
 663                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 664                         pi.RedirectStandardOutput = true;
 665                         pi.RedirectStandardError = true;
 666                         pi.UseShellExecute = false;
 667
 668                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 669                         if (mono != null) {
 670                                 pi.FileName = mono;
 671                         }
 672 #endif
 673                 }
 674
 675                 public bool UpdateVerificationDataFile {
 676                         set {
 677                                 update_verif_file = value;
 678                         }
 679                 }
 680
 681                 protected override bool GetExtraOptions(string file, out string[] compiler_options,
 682                                                         out string[] dependencies) {
 683                         if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
 684                                 return false;
 685
 686                         doc_output = null;
 687                         if (compiler_options == null)
 688                                 return true;
 689
 690                         foreach (string one_opt in compiler_options) {
 691                                 if (one_opt.StartsWith ("-doc:")) {
 692                                         doc_output = one_opt.Split (':', '/')[1];
 693                                 }
 694                         }
 695                         return true;
 696                 }
 697
 698                 protected override bool Check(TestCase test)
 699                 {
 700                         string filename = test.FileName;
 701                         try {
 702                                 if (!base.Check (test)) {
 703                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 704                                         return false;
 705                                 }
 706                         }
 707                         catch (Exception e) {
 708                                 if (e.InnerException != null)
 709                                         e = e.InnerException;
 710
 711                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 712                                 return false;
 713                         }
 714
 715                         // Test setup
 716                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 717                                 if (verbose)
 718                                         LogFileLine (filename, "OK");
 719                                 --total;
 720                                 return true;
 721                         }
 722
 723                         MethodInfo mi = null;
 724                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 725
 726                         // Enable .dll only tests (no execution required)
 727                         if (!File.Exists(file)) {
 728                                 HandleFailure (filename, TestResult.Success, null);
 729                                 return true;
 730                         }
 731
 732                         Assembly assembly = null;
 733                         try {
 734                                 assembly = Assembly.LoadFile (file);
 735                                 mi = assembly.EntryPoint;
 736                         }
 737                         catch (FileNotFoundException) {
 738                                 if (File.Exists (file)) {
 739                                         Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 740                                 }
 741                         }
 742                         catch (Exception e) {
 743                                 HandleFailure (filename, TestResult.LoadError, e.ToString ());
 744                                 return false;
 745                         }
 746
 747                         if (!ExecuteFile (mi, file, filename))
 748                                 return false;
 749
 750                         if (doc_output != null) {
 751                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 752                                 try {
 753 #if !NET_2_1
 754                                         XmlComparer.Compare (ref_file, doc_output);
 755 #endif
 756                                 }
 757                                 catch (Exception e) {
 758                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 759                                         return false;
 760                                 }
 761                         } else {
 762                                 if (verif_file != null) {
 763                                         PositiveTestCase pt = (PositiveTestCase) test;
 764                                         pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data [filename];
 765                                         if (!CheckILSize (assembly, pt))
 766                                                 return false;
 767                                 }
 768                         }
 769
 770                         HandleFailure (filename, TestResult.Success, null);
 771                         return true;
 772                 }
 773
 774                 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
 775                 {
 776                         return new PositiveTestCase (filename, options, deps);
 777                 }
 778
 779                 bool CheckILSize (Assembly assembly, PositiveTestCase test)
 780                 {
 781                         bool success = true;
 782                         Type[] types = assembly.GetTypes ();
 783                         foreach (Type t in types) {
 784                                 if (!t.IsClass && t.IsValueType)
 785                                         continue;
 786
 787                                 if (test.VerificationProvider == null) {
 788                                         if (!update_verif_file)
 789                                                 LogFileLine (test.FileName, "Missing IL verification data");
 790                                         test.CreateNewTest ();
 791                                 }
 792
 793                                 foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
 794                                         MethodBase mi = m as MethodBase;
 795                                         if (mi == null)
 796                                                 continue;
 797
 798                                         if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
 799                                                 continue;
 800
 801                                         success &= test.CompareIL (mi, this);
 802                                 }
 803                         }
 804
 805                         return success;
 806                 }
 807
 808                 bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
 809                 {
 810                         TextWriter stdout = Console.Out;
 811                         TextWriter stderr = Console.Error;
 812                         Console.SetOut (TextWriter.Null);
 813                         Console.SetError (TextWriter.Null);
 814                         ParameterInfo[] pi = entry_point.GetParameters ();
 815                         object[] args = pi.Length == 0 ? null : default_args;
 816
 817                         object result = null;
 818                         try {
 819                                 try {
 820                                         result = entry_point.Invoke (null, args);
 821                                 } finally {
 822                                         Console.SetOut (stdout);
 823                                         Console.SetError (stderr);
 824                                 }
 825                         }
 826                         catch (Exception e) {
 827                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 828                                 return false;
 829                         }
 830
 831                         if (result is int && (int)result != 0) {
 832                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 833                                 return false;
 834                         }
 835                         return true;
 836                 }
 837
 838                 public void HandleFailure (string file, TestResult status, string extra)
 839                 {
 840                         switch (status) {
 841                                 case TestResult.Success:
 842                                         success++;
 843                                         if (know_issues.Contains (file)) {
 844                                                 LogFileLine (file, "FIXED ISSUE");
 845                                                 return;
 846                                         }
 847                                         if (verbose)
 848                                                 LogFileLine (file, "OK");
 849                                         return;
 850
 851                                 case TestResult.CompileError:
 852                                         if (know_issues.Contains (file)) {
 853                                                 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
 854                                                 know_issues.Remove (file);
 855                                                 return;
 856                                         }
 857                                         LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
 858                                         break;
 859
 860                                 case TestResult.ExecError:
 861                                         if (know_issues.Contains (file)) {
 862                                                 LogFileLine (file, "KNOWN ISSUE (Execution error)");
 863                                                 know_issues.Remove (file);
 864                                                 return;
 865                                         }
 866                                         LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
 867                                         break;
 868
 869                                 case TestResult.XmlError:
 870                                         if (know_issues.Contains (file)) {
 871                                                 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
 872                                                 know_issues.Remove (file);
 873                                                 return;
 874                                         }
 875                                         LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 876                                         break;
 877
 878                                 case TestResult.LoadError:
 879                                         LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
 880                                         break;
 881
 882                                 case TestResult.ILError:
 883                                         LogFileLine (file, "IL REGRESSION: " + extra);
 884                                         extra = null;
 885                                         break;
 886                         }
 887
 888                         if (extra != null)
 889                                 LogLine ("{0}", extra);
 890
 891                         if (!regression.Contains (file))
 892                                 regression.Add (file);
 893                 }
 894
 895                 public override void Initialize ()
 896                 {
 897                         if (verif_file != null) {
 898 #if NET_2_0
 899                                 LoadVerificationData (verif_file);
 900 #else
 901                                 throw new NotSupportedException ();
 902 #endif
 903                         }
 904
 905                         base.Initialize ();
 906                 }
 907
 908                 public override void CleanUp ()
 909                 {
 910                         base.CleanUp ();
 911
 912                         if (update_verif_file) {
 913 #if NET_2_0
 914                                 UpdateVerificationData (verif_file);
 915 #else
 916                                 throw new NotSupportedException ();
 917 #endif
 918                         }
 919                 }
 920
 921 #if NET_2_0
 922                 void LoadVerificationData (string file)
 923                 {
 924                         LogLine ("Loading verification data from `{0}' ...", file);
 925
 926                         using (XmlReader r = XmlReader.Create (file)) {
 927                                 r.ReadStartElement ("tests");
 928                                 verif_data = new Hashtable ();
 929
 930                                 while (r.Read ()) {
 931                                         if (r.Name != "test")
 932                                                 continue;
 933
 934                                         string name = r.GetAttribute ("name");
 935                                         PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
 936                                         verif_data.Add (name, tc);
 937                                 }
 938                         }
 939                 }
 940
 941                 void UpdateVerificationData (string file)
 942                 {
 943                         LogLine ("Updating verification data `{0}' ...", file);
 944
 945                         XmlWriterSettings s = new XmlWriterSettings ();
 946                         s.Indent = true;
 947                         using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
 948                                 w.WriteStartDocument ();
 949                                 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
 950                                 w.WriteStartElement ("tests");
 951                                 foreach (PositiveTestCase tc in tests) {
 952                                         if (tc.VerificationProvider != null)
 953                                                 tc.VerificationProvider.WriteCodeInfoTo (w);
 954                                 }
 955                                 w.WriteEndElement ();
 956                         }
 957                 }
 958 #endif
 959         }
 960
 961         class NegativeChecker: Checker
 962         {
 963                 string expected_message;
 964                 string error_message;
 965                 bool check_msg;
 966                 bool check_error_line;
 967                 bool is_warning;
 968                 IDictionary wrong_warning;
 969
 970                 protected enum CompilerError {
 971                         Expected,
 972                         Wrong,
 973                         Missing,
 974                         WrongMessage,
 975                         MissingLocation,
 976                         Duplicate
 977                 }
 978
 979                 public NegativeChecker (ITester tester, bool check_msg):
 980                         base (tester)
 981                 {
 982                         this.check_msg = check_msg;
 983                         wrong_warning = new Hashtable ();
 984                 }
 985
 986                 protected override bool AnalyzeTestFile (string file, ref int row, string line,
 987                                                         ref string[] compiler_options,
 988                                                         ref string[] dependencies)
 989                 {
 990                         if (row == 1) {
 991                                 expected_message = null;
 992
 993                                 int index = line.IndexOf (':');
 994                                 if (index == -1 || index > 15) {
 995                                         LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
 996                                         ++syntax_errors;
 997                                         base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
 998                                                               ref dependencies);
 999                                         return false;
1000                                 }
1001
1002                                 expected_message = line.Substring (index + 1).Trim ();
1003                         }
1004
1005                         if (row == 2) {
1006                                 string filtered = line.Replace(" ", "");
1007
1008                                 // Some error tests require to have different error text for different runtimes.
1009                                 if (filtered.StartsWith ("//GMCS")) {
1010                                         row = 1;
1011 #if !NET_2_0
1012                                         return true;
1013 #else
1014                                         return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1015 #endif
1016                                 }
1017
1018                                 check_error_line = !filtered.StartsWith ("//Line:0");
1019
1020                                 if (!filtered.StartsWith ("//Line:")) {
1021                                         LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1022                                         ++syntax_errors;
1023                                         return false;
1024                                 }
1025                         }
1026
1027                         if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1028                                 return false;
1029
1030                         is_warning = false;
1031                         if (compiler_options != null) {
1032                                 foreach (string s in compiler_options) {
1033                                         if (s.EndsWith ("warnaserror"))
1034                                                 is_warning = true;
1035                                 }
1036                         }
1037                         return true;
1038                 }
1039
1040
1041                 protected override bool Check (TestCase test)
1042                 {
1043                         string filename = test.FileName;
1044
1045                         int start_char = 0;
1046                         while (Char.IsLetter (filename, start_char))
1047                                 ++start_char;
1048
1049                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1050                         string expected = filename.Substring (start_char, end_char - start_char);
1051
1052                         try {
1053                                 if (base.Check (test)) {
1054                                         HandleFailure (filename, CompilerError.Missing);
1055                                         return false;
1056                                 }
1057                         }
1058                         catch (Exception e) {
1059                                 HandleFailure (filename, CompilerError.Missing);
1060                                 if (e.InnerException != null)
1061                                         e = e.InnerException;
1062
1063                                 Log (e.ToString ());
1064                                 return false;
1065                         }
1066
1067                         int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1068                         if (tester.IsWarning (err_id)) {
1069                                 if (!is_warning)
1070                                         wrong_warning [err_id] = true;
1071                         } else {
1072                                 if (is_warning)
1073                                         wrong_warning [err_id] = false;
1074                         }
1075
1076                         CompilerError result_code = GetCompilerError (expected, tester.Output);
1077                         if (HandleFailure (filename, result_code)) {
1078                                 success++;
1079                                 return true;
1080                         }
1081
1082                         if (result_code == CompilerError.Wrong)
1083                                 LogLine (tester.Output);
1084
1085                         return false;
1086                 }
1087
1088                 CompilerError GetCompilerError (string expected, string buffer)
1089                 {
1090                         const string error_prefix = "CS";
1091                         const string ignored_error = "error CS5001";
1092                         string tested_text = "error " + error_prefix + expected;
1093                         StringReader sr = new StringReader (buffer);
1094                         string line = sr.ReadLine ();
1095                         ArrayList ld = new ArrayList ();
1096                         CompilerError result = CompilerError.Missing;
1097                         while (line != null) {
1098                                 if (ld.Contains (line)) {
1099                                         if (line.IndexOf ("Location of the symbol related to previous") == -1)
1100                                                 return CompilerError.Duplicate;
1101                                 }
1102                                 ld.Add (line);
1103
1104                                 if (result != CompilerError.Expected) {
1105                                         if (line.IndexOf (tested_text) != -1) {
1106                                                 if (check_msg) {
1107                                                         int first = line.IndexOf (':');
1108                                                         int second = line.IndexOf (':', first + 1);
1109                                                         if (second == -1 || !check_error_line)
1110                                                                 second = first;
1111
1112                                                         string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1113                                                         if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1114                                                                 error_message = msg;
1115                                                                 return CompilerError.WrongMessage;
1116                                                         }
1117
1118                                                         if (check_error_line && line.IndexOf (".cs(") == -1)
1119                                                                 return CompilerError.MissingLocation;
1120                                                 }
1121                                                 result = CompilerError.Expected;
1122                                         } else if (line.IndexOf (error_prefix) != -1 &&
1123                                                 line.IndexOf (ignored_error) == -1)
1124                                                 result = CompilerError.Wrong;
1125                                 }
1126
1127                                 line = sr.ReadLine ();
1128                         }
1129
1130                         return result;
1131                 }
1132
1133                 bool HandleFailure (string file, CompilerError status)
1134                 {
1135                         switch (status) {
1136                                 case CompilerError.Expected:
1137                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1138                                                 LogFileLine (file, "FIXED ISSUE");
1139                                                 return true;
1140                                         }
1141
1142                                         if (verbose)
1143                                                 LogFileLine (file, "OK");
1144                                         return true;
1145
1146                                 case CompilerError.Wrong:
1147                                         if (know_issues.Contains (file)) {
1148                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1149                                                 know_issues.Remove (file);
1150                                                 return false;
1151                                         }
1152                                         if (no_error_list.Contains (file)) {
1153                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1154                                                 no_error_list.Remove (file);
1155                                         }
1156                                         else {
1157                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1158                                         }
1159                                         break;
1160
1161                                 case CompilerError.WrongMessage:
1162                                         if (know_issues.Contains (file)) {
1163                                                 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1164                                                 know_issues.Remove (file);
1165                                                 return false;
1166                                         }
1167                                         if (no_error_list.Contains (file)) {
1168                                                 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1169                                                 no_error_list.Remove (file);
1170                                         }
1171                                         else {
1172                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1173                                                 LogLine ("Exp: {0}", expected_message);
1174                                                 LogLine ("Was: {0}", error_message);
1175                                         }
1176                                         break;
1177
1178                                 case CompilerError.Missing:
1179                                         if (no_error_list.Contains (file)) {
1180                                                 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1181                                                 no_error_list.Remove (file);
1182                                                 return false;
1183                                         }
1184
1185                                         if (know_issues.Contains (file)) {
1186                                                 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1187                                                 know_issues.Remove (file);
1188                                         }
1189                                         else {
1190                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1191                                         }
1192
1193                                         break;
1194
1195                                 case CompilerError.MissingLocation:
1196                                         if (know_issues.Contains (file)) {
1197                                                 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1198                                                 know_issues.Remove (file);
1199                                                 return false;
1200                                         }
1201                                         if (no_error_list.Contains (file)) {
1202                                                 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1203                                                 no_error_list.Remove (file);
1204                                         }
1205                                         else {
1206                                                 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1207                                         }
1208                                         break;
1209
1210                                 case CompilerError.Duplicate:
1211                                         // Will become an error soon
1212                                         LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1213                                         return true;
1214                         }
1215
1216                         regression.Add (file);
1217                         return false;
1218                 }
1219
1220                 protected override void PrintSummary()
1221                 {
1222                         base.PrintSummary ();
1223
1224                         if (wrong_warning.Count > 0) {
1225                                 LogLine ("");
1226                                 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1227                                 LogLine ("");
1228                                 foreach (DictionaryEntry de in wrong_warning)
1229                                         LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1230                         }
1231                 }
1232
1233         }
1234
1235         class Tester {
1236
1237                 static int Main(string[] args)
1238                 {
1239                         string temp;
1240
1241                         if (GetOption ("help", args, false, out temp)) {
1242                                 Usage ();
1243                                 return 0;
1244                         }
1245
1246                         string compiler;
1247                         if (!GetOption ("compiler", args, true, out compiler)) {
1248                                 Usage ();
1249                                 return 1;
1250                         }
1251
1252                         ITester tester;
1253                         try {
1254                                 Console.WriteLine ("Loading " + compiler + " ...");
1255                                 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1256                         }
1257                         catch (Exception) {
1258 #if NET_2_1
1259                                 throw;
1260 #else
1261                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1262                                 if (!File.Exists (compiler)) {
1263                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1264                                         return 1;
1265                                 }
1266                                 tester = new ProcessTester (compiler);
1267 #endif
1268                         }
1269
1270                         string mode;
1271                         if (!GetOption ("mode", args, true, out mode)) {
1272                                 Usage ();
1273                                 return 1;
1274                         }
1275
1276                         Checker checker;
1277                         switch (mode) {
1278                                 case "neg":
1279                                         checker = new NegativeChecker (tester, true);
1280                                         break;
1281                                 case "pos":
1282                                         string iltest;
1283                                         GetOption ("il", args, false, out iltest);
1284                                         checker = new PositiveChecker (tester, iltest);
1285
1286                                         if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1287                                                 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1288                                         }
1289
1290                                         break;
1291                                 default:
1292                                         Console.Error.WriteLine ("Invalid -mode argument");
1293                                         return 1;
1294                         }
1295
1296
1297                         if (GetOption ("issues", args, true, out temp))
1298                                 checker.IssueFile = temp;
1299                         if (GetOption ("log", args, true, out temp))
1300                                 checker.LogFile = temp;
1301                         if (GetOption ("verbose", args, false, out temp))
1302                                 checker.Verbose = true;
1303                         if (GetOption ("compiler-options", args, true, out temp)) {
1304                                 string[] extra = temp.Split (' ');
1305                                 checker.ExtraCompilerOptions = extra;
1306                         }
1307
1308                         string test_pattern;
1309                         if (!GetOption ("files", args, true, out test_pattern)) {
1310                                 Usage ();
1311                                 return 1;
1312                         }
1313
1314                         string [] files = Directory.GetFiles (".", test_pattern);
1315                         if (files.Length == 0) {
1316                                 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1317                                 return 2;
1318                         }
1319
1320                         checker.Initialize ();
1321
1322                         foreach (string s in files) {
1323                                 string filename = Path.GetFileName (s);
1324                                 if (Char.IsUpper (filename, 0)) { // Windows hack
1325                                         continue;
1326                                 }
1327
1328                                 if (filename.EndsWith ("-p2.cs"))
1329                                         continue;
1330
1331                                 checker.Do (filename);
1332                         }
1333
1334                         checker.CleanUp ();
1335
1336                         checker.Dispose ();
1337
1338                         return checker.ResultCode;
1339                 }
1340
1341                 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1342                 {
1343                         opt = "-" + opt;
1344                         foreach (string a in args) {
1345                                 if (a.StartsWith (opt)) {
1346                                         int sep = a.IndexOf (':');
1347                                         if (sep > 0) {
1348                                                 value = a.Substring (sep + 1);
1349                                         } else {
1350                                                 value = null;
1351                                                 if (req_arg) {
1352                                                         Console.Error.WriteLine ("Missing argument in option " + opt);
1353                                                         return false;
1354                                                 }
1355                                         }
1356
1357                                         return true;
1358                                 }
1359                         }
1360
1361                         value = null;
1362                         return false;
1363                 }
1364
1365                 static void Usage ()
1366                 {
1367                         Console.WriteLine (
1368                                 "Mono compiler tester, (C) 2008 Novell, Inc.\n" +
1369                                 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1370                                 "   \n" +
1371                                 "   -compiler:FILE   The file which will be used to compiler tests\n" +
1372                                 "   -compiler-options:OPTIONS  Add global compiler options\n" +
1373                                 "   -il:IL-FILE      XML file with expected IL details for each test\n" +
1374                                 "   -issues:FILE     The list of expected failures\n" +
1375                                 "   -log:FILE        Writes any output also to the file\n" +
1376                                 "   -help            Lists all options\n" +
1377                                 "   -mode:[pos|neg]  Specifies compiler test mode\n" +
1378                                 "   -update-il       Updates IL-FILE to match compiler output\n" +
1379                                 "   -verbose         Prints more details during testing\n"
1380                                 );
1381                 }
1382         }
1383 }