mcs/tools/compiler-tester/compiler-tester.cs

   1 using System;
   2 using System.IO;
   3 using System.Diagnostics;
   4 using System.Reflection;
   5 using System.Text;
   6 using System.Collections;
   7
   8 namespace TestRunner {
   9
  10         interface ITester
  11         {
  12                 string Output { get; }
  13                 bool Invoke (string[] args);
  14         }
  15
  16         class ReflectionTester: ITester {
  17                 MethodInfo ep;
  18                 object[] method_arg;
  19                 StringWriter output;
  20
  21                 public ReflectionTester (Assembly a)
  22                 {
  23                         ep = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint").GetMethod ("InvokeCompiler",
  24                                 BindingFlags.Static | BindingFlags.Public);
  25                         if (ep == null)
  26                                 throw new MissingMethodException ("static InvokeCompiler");
  27                         method_arg = new object [2];
  28                 }
  29
  30                 public string Output {
  31                         get {
  32                                 return output.GetStringBuilder ().ToString ();
  33                         }
  34                 }
  35
  36                 public bool Invoke(string[] args)
  37                 {
  38                         output = new StringWriter ();
  39                         method_arg [0] = args;
  40                         method_arg [1] = output;
  41                         return (bool)ep.Invoke (null, method_arg);
  42                 }
  43         }
  44
  45         class ProcessTester: ITester
  46         {
  47                 ProcessStartInfo pi;
  48                 string output;
  49
  50                 public ProcessTester (string p_path)
  51                 {
  52                         pi = new ProcessStartInfo ();
  53                         pi.FileName = p_path;
  54                         pi.CreateNoWindow = true;
  55                         pi.WindowStyle = ProcessWindowStyle.Hidden;
  56                         pi.RedirectStandardOutput = true;
  57                         pi.RedirectStandardError = true;
  58                         pi.UseShellExecute = false;
  59                 }
  60
  61                 public string Output {
  62                         get {
  63                                 return output;
  64                         }
  65                 }
  66
  67                 public bool Invoke(string[] args)
  68                 {
  69                         StringBuilder sb = new StringBuilder ("/nologo ");
  70                         foreach (string s in args) {
  71                                 sb.Append (s);
  72                                 sb.Append (" ");
  73                         }
  74                         pi.Arguments = sb.ToString ();
  75                         Process p = Process.Start (pi);
  76                         output = p.StandardError.ReadToEnd ();
  77                         if (output.Length == 0)
  78                             output = p.StandardOutput.ReadToEnd ();
  79                         p.WaitForExit ();
  80                         return p.ExitCode == 0;
  81                 }
  82         }
  83
  84         class Checker: IDisposable
  85         {
  86                 protected ITester tester;
  87                 protected int success;
  88                 protected int total;
  89                 protected int ignored;
  90                 string issue_file;
  91                 StreamWriter log_file;
  92                 protected string[] compiler_options;
  93
  94                 protected ArrayList regression = new ArrayList ();
  95                 protected ArrayList know_issues = new ArrayList ();
  96                 protected ArrayList ignore_list = new ArrayList ();
  97                 protected ArrayList no_error_list = new ArrayList ();
  98
  99                 protected Checker (ITester tester, string log_file, string issue_file)
 100                 {
 101                         this.tester = tester;
 102                         this.issue_file = issue_file;
 103                         ReadWrongErrors (issue_file);
 104                         this.log_file = new StreamWriter (log_file, false);
 105                 }
 106
 107                 protected virtual bool GetExtraOptions (string file)
 108                 {
 109                         int row = 0;
 110                         using (StreamReader sr = new StreamReader (file)) {
 111                                 String line;
 112                                 while (row++ < 3 && (line = sr.ReadLine()) != null) {
 113                                         if (!AnalyzeTestFile (row, line))
 114                                                 return false;
 115                                 }
 116                         }
 117                         return true;
 118                 }
 119
 120                 protected virtual bool AnalyzeTestFile (int row, string line)
 121                 {
 122                         const string options = "// Compiler options:";
 123
 124                         if (row == 1)
 125                                 compiler_options = null;
 126
 127                         int index = line.IndexOf (options);
 128                         if (index != -1) {
 129                                 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
 130                                 for (int i = 0; i < compiler_options.Length; i++)
 131                                         compiler_options[i] = compiler_options[i].TrimStart ();
 132                         }
 133                         return true;
 134                 }
 135
 136                 public void Do (string filename)
 137                 {
 138                         Log (filename);
 139                         Log ("...\t");
 140
 141                         if (ignore_list.Contains (filename)) {
 142                                 ++ignored;
 143                                 LogLine ("NOT TESTED");
 144                                 return;
 145                         }
 146
 147                         if (!GetExtraOptions (filename)) {
 148                                 return;
 149                         }
 150
 151                         ++total;
 152                         Check (filename);
 153                 }
 154
 155                 protected virtual bool Check (string filename)
 156                 {
 157                         string[] test_args;
 158
 159                         if (compiler_options != null) {
 160                                 test_args = new string [1 + compiler_options.Length];
 161                                 compiler_options.CopyTo (test_args, 0);
 162                         } else {
 163                                 test_args = new string [1];
 164                         }
 165                         test_args [test_args.Length - 1] = filename;
 166
 167                         return tester.Invoke (test_args);
 168                 }
 169
 170
 171                 void ReadWrongErrors (string file)
 172                 {
 173                         const string ignored = "IGNORE";
 174                         const string no_error = "NO ERROR";
 175
 176                         using (StreamReader sr = new StreamReader (file)) {
 177                                 string line;
 178                                 while ((line = sr.ReadLine()) != null) {
 179                                         if (line.StartsWith ("#"))
 180                                                 continue;
 181
 182                                         ArrayList active_cont = know_issues;
 183
 184                                         if (line.IndexOf (ignored) > 0)
 185                                                 active_cont = ignore_list;
 186                                         else if (line.IndexOf (no_error) > 0)
 187                                                 active_cont = no_error_list;
 188
 189                                         string file_name = line.Split (' ')[0];
 190                                         if (file_name.Length == 0)
 191                                                 continue;
 192
 193                                         active_cont.Add (file_name);
 194                                 }
 195                         }
 196                 }
 197
 198                 public void PrintSummary ()
 199                 {
 200                         LogLine ("Done" + Environment.NewLine);
 201                         LogLine ("{0} test cases passed ({1:.##%})", success, (float) (success) / (float)total);
 202
 203                         if (ignored > 0)
 204                                 LogLine ("{0} test cases ignored", ignored);
 205
 206                         know_issues.AddRange (no_error_list);
 207                         if (know_issues.Count > 0) {
 208                                 LogLine ("");
 209                                 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
 210                                 foreach (string s in know_issues)
 211                                         LogLine (s);
 212                         }
 213                         if (regression.Count > 0) {
 214                                 LogLine ("");
 215                                 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
 216                                 foreach (string s in regression)
 217                                         LogLine (s);
 218                         }
 219                 }
 220
 221                 public int ResultCode
 222                 {
 223                         get {
 224                                 return regression.Count == 0 ? 0 : 1;
 225                         }
 226                 }
 227
 228                 protected void Log (string msg, params object [] rest)
 229                 {
 230                         Console.Write (msg, rest);
 231                         log_file.Write (msg, rest);
 232                 }
 233
 234                 protected void LogLine (string msg, params object [] rest)
 235                 {
 236                         Console.WriteLine (msg, rest);
 237                         log_file.WriteLine (msg, rest);
 238                 }
 239
 240                 #region IDisposable Members
 241
 242                 public void Dispose()
 243                 {
 244                         log_file.Close ();
 245                 }
 246
 247                 #endregion
 248         }
 249
 250         class PositiveChecker: Checker {
 251                 readonly string files_folder;
 252                 readonly static object[] default_args = new object[1] { new string[] {} };
 253                 string doc_output;
 254
 255                 // When we cannot load assembly to domain we are automaticaly switching to process calling
 256                 bool appdomain_limit_reached;
 257
 258                 ProcessStartInfo pi;
 259                 readonly string mono;
 260
 261                 // This is really pain
 262                 // This number is highly experimental on my box I am not able to load more than 1000 files to domain
 263                 // Every files is there twice and we need some space for tests which reference assemblies
 264                 const int MAX_TESTS_IN_DOMAIN = 420;
 265                 int test_counter;
 266
 267                 protected enum TestResult {
 268                         CompileError,
 269                         ExecError,
 270                         LoadError,
 271                         XmlError,
 272                         Success
 273                 }
 274
 275                 public PositiveChecker (ITester tester, string log_file, string issue_file):
 276                         base (tester, log_file, issue_file)
 277                 {
 278                         files_folder = Directory.GetCurrentDirectory ();
 279
 280                         pi = new ProcessStartInfo ();
 281                         pi.CreateNoWindow = true;
 282                         pi.WindowStyle = ProcessWindowStyle.Hidden;
 283                         pi.RedirectStandardOutput = true;
 284                         pi.RedirectStandardError = true;
 285                         pi.UseShellExecute = false;
 286
 287                         mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
 288                         if (mono != null) {
 289                                 pi.FileName = mono;
 290                         }
 291                 }
 292
 293                 protected override bool GetExtraOptions(string file) {
 294                         if (!base.GetExtraOptions (file))
 295                                 return false;
 296
 297                         doc_output = null;
 298                         if (compiler_options == null)
 299                                 return true;
 300
 301                         foreach (string one_opt in compiler_options) {
 302                                 if (one_opt.StartsWith ("-doc:")) {
 303                                         doc_output = one_opt.Split (':')[1];
 304                                 }
 305                         }
 306                         return true;
 307                 }
 308
 309                 protected override bool Check(string filename) {
 310                         try {
 311                                 if (!base.Check (filename)) {
 312                                         HandleFailure (filename, TestResult.CompileError, tester.Output);
 313                                         return false;
 314                                 }
 315                         }
 316                         catch (Exception e) {
 317                                 HandleFailure (filename, TestResult.CompileError, e.ToString ());
 318                                 return false;
 319                         }
 320
 321                         // Test setup
 322                         if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
 323                                 LogLine ("OK");
 324                                 --total;
 325                                 return true;
 326                         }
 327
 328                         MethodInfo mi = null;
 329                         string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
 330                         if (!appdomain_limit_reached) {
 331                                 try {
 332                                         mi = Assembly.LoadFile (file).EntryPoint;
 333                                         if (test_counter++ > MAX_TESTS_IN_DOMAIN)
 334                                                 appdomain_limit_reached = true;
 335                                 }
 336                                 catch (FileNotFoundException) {
 337                                         if (File.Exists (file)) {
 338                                                 Console.WriteLine ("APPDOMAIN LIMIT REACHED");
 339                                                 appdomain_limit_reached = true;
 340                                         }
 341                                 }
 342                                 catch (Exception e) {
 343                                         HandleFailure (filename, TestResult.LoadError, e.ToString ());
 344                                         return false;
 345                                 }
 346                         }
 347
 348                         if (appdomain_limit_reached) {
 349                                 if (!ExecuteFile (file, filename))
 350                                         return false;
 351                         } else {
 352                                 if (!ExecuteFile (mi, filename))
 353                                         return false;
 354                         }
 355
 356                         if (doc_output != null) {
 357                                 string ref_file = filename.Replace (".cs", "-ref.xml");
 358                                 try {
 359                                         XmlComparer.Compare (ref_file, doc_output);
 360                                 }
 361                                 catch (Exception e) {
 362                                         HandleFailure (filename, TestResult.XmlError, e.Message);
 363                                         return false;
 364                                 }
 365                         }
 366
 367                         HandleFailure (filename, TestResult.Success, null);
 368                         return true;
 369                 }
 370
 371                 bool ExecuteFile (string exe_name, string filename)
 372                 {
 373                         if (mono == null)
 374                                 pi.FileName = exe_name;
 375                         else
 376                                 pi.Arguments = exe_name;
 377
 378                         Process p = Process.Start (pi);
 379                         p.WaitForExit ();
 380
 381                         // TODO: How can I recognize return type void ?
 382                         if (p.ExitCode == 0)
 383                                 return true;
 384
 385                         HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + p.ExitCode.ToString ());
 386                         return false;
 387                 }
 388
 389                 bool ExecuteFile (MethodInfo entry_point, string filename)
 390                 {
 391                         TextWriter standart_ouput = Console.Out;
 392                         TextWriter standart_error = Console.Error;
 393                         Console.SetOut (TextWriter.Null);
 394                         Console.SetError (TextWriter.Null);
 395                         ParameterInfo[] pi = entry_point.GetParameters ();
 396                         object[] args = pi.Length == 0 ? null : default_args;
 397
 398                         object result = null;
 399                         try {
 400                                 result = entry_point.Invoke (null, args);
 401                                 Console.SetOut (standart_ouput);
 402                                 Console.SetError (standart_error);
 403                         }
 404                         catch (Exception e) {
 405                                 Console.SetOut (standart_ouput);
 406                                 Console.SetError (standart_error);
 407                                 HandleFailure (filename, TestResult.ExecError, e.ToString ());
 408                                 return false;
 409                         }
 410
 411                         if (result is int && (int)result != 0) {
 412                                 HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
 413                                 return false;
 414                         }
 415                         return true;
 416                 }
 417
 418                 void HandleFailure (string file, TestResult status, string extra)
 419                 {
 420                         switch (status) {
 421                                 case TestResult.Success:
 422                                         success++;
 423                                         if (know_issues.Contains (file)) {
 424                                                 LogLine ("FIXED ISSUE");
 425                                                 return;
 426                                         }
 427                                         LogLine ("OK");
 428                                         return;
 429
 430                                 case TestResult.CompileError:
 431                                         if (know_issues.Contains (file)) {
 432                                                 LogLine ("KNOWN ISSUE (Compilation error)");
 433                                                 know_issues.Remove (file);
 434                                                 return;
 435                                         }
 436                                         LogLine ("REGRESSION (SUCCESS -> COMPILATION ERROR)");
 437                                         break;
 438
 439                                 case TestResult.ExecError:
 440                                         if (know_issues.Contains (file)) {
 441                                                 LogLine ("KNOWN ISSUE (Execution error)");
 442                                                 know_issues.Remove (file);
 443                                                 return;
 444                                         }
 445                                         LogLine ("REGRESSION (SUCCESS -> EXECUTION ERROR)");
 446                                         break;
 447
 448                                 case TestResult.XmlError:
 449                                         if (know_issues.Contains (file)) {
 450                                                 LogLine ("KNOWN ISSUE (Xml comparision error)");
 451                                                 know_issues.Remove (file);
 452                                                 return;
 453                                         }
 454                                         LogLine ("REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
 455                                         break;
 456
 457                                 case TestResult.LoadError:
 458                                         LogLine ("REGRESSION (SUCCESS -> LOAD ERROR)");
 459                                         break;
 460                         }
 461
 462                         if (extra != null)
 463                                 LogLine (extra);
 464
 465                         regression.Add (file);
 466                 }
 467         }
 468
 469         class NegativeChecker: Checker
 470         {
 471                 string expected_message;
 472                 string error_message;
 473
 474                 protected enum CompilerError {
 475                         Expected,
 476                         Wrong,
 477                         Missing,
 478                         WrongMessage
 479                 }
 480
 481                 public NegativeChecker (ITester tester, string log_file, string issue_file):
 482                         base (tester, log_file, issue_file)
 483                 {
 484                 }
 485
 486                 protected override bool AnalyzeTestFile(int row, string line)
 487                 {
 488                         if (row == 1) {
 489                                 expected_message = null;
 490
 491                                 int index = line.IndexOf (':');
 492                                 if (index == -1 || index > 15) {
 493                                         LogLine ("IGNORING: Wrong test file syntax (missing error mesage text)");
 494                                         ++ignored;
 495                                         base.AnalyzeTestFile (row, line);
 496                                         return false;
 497                                 }
 498
 499                                 expected_message = line.Substring (index + 1).Trim ();
 500                         }
 501
 502                         return base.AnalyzeTestFile (row, line);
 503                 }
 504
 505
 506                 protected override bool Check (string filename)
 507                 {
 508                         int start_char = 0;
 509                         while (Char.IsLetter (filename, start_char))
 510                                 ++start_char;
 511
 512                         int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
 513                         string expected = filename.Substring (start_char, end_char - start_char);
 514
 515                         try {
 516                                 if (base.Check (filename)) {
 517                                         HandleFailure (filename, CompilerError.Missing);
 518                                         return false;
 519                                 }
 520                         }
 521                         catch (Exception e) {
 522                                 HandleFailure (filename, CompilerError.Missing);
 523                                 Log (e.ToString ());
 524                                 return false;
 525                         }
 526
 527                         CompilerError result_code = GetCompilerError (expected, tester.Output);
 528                         if (HandleFailure (filename, result_code)) {
 529                                 success++;
 530                                 return true;
 531                         }
 532
 533                         if (result_code == CompilerError.Wrong)
 534                                 LogLine (tester.Output);
 535
 536                         return false;
 537                 }
 538
 539                 CompilerError GetCompilerError (string expected, string buffer)
 540                 {
 541                         const string error_prefix = "CS";
 542                         const string ignored_error = "error CS5001";
 543                         string tested_text = "error " + error_prefix + expected;
 544                         StringReader sr = new StringReader (buffer);
 545                         string line = sr.ReadLine ();
 546                         bool any_error = false;
 547                         while (line != null) {
 548
 549                                 if (line.IndexOf (tested_text) != -1) {
 550 //                                      string msg = line.Substring (line.IndexOf (':', 22) + 1).TrimEnd ('.').Trim ();
 551 //                                      if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
 552 //                                              error_message = msg;
 553 //                                              return CompilerError.WrongMessage;
 554 //                                      }
 555                                         return CompilerError.Expected;
 556                                 }
 557
 558                                 if (line.IndexOf (error_prefix) != -1 &&
 559                                         line.IndexOf (ignored_error) == -1)
 560                                         any_error = true;
 561
 562                                 line = sr.ReadLine ();
 563                         }
 564
 565                         return any_error ? CompilerError.Wrong : CompilerError.Missing;
 566                 }
 567
 568                 bool HandleFailure (string file, CompilerError status)
 569                 {
 570                         switch (status) {
 571                                 case CompilerError.Expected:
 572                                         if (know_issues.Contains (file) || no_error_list.Contains (file)) {
 573                                                 LogLine ("FIXED ISSUE");
 574                                                 return true;
 575                                         }
 576                                         LogLine ("OK");
 577                                         return true;
 578
 579                                 case CompilerError.Wrong:
 580                                         if (know_issues.Contains (file)) {
 581                                                 LogLine ("KNOWN ISSUE (Wrong error reported)");
 582                                                 know_issues.Remove (file);
 583                                                 return false;
 584                                         }
 585                                         if (no_error_list.Contains (file)) {
 586                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR CODE)");
 587                                                 no_error_list.Remove (file);
 588                                         }
 589                                         else {
 590                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
 591                                         }
 592                                         break;
 593
 594                                 case CompilerError.WrongMessage:
 595                                         if (know_issues.Contains (file)) {
 596                                                 LogLine ("KNOWN ISSUE (Wrong error message reported)");
 597                                                 know_issues.Remove (file);
 598                                                 return false;
 599                                         }
 600                                         if (no_error_list.Contains (file)) {
 601                                                 LogLine ("REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
 602                                                 no_error_list.Remove (file);
 603                                         }
 604                                         else {
 605                                                 LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
 606                                                 Console.WriteLine ("E: {0}", expected_message);
 607                                                 Console.WriteLine ("W: {0}", error_message);
 608                                         }
 609                                         break;
 610
 611                                 case CompilerError.Missing:
 612                                         if (no_error_list.Contains (file)) {
 613                                                 LogLine ("KNOWN ISSUE (No error reported)");
 614                                                 no_error_list.Remove (file);
 615                                                 return false;
 616                                         }
 617
 618                                         if (know_issues.Contains (file)) {
 619                                                 LogLine ("REGRESSION (WRONG ERROR -> NO ERROR)");
 620                                                 know_issues.Remove (file);
 621                                         }
 622                                         else {
 623                                                 LogLine ("REGRESSION (CORRECT ERROR -> NO ERROR)");
 624                                         }
 625
 626                                         break;
 627                         }
 628
 629                         regression.Add (file);
 630                         return false;
 631                 }
 632         }
 633
 634         class Tester {
 635
 636                 static int Main(string[] args) {
 637                         if (args.Length != 5) {
 638                                 Console.Error.WriteLine ("Usage: TestRunner [negative|positive] test-pattern compiler know-issues log-file");
 639                                 return 1;
 640                         }
 641
 642                         string mode = args[0].ToLower ();
 643                         string test_pattern = args [1];
 644                         string mcs = args [2];
 645                         string issue_file = args [3];
 646                         string log_fname = args [4];
 647
 648                         string[] files = Directory.GetFiles (".", test_pattern);
 649
 650                         ITester tester;
 651                         try {
 652                                 Console.WriteLine ("Loading: " + mcs);
 653                                 tester = new ReflectionTester (Assembly.LoadFile (mcs));
 654                         }
 655                         catch (Exception) {
 656                                 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
 657                                 if (!File.Exists (mcs)) {
 658                                         Console.Error.WriteLine ("ERROR: Tested compiler was not found");
 659                                         return 1;
 660                                 }
 661                                 tester = new ProcessTester (mcs);
 662                         }
 663
 664                         Checker checker;
 665                         switch (mode) {
 666                                 case "negative":
 667                                         checker = new NegativeChecker (tester, log_fname, issue_file);
 668                                         break;
 669                                 case "positive":
 670                                         checker = new PositiveChecker (tester, log_fname, issue_file);
 671                                         break;
 672                                 default:
 673                                         Console.Error.WriteLine ("You must specify testing mode (positive or negative)");
 674                                         return 1;
 675                         }
 676
 677                         foreach (string s in files) {
 678                                 string filename = Path.GetFileName (s);
 679                                 if (Char.IsUpper (filename, 0)) { // Windows hack
 680                                         continue;
 681                                 }
 682
 683                                 if (filename.EndsWith ("-p2.cs"))
 684                                         continue;
 685
 686                                 checker.Do (filename);
 687                         }
 688
 689                         checker.PrintSummary ();
 690
 691                         checker.Dispose ();
 692
 693                         return checker.ResultCode;
 694                 }
 695         }
 696 }