X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Ftools%2Fcompiler-tester%2Fcompiler-tester.cs;h=07bea6c36c891725f2bd7852ae8aec8f2ded9dd5;hb=2602f1367e4d1d9ae2207ed13040e3594dc40121;hp=973c0e5530011448018ae5a8c9e2667ec166fa79;hpb=0794a76bf95e6535068227930598f78fcfeb2903;p=mono.git diff --git a/mcs/tools/compiler-tester/compiler-tester.cs b/mcs/tools/compiler-tester/compiler-tester.cs index 973c0e55300..07bea6c36c8 100644 --- a/mcs/tools/compiler-tester/compiler-tester.cs +++ b/mcs/tools/compiler-tester/compiler-tester.cs @@ -6,7 +6,7 @@ // // -// Copyright (C) 2008 Novell, Inc (http://www.novell.com) +// Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com) // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the @@ -35,6 +35,7 @@ using System.Reflection; using System.Text; using System.Collections; using System.Xml; +using System.Collections.Generic; namespace TestRunner { @@ -135,7 +136,7 @@ namespace TestRunner { } #endif - class TestCase + class TestCase : MarshalByRefObject { public readonly string FileName; public readonly string[] CompilerOptions; @@ -151,9 +152,9 @@ namespace TestRunner { class PositiveTestCase : TestCase { - public class VerificationData + public class VerificationData : MarshalByRefObject { - public class MethodData + public class MethodData : MarshalByRefObject { public MethodData (MethodBase mi, int il_size) { @@ -172,17 +173,22 @@ namespace TestRunner { public string Type; public string MethodName; public int ILSize; + public bool Checked; } - string test_file; ArrayList methods; + public bool IsNewSet; public VerificationData (string test_file) { +#if NET_2_0 this.test_file = test_file; +#endif } -#if NET_2_0 +#if NET_2_0 + string test_file; + public static VerificationData FromFile (string name, XmlReader r) { VerificationData tc = new VerificationData (name); @@ -213,6 +219,9 @@ namespace TestRunner { string type = null; foreach (MethodData data in methods) { + if (!data.Checked) + continue; + if (type != data.Type) { if (type != null) w.WriteEndElement (); @@ -229,7 +238,9 @@ namespace TestRunner { w.WriteEndElement (); w.WriteEndElement (); } - w.WriteEndElement (); + + if (type != null) + w.WriteEndElement (); w.WriteEndElement (); } @@ -237,6 +248,9 @@ namespace TestRunner { public MethodData FindMethodData (string method_name, string declaring_type) { + if (methods == null) + return null; + foreach (MethodData md in methods) { if (md.MethodName == method_name && md.Type == declaring_type) return md; @@ -244,6 +258,16 @@ namespace TestRunner { return null; } + + public void AddNewMethod (MethodBase mb, int il_size) + { + if (methods == null) + methods = new ArrayList (); + + MethodData md = new MethodData (mb, il_size); + md.Checked = true; + methods.Add (md); + } } VerificationData verif_data; @@ -253,6 +277,12 @@ namespace TestRunner { { } + public void CreateNewTest () + { + verif_data = new VerificationData (FileName); + verif_data.IsNewSet = true; + } + public VerificationData VerificationProvider { set { verif_data = value; @@ -261,38 +291,9 @@ namespace TestRunner { return verif_data; } } - - public bool CompareIL (MethodBase mi, PositiveChecker checker) - { - if (verif_data == null) - verif_data = new VerificationData (FileName); - - string m_name = mi.ToString (); - VerificationData.MethodData md = verif_data.FindMethodData (m_name, mi.DeclaringType.ToString ()); - if (md == null) { - checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, m_name + " (new method?)"); - return false; - } - -#if NET_2_0 - MethodBody body = mi.GetMethodBody (); - int il_size = 0; - if (body != null) - il_size = body.GetILAsByteArray ().Length; - - if (md.ILSize == il_size) - return true; - - checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, - string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size)); - return false; -#else - throw new NotSupportedException (); -#endif - } } - class Checker: IDisposable + class Checker: MarshalByRefObject, IDisposable { protected ITester tester; protected int success; @@ -301,6 +302,7 @@ namespace TestRunner { protected int syntax_errors; string issue_file; StreamWriter log_file; + protected string[] extra_compiler_options; // protected string[] compiler_options; // protected string[] dependencies; @@ -312,6 +314,7 @@ namespace TestRunner { protected ArrayList no_error_list = new ArrayList (); protected bool verbose; + protected bool safe_execution; int total_known_issues; @@ -339,6 +342,18 @@ namespace TestRunner { } } + public bool SafeExecution { + set { + safe_execution = value; + } + } + + public string[] ExtraCompilerOptions { + set { + extra_compiler_options = value; + } + } + protected virtual bool GetExtraOptions (string file, out string[] compiler_options, out string[] dependencies) { @@ -393,6 +408,9 @@ namespace TestRunner { if (test_hash.Contains (filename)) return true; + if (verbose) + Log (filename + "...\t"); + if (ignore_list.Contains (filename)) { ++ignored; LogFileLine (filename, "NOT TESTED"); @@ -405,6 +423,17 @@ namespace TestRunner { return false; } + if (extra_compiler_options != null) { + if (compiler_options == null) + compiler_options = extra_compiler_options; + else { + string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length]; + extra_compiler_options.CopyTo (new_options, 0); + compiler_options.CopyTo (new_options, extra_compiler_options.Length); + compiler_options = new_options; + } + } + TestCase test = CreateTestCase (filename, compiler_options, dependencies); test_hash.Add (filename, test); @@ -428,12 +457,13 @@ namespace TestRunner { string[] test_args; if (test.CompilerOptions != null) { - test_args = new string [1 + test.CompilerOptions.Length]; + test_args = new string [2 + test.CompilerOptions.Length]; test.CompilerOptions.CopyTo (test_args, 0); } else { - test_args = new string [1]; + test_args = new string [2]; } - test_args [test_args.Length - 1] = test.FileName; + test_args [test_args.Length - 2] = test.FileName; + test_args [test_args.Length - 1] = "-debug"; return tester.Invoke (test_args); } @@ -517,6 +547,13 @@ namespace TestRunner { log_file.Write (msg, rest); } + protected void LogLine (string msg) + { + Console.WriteLine (msg); + if (log_file != null) + log_file.WriteLine (msg); + } + protected void LogLine (string msg, params object [] rest) { Console.WriteLine (msg, rest); @@ -524,9 +561,12 @@ namespace TestRunner { log_file.WriteLine (msg, rest); } - protected void LogFileLine (string file, string msg, params object [] args) + public void LogFileLine (string file, string msg, params object [] args) { - string s = file + "...\t" + string.Format (msg, args); + string s = verbose ? + string.Format (msg, args) : + file + "...\t" + string.Format (msg, args); + Console.WriteLine (s); if (log_file != null) log_file.WriteLine (s); @@ -600,6 +640,9 @@ namespace TestRunner { set { update_verif_file = value; } + get { + return update_verif_file; + } } protected override bool GetExtraOptions(string file, out string[] compiler_options, @@ -619,6 +662,127 @@ namespace TestRunner { return true; } + class DomainTester : MarshalByRefObject + { + public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file) + { + Assembly assembly = Assembly.LoadFile (file); + + bool success = true; + Type[] types = assembly.GetTypes (); + foreach (Type t in types) { + + // Skip interfaces + if (!t.IsClass && !t.IsValueType) + continue; + + if (test.VerificationProvider == null) { + if (!checker.UpdateVerificationDataFile) + checker.LogFileLine (test.FileName, "Missing IL verification data"); + test.CreateNewTest (); + } + + foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) { + MethodBase mi = m as MethodBase; + if (mi == null) + continue; + + if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0) + continue; + + success &= CompareIL (mi, test, checker); + } + } + + return success; + } + + bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker) + { + string m_name = mi.ToString (); + string decl_type = mi.DeclaringType.ToString (); + PositiveTestCase.VerificationData data_provider = test.VerificationProvider; + + PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type); + if (md == null) { + data_provider.AddNewMethod (mi, GetILSize (mi)); + if (!data_provider.IsNewSet) { + checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)"); + return false; + } + + return true; + } + + if (md.Checked) { + checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate"); + return false; + } + + md.Checked = true; + + int il_size = GetILSize (mi); + if (md.ILSize == il_size) + return true; + + if (md.ILSize > il_size) { + checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size); + md.ILSize = il_size; + return true; + } + + checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, + string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size)); + + md.ILSize = il_size; + + return false; + } + + static int GetILSize (MethodBase mi) + { +#if NET_2_0 + MethodBody body = mi.GetMethodBody (); + if (body != null) + return body.GetILAsByteArray ().Length; +#endif + return 0; + } + + bool ExecuteFile (MethodInfo entry_point, string filename) + { + TextWriter stdout = Console.Out; + TextWriter stderr = Console.Error; + Console.SetOut (TextWriter.Null); + Console.SetError (TextWriter.Null); + ParameterInfo[] pi = entry_point.GetParameters (); + object[] args = pi.Length == 0 ? null : default_args; + + object result = null; + try { + try { + result = entry_point.Invoke (null, args); + } finally { + Console.SetOut (stdout); + Console.SetError (stderr); + } + } catch (Exception e) { + throw new ApplicationException (e.ToString ()); + } + + if (result is int && (int) result != 0) + throw new ApplicationException ("Wrong return code: " + result.ToString ()); + + return true; + } + + public bool Test (string file) + { + Assembly assembly = Assembly.LoadFile (file); + return ExecuteFile (assembly.EntryPoint, file); + } + } + protected override bool Check(TestCase test) { string filename = test.FileName; @@ -644,7 +808,6 @@ namespace TestRunner { return true; } - MethodInfo mi = null; string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe"); // Enable .dll only tests (no execution required) @@ -653,42 +816,59 @@ namespace TestRunner { return true; } - Assembly assembly = null; - try { - assembly = Assembly.LoadFile (file); - mi = assembly.EntryPoint; - } - catch (FileNotFoundException) { - if (File.Exists (file)) { - Console.WriteLine ("APPDOMAIN LIMIT REACHED"); - } - } - catch (Exception e) { - HandleFailure (filename, TestResult.LoadError, e.ToString ()); - return false; + AppDomain domain = null; +#if !NET_2_1 + if (safe_execution) { + // Create a new AppDomain, with the current directory as the base. + AppDomainSetup setupInfo = new AppDomainSetup (); + setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory; + setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain; + domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo); } - - if (!ExecuteFile (mi, file, filename)) - return false; - - if (doc_output != null) { - string ref_file = filename.Replace (".cs", "-ref.xml"); +#endif + try { + DomainTester tester; try { #if !NET_2_1 - XmlComparer.Compare (ref_file, doc_output); + if (domain != null) + tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName); + else #endif - } - catch (Exception e) { - HandleFailure (filename, TestResult.XmlError, e.Message); + tester = new DomainTester (); + + if (!tester.Test (file)) + return false; + + } catch (ApplicationException e) { + HandleFailure (filename, TestResult.ExecError, e.Message); + return false; + } catch (Exception e) { + HandleFailure (filename, TestResult.LoadError, e.ToString ()); return false; } - } else { - if (verif_file != null) { - PositiveTestCase pt = (PositiveTestCase) test; - pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data [filename]; - if (!CheckILSize (assembly, pt)) + + if (doc_output != null) { + string ref_file = filename.Replace (".cs", "-ref.xml"); + try { +#if !NET_2_1 + XmlComparer.Compare (ref_file, doc_output); +#endif + } catch (Exception e) { + HandleFailure (filename, TestResult.XmlError, e.Message); return false; + } + } else { + if (verif_file != null) { + PositiveTestCase pt = (PositiveTestCase) test; + pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename]; + + if (!tester.CheckILSize (pt, this, file)) + return false; + } } + } finally { + if (domain != null) + AppDomain.Unload (domain); } HandleFailure (filename, TestResult.Success, null); @@ -700,81 +880,6 @@ namespace TestRunner { return new PositiveTestCase (filename, options, deps); } - bool CheckILSize (Assembly assembly, PositiveTestCase test) - { - bool success = true; - Type[] types = assembly.GetTypes (); - foreach (Type t in types) { - if (!t.IsClass && t.IsValueType) - continue; - - foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) { - MethodBase mi = m as MethodBase; - if (mi == null) - continue; - - if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0) - continue; - - success &= test.CompareIL (mi, this); - } - } - - return success; - } - -#if !NET_2_1 - int ExecFile (string exe_name, string filename) - { - if (mono == null) - pi.FileName = exe_name; - else - pi.Arguments = exe_name; - - Process p = Process.Start (pi); - p.WaitForExit (); - return p.ExitCode; - } -#endif - - bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename) - { - TextWriter stdout = Console.Out; - TextWriter stderr = Console.Error; - Console.SetOut (TextWriter.Null); - Console.SetError (TextWriter.Null); - ParameterInfo[] pi = entry_point.GetParameters (); - object[] args = pi.Length == 0 ? null : default_args; - - object result = null; - try { - try { - result = entry_point.Invoke (null, args); - } finally { - Console.SetOut (stdout); - Console.SetError (stderr); - } - } - catch (Exception e) { -#if !NET_2_1 - int exit_code = ExecFile (exe_name, filename); - if (exit_code == 0) { - LogLine ("(appdomain method failed, external executable succeeded)"); - LogLine (e.ToString ()); - return true; - } -#endif - HandleFailure (filename, TestResult.ExecError, e.ToString ()); - return false; - } - - if (result is int && (int)result != 0) { - HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ()); - return false; - } - return true; - } - public void HandleFailure (string file, TestResult status, string extra) { switch (status) { @@ -820,7 +925,9 @@ namespace TestRunner { break; case TestResult.ILError: - LogFileLine (file, "IL REGRESSION: " + extra); + if (!update_verif_file) { + LogFileLine (file, "IL REGRESSION: " + extra); + } extra = null; break; } @@ -828,7 +935,8 @@ namespace TestRunner { if (extra != null) LogLine ("{0}", extra); - regression.Add (file); + if (!regression.Contains (file)) + regression.Add (file); } public override void Initialize () @@ -860,7 +968,7 @@ namespace TestRunner { #if NET_2_0 void LoadVerificationData (string file) { - LogLine ("Loading verification data {0} ...", file); + LogLine ("Loading verification data from `{0}' ...", file); using (XmlReader r = XmlReader.Create (file)) { r.ReadStartElement ("tests"); @@ -879,11 +987,11 @@ namespace TestRunner { void UpdateVerificationData (string file) { - LogLine ("Updating verification data {0}...", file); + LogLine ("Updating verification data `{0}' ...", file); XmlWriterSettings s = new XmlWriterSettings (); s.Indent = true; - using (XmlWriter w = XmlWriter.Create (file, s)) { + using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) { w.WriteStartDocument (); w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test"); w.WriteStartElement ("tests"); @@ -969,7 +1077,7 @@ namespace TestRunner { is_warning = false; if (compiler_options != null) { foreach (string s in compiler_options) { - if (s.EndsWith ("warnaserror")) + if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror")) is_warning = true; } } @@ -1034,7 +1142,7 @@ namespace TestRunner { ArrayList ld = new ArrayList (); CompilerError result = CompilerError.Missing; while (line != null) { - if (ld.Contains (line)) { + if (ld.Contains (line) && result == CompilerError.Expected) { if (line.IndexOf ("Location of the symbol related to previous") == -1) return CompilerError.Duplicate; } @@ -1045,8 +1153,13 @@ namespace TestRunner { if (check_msg) { int first = line.IndexOf (':'); int second = line.IndexOf (':', first + 1); - if (second == -1 || !check_error_line) + if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) { + if (check_error_line) { + second = line.IndexOf (':', second + 1); + } + } else if (second == -1 || !check_error_line) { second = first; + } string msg = line.Substring (second + 1).TrimEnd ('.').Trim (); if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) { @@ -1213,14 +1326,17 @@ namespace TestRunner { } Checker checker; + bool positive; switch (mode) { case "neg": checker = new NegativeChecker (tester, true); + positive = false; break; case "pos": string iltest; GetOption ("il", args, false, out iltest); checker = new PositiveChecker (tester, iltest); + positive = true; if (iltest != null && GetOption ("update-il", args, false, out temp)) { ((PositiveChecker) checker).UpdateVerificationDataFile = true; @@ -1239,7 +1355,12 @@ namespace TestRunner { checker.LogFile = temp; if (GetOption ("verbose", args, false, out temp)) checker.Verbose = true; - + if (GetOption ("safe-execution", args, false, out temp)) + checker.SafeExecution = true; + if (GetOption ("compiler-options", args, true, out temp)) { + string[] extra = temp.Split (' '); + checker.ExtraCompilerOptions = extra; + } string test_pattern; if (!GetOption ("files", args, true, out test_pattern)) { @@ -1247,14 +1368,41 @@ namespace TestRunner { return 1; } - string [] files = Directory.GetFiles (".", test_pattern); - if (files.Length == 0) { + var files = new List (); + switch (test_pattern) { + case "v1": + files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs")); + break; + case "v2": + files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs")); + goto case "v1"; + case "v4": + files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs")); + goto case "v2"; + default: + files.AddRange (Directory.GetFiles (".", test_pattern)); + break; + } + + if (files.Count == 0) { Console.Error.WriteLine ("No files matching `{0}' found", test_pattern); return 2; } checker.Initialize (); +/* + files.Sort ((a, b) => { + if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) { + if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal)) + return -1; + } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) { + if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal)) + return 1; + } + return a.CompareTo (b); + }); +*/ foreach (string s in files) { string filename = Path.GetFileName (s); if (Char.IsUpper (filename, 0)) { // Windows hack @@ -1301,15 +1449,17 @@ namespace TestRunner { static void Usage () { Console.WriteLine ( - "Mono compiler tester, (C) 2008 Novell, Inc.\n" + + "Mono compiler tester, (C) 2009 Novell, Inc.\n" + "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" + " \n" + " -compiler:FILE The file which will be used to compiler tests\n" + + " -compiler-options:OPTIONS Add global compiler options\n" + " -il:IL-FILE XML file with expected IL details for each test\n" + " -issues:FILE The list of expected failures\n" + " -log:FILE Writes any output also to the file\n" + " -help Lists all options\n" + " -mode:[pos|neg] Specifies compiler test mode\n" + + " -safe-execution Runs compiled executables in separate app-domain\n" + " -update-il Updates IL-FILE to match compiler output\n" + " -verbose Prints more details during testing\n" );