X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mcs%2Ftools%2Fcompiler-tester%2Fcompiler-tester.cs;h=07bea6c36c891725f2bd7852ae8aec8f2ded9dd5;hb=2602f1367e4d1d9ae2207ed13040e3594dc40121;hp=2ef3c2ec2c54beebfa366d7ef95eb56afa4cfda4;hpb=b5cfba1835f2ba823796f825410e0062b7e4c9a3;p=mono.git diff --git a/mcs/tools/compiler-tester/compiler-tester.cs b/mcs/tools/compiler-tester/compiler-tester.cs index 2ef3c2ec2c5..07bea6c36c8 100644 --- a/mcs/tools/compiler-tester/compiler-tester.cs +++ b/mcs/tools/compiler-tester/compiler-tester.cs @@ -1,9 +1,41 @@ +// +// compiler-tester.cs +// +// Author: +// Marek Safar (marek.safar@gmail.com) +// + +// +// Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com) +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// + using System; using System.IO; using System.Diagnostics; using System.Reflection; using System.Text; using System.Collections; +using System.Xml; +using System.Collections.Generic; namespace TestRunner { @@ -23,6 +55,10 @@ namespace TestRunner { public ReflectionTester (Assembly a) { Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint"); + + if (t == null) + Console.Error.WriteLine ("null, huh?"); + ep = t.GetMethod ("InvokeCompiler", BindingFlags.Static | BindingFlags.Public); if (ep == null) @@ -54,6 +90,7 @@ namespace TestRunner { } } +#if !NET_2_1 class ProcessTester: ITester { ProcessStartInfo pi; @@ -97,8 +134,9 @@ namespace TestRunner { throw new NotImplementedException (); } } +#endif - class TestCase + class TestCase : MarshalByRefObject { public readonly string FileName; public readonly string[] CompilerOptions; @@ -112,14 +150,159 @@ namespace TestRunner { } } - class Checker: IDisposable + class PositiveTestCase : TestCase + { + public class VerificationData : MarshalByRefObject + { + public class MethodData : MarshalByRefObject + { + public MethodData (MethodBase mi, int il_size) + { + this.Type = mi.DeclaringType.ToString (); + this.MethodName = mi.ToString (); + this.ILSize = il_size; + } + + public MethodData (string type_name, string method_name, int il_size) + { + this.Type = type_name; + this.MethodName = method_name; + this.ILSize = il_size; + } + + public string Type; + public string MethodName; + public int ILSize; + public bool Checked; + } + + ArrayList methods; + public bool IsNewSet; + + public VerificationData (string test_file) + { +#if NET_2_0 + this.test_file = test_file; +#endif + } + +#if NET_2_0 + string test_file; + + public static VerificationData FromFile (string name, XmlReader r) + { + VerificationData tc = new VerificationData (name); + ArrayList methods = new ArrayList (); + r.Read (); + while (r.ReadToNextSibling ("type")) { + string type_name = r ["name"]; + r.Read (); + while (r.ReadToNextSibling ("method")) { + string m_name = r ["name"]; + + r.ReadToDescendant ("size"); + int il_size = r.ReadElementContentAsInt (); + methods.Add (new MethodData (type_name, m_name, il_size)); + r.Read (); + } + r.Read (); + } + + tc.methods = methods; + return tc; + } + + public void WriteCodeInfoTo (XmlWriter w) + { + w.WriteStartElement ("test"); + w.WriteAttributeString ("name", test_file); + + string type = null; + foreach (MethodData data in methods) { + if (!data.Checked) + continue; + + if (type != data.Type) { + if (type != null) + w.WriteEndElement (); + + type = data.Type; + w.WriteStartElement ("type"); + w.WriteAttributeString ("name", type); + } + + w.WriteStartElement ("method"); + w.WriteAttributeString ("name", data.MethodName); + w.WriteStartElement ("size"); + w.WriteValue (data.ILSize); + w.WriteEndElement (); + w.WriteEndElement (); + } + + if (type != null) + w.WriteEndElement (); + + w.WriteEndElement (); + } +#endif + + public MethodData FindMethodData (string method_name, string declaring_type) + { + if (methods == null) + return null; + + foreach (MethodData md in methods) { + if (md.MethodName == method_name && md.Type == declaring_type) + return md; + } + + return null; + } + + public void AddNewMethod (MethodBase mb, int il_size) + { + if (methods == null) + methods = new ArrayList (); + + MethodData md = new MethodData (mb, il_size); + md.Checked = true; + methods.Add (md); + } + } + + VerificationData verif_data; + + public PositiveTestCase (string filename, string [] options, string [] deps) + : base (filename, options, deps) + { + } + + public void CreateNewTest () + { + verif_data = new VerificationData (FileName); + verif_data.IsNewSet = true; + } + + public VerificationData VerificationProvider { + set { + verif_data = value; + } + get { + return verif_data; + } + } + } + + class Checker: MarshalByRefObject, IDisposable { protected ITester tester; protected int success; protected int total; protected int ignored; + protected int syntax_errors; string issue_file; StreamWriter log_file; + protected string[] extra_compiler_options; // protected string[] compiler_options; // protected string[] dependencies; @@ -129,15 +312,46 @@ namespace TestRunner { protected ArrayList know_issues = new ArrayList (); protected ArrayList ignore_list = new ArrayList (); protected ArrayList no_error_list = new ArrayList (); + + protected bool verbose; + protected bool safe_execution; int total_known_issues; - protected Checker (ITester tester, string log_file, string issue_file) + protected Checker (ITester tester) { this.tester = tester; - this.issue_file = issue_file; - ReadWrongErrors (issue_file); - this.log_file = new StreamWriter (log_file, false); + } + + public string IssueFile { + set { + this.issue_file = value; + ReadWrongErrors (issue_file); + } + } + + public string LogFile { + set { + this.log_file = new StreamWriter (value, false); + } + } + + public bool Verbose { + set { + verbose = value; + } + } + + public bool SafeExecution { + set { + safe_execution = value; + } + } + + public string[] ExtraCompilerOptions { + set { + extra_compiler_options = value; + } } protected virtual bool GetExtraOptions (string file, out string[] compiler_options, @@ -150,7 +364,7 @@ namespace TestRunner { using (StreamReader sr = new StreamReader (file)) { String line; while (row++ < 3 && (line = sr.ReadLine()) != null) { - if (!AnalyzeTestFile (ref row, line, ref compiler_options, + if (!AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies)) return false; } @@ -161,7 +375,7 @@ namespace TestRunner { return true; } - protected virtual bool AnalyzeTestFile (ref int row, string line, + protected virtual bool AnalyzeTestFile (string file, ref int row, string line, ref string[] compiler_options, ref string[] dependencies) { @@ -194,26 +408,40 @@ namespace TestRunner { if (test_hash.Contains (filename)) return true; + if (verbose) + Log (filename + "...\t"); + if (ignore_list.Contains (filename)) { ++ignored; - LogLine (filename + "...\tNOT TESTED"); + LogFileLine (filename, "NOT TESTED"); return false; } string[] compiler_options, dependencies; if (!GetExtraOptions (filename, out compiler_options, out dependencies)) { - LogLine (filename + "...\tERROR"); + LogFileLine (filename, "ERROR"); return false; } - TestCase test = new TestCase (filename, compiler_options, dependencies); + if (extra_compiler_options != null) { + if (compiler_options == null) + compiler_options = extra_compiler_options; + else { + string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length]; + extra_compiler_options.CopyTo (new_options, 0); + compiler_options.CopyTo (new_options, extra_compiler_options.Length); + compiler_options = new_options; + } + } + + TestCase test = CreateTestCase (filename, compiler_options, dependencies); test_hash.Add (filename, test); ++total; if (dependencies != null) { foreach (string dependency in dependencies) { if (!Do (dependency)) { - LogLine (filename + "...\tDEPENDENCY FAILED"); + LogFileLine (filename, "DEPENDENCY FAILED"); return false; } } @@ -221,9 +449,6 @@ namespace TestRunner { tests.Add (test); - Log (filename); - Log ("...\t"); - return Check (test); } @@ -232,16 +457,21 @@ namespace TestRunner { string[] test_args; if (test.CompilerOptions != null) { - test_args = new string [1 + test.CompilerOptions.Length]; + test_args = new string [2 + test.CompilerOptions.Length]; test.CompilerOptions.CopyTo (test_args, 0); } else { - test_args = new string [1]; + test_args = new string [2]; } - test_args [test_args.Length - 1] = test.FileName; + test_args [test_args.Length - 2] = test.FileName; + test_args [test_args.Length - 1] = "-debug"; return tester.Invoke (test_args); } + protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps) + { + return new TestCase (filename, options, deps); + } void ReadWrongErrors (string file) { @@ -271,16 +501,22 @@ namespace TestRunner { total_known_issues = know_issues.Count; } - public virtual void PrintSummary () + protected virtual void PrintSummary () { LogLine ("Done" + Environment.NewLine); - LogLine ("{0} test cases passed ({1:.##%})", success, (float) (success) / (float)total); - + float rate = 0; + if (total > 0) + rate = (float) (success) / (float)total; + LogLine ("{0} test cases passed ({1:0.##%})", success, rate); + + if (syntax_errors > 0) + LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors); + if (ignored > 0) - LogLine ("{0} test cases ignored", ignored); + LogLine ("{0} test(s) ignored", ignored); if (total_known_issues - know_issues.Count > 0) - LogLine ("{0} known issues", total_known_issues - know_issues.Count); + LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count); know_issues.AddRange (no_error_list); if (know_issues.Count > 0) { @@ -307,55 +543,85 @@ namespace TestRunner { protected void Log (string msg, params object [] rest) { Console.Write (msg, rest); - log_file.Write (msg, rest); + if (log_file != null) + log_file.Write (msg, rest); + } + + protected void LogLine (string msg) + { + Console.WriteLine (msg); + if (log_file != null) + log_file.WriteLine (msg); } protected void LogLine (string msg, params object [] rest) { Console.WriteLine (msg, rest); - log_file.WriteLine (msg, rest); + if (log_file != null) + log_file.WriteLine (msg, rest); + } + + public void LogFileLine (string file, string msg, params object [] args) + { + string s = verbose ? + string.Format (msg, args) : + file + "...\t" + string.Format (msg, args); + + Console.WriteLine (s); + if (log_file != null) + log_file.WriteLine (s); } #region IDisposable Members public void Dispose() { - log_file.Close (); + if (log_file != null) + log_file.Close (); } #endregion + + public virtual void Initialize () + { + } + + public virtual void CleanUp () + { + PrintSummary (); + } } - class PositiveChecker: Checker { + class PositiveChecker: Checker + { readonly string files_folder; readonly static object[] default_args = new object[1] { new string[] {} }; string doc_output; + string verif_file; + bool update_verif_file; + Hashtable verif_data; - // When we cannot load assembly to domain we are automaticaly switching to process calling - bool appdomain_limit_reached; - +#if !NET_2_1 ProcessStartInfo pi; +#endif readonly string mono; - // This is really pain - // This number is highly experimental on my box I am not able to load more than 1000 files to domain - // Every files is there twice and we need some space for tests which reference assemblies - const int MAX_TESTS_IN_DOMAIN = 420; - int test_counter; - - protected enum TestResult { + public enum TestResult { CompileError, ExecError, LoadError, XmlError, - Success + Success, + ILError } - public PositiveChecker (ITester tester, string log_file, string issue_file): - base (tester, log_file, issue_file) + public PositiveChecker (ITester tester, string verif_file): + base (tester) { files_folder = Directory.GetCurrentDirectory (); + this.verif_file = verif_file; +#if !NET_2_1 pi = new ProcessStartInfo (); pi.CreateNoWindow = true; pi.WindowStyle = ProcessWindowStyle.Hidden; @@ -367,6 +633,16 @@ namespace TestRunner { if (mono != null) { pi.FileName = mono; } +#endif + } + + public bool UpdateVerificationDataFile { + set { + update_verif_file = value; + } + get { + return update_verif_file; + } } protected override bool GetExtraOptions(string file, out string[] compiler_options, @@ -380,12 +656,133 @@ namespace TestRunner { foreach (string one_opt in compiler_options) { if (one_opt.StartsWith ("-doc:")) { - doc_output = one_opt.Split (':')[1]; + doc_output = one_opt.Split (':', '/')[1]; } } return true; } + class DomainTester : MarshalByRefObject + { + public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file) + { + Assembly assembly = Assembly.LoadFile (file); + + bool success = true; + Type[] types = assembly.GetTypes (); + foreach (Type t in types) { + + // Skip interfaces + if (!t.IsClass && !t.IsValueType) + continue; + + if (test.VerificationProvider == null) { + if (!checker.UpdateVerificationDataFile) + checker.LogFileLine (test.FileName, "Missing IL verification data"); + test.CreateNewTest (); + } + + foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) { + MethodBase mi = m as MethodBase; + if (mi == null) + continue; + + if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0) + continue; + + success &= CompareIL (mi, test, checker); + } + } + + return success; + } + + bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker) + { + string m_name = mi.ToString (); + string decl_type = mi.DeclaringType.ToString (); + PositiveTestCase.VerificationData data_provider = test.VerificationProvider; + + PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type); + if (md == null) { + data_provider.AddNewMethod (mi, GetILSize (mi)); + if (!data_provider.IsNewSet) { + checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)"); + return false; + } + + return true; + } + + if (md.Checked) { + checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate"); + return false; + } + + md.Checked = true; + + int il_size = GetILSize (mi); + if (md.ILSize == il_size) + return true; + + if (md.ILSize > il_size) { + checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size); + md.ILSize = il_size; + return true; + } + + checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, + string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size)); + + md.ILSize = il_size; + + return false; + } + + static int GetILSize (MethodBase mi) + { +#if NET_2_0 + MethodBody body = mi.GetMethodBody (); + if (body != null) + return body.GetILAsByteArray ().Length; +#endif + return 0; + } + + bool ExecuteFile (MethodInfo entry_point, string filename) + { + TextWriter stdout = Console.Out; + TextWriter stderr = Console.Error; + Console.SetOut (TextWriter.Null); + Console.SetError (TextWriter.Null); + ParameterInfo[] pi = entry_point.GetParameters (); + object[] args = pi.Length == 0 ? null : default_args; + + object result = null; + try { + try { + result = entry_point.Invoke (null, args); + } finally { + Console.SetOut (stdout); + Console.SetError (stderr); + } + } catch (Exception e) { + throw new ApplicationException (e.ToString ()); + } + + if (result is int && (int) result != 0) + throw new ApplicationException ("Wrong return code: " + result.ToString ()); + + return true; + } + + public bool Test (string file) + { + Assembly assembly = Assembly.LoadFile (file); + return ExecuteFile (assembly.EntryPoint, file); + } + } + protected override bool Check(TestCase test) { string filename = test.FileName; @@ -396,167 +793,216 @@ namespace TestRunner { } } catch (Exception e) { + if (e.InnerException != null) + e = e.InnerException; + HandleFailure (filename, TestResult.CompileError, e.ToString ()); return false; } // Test setup if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) { - LogLine ("OK"); + if (verbose) + LogFileLine (filename, "OK"); --total; return true; } - MethodInfo mi = null; string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe"); - if (!appdomain_limit_reached) { + + // Enable .dll only tests (no execution required) + if (!File.Exists(file)) { + HandleFailure (filename, TestResult.Success, null); + return true; + } + + AppDomain domain = null; +#if !NET_2_1 + if (safe_execution) { + // Create a new AppDomain, with the current directory as the base. + AppDomainSetup setupInfo = new AppDomainSetup (); + setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory; + setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain; + domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo); + } +#endif + try { + DomainTester tester; try { - mi = Assembly.LoadFile (file).EntryPoint; - if (test_counter++ > MAX_TESTS_IN_DOMAIN) - appdomain_limit_reached = true; - } - catch (FileNotFoundException) { - if (File.Exists (file)) { - Console.WriteLine ("APPDOMAIN LIMIT REACHED"); - appdomain_limit_reached = true; - } - } - catch (Exception e) { +#if !NET_2_1 + if (domain != null) + tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName); + else +#endif + tester = new DomainTester (); + + if (!tester.Test (file)) + return false; + + } catch (ApplicationException e) { + HandleFailure (filename, TestResult.ExecError, e.Message); + return false; + } catch (Exception e) { HandleFailure (filename, TestResult.LoadError, e.ToString ()); return false; } - } - if (appdomain_limit_reached) { - if (!ExecuteFile (file, filename)) - return false; - } else { - if (!ExecuteFile (mi, file, filename)) - return false; - } + if (doc_output != null) { + string ref_file = filename.Replace (".cs", "-ref.xml"); + try { +#if !NET_2_1 + XmlComparer.Compare (ref_file, doc_output); +#endif + } catch (Exception e) { + HandleFailure (filename, TestResult.XmlError, e.Message); + return false; + } + } else { + if (verif_file != null) { + PositiveTestCase pt = (PositiveTestCase) test; + pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename]; - if (doc_output != null) { - string ref_file = filename.Replace (".cs", "-ref.xml"); - try { - XmlComparer.Compare (ref_file, doc_output); - } - catch (Exception e) { - HandleFailure (filename, TestResult.XmlError, e.Message); - return false; + if (!tester.CheckILSize (pt, this, file)) + return false; + } } + } finally { + if (domain != null) + AppDomain.Unload (domain); } HandleFailure (filename, TestResult.Success, null); return true; } - int ExecFile (string exe_name, string filename) + protected override TestCase CreateTestCase (string filename, string [] options, string [] deps) { - if (mono == null) - pi.FileName = exe_name; - else - pi.Arguments = exe_name; - - Process p = Process.Start (pi); - p.WaitForExit (); - return p.ExitCode; - } - - bool ExecuteFile (string exe_name, string filename) - { - int exit_code = ExecFile (exe_name, filename); - if (exit_code == 0) - return true; - - HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + exit_code); - return false; + return new PositiveTestCase (filename, options, deps); } - bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename) - { - TextWriter stdout = Console.Out; - TextWriter stderr = Console.Error; - Console.SetOut (TextWriter.Null); - Console.SetError (TextWriter.Null); - ParameterInfo[] pi = entry_point.GetParameters (); - object[] args = pi.Length == 0 ? null : default_args; - - object result = null; - try { - try { - result = entry_point.Invoke (null, args); - } finally { - Console.SetOut (stdout); - Console.SetError (stderr); - } - } - catch (Exception e) { - int exit_code = ExecFile (exe_name, filename); - if (exit_code == 0) { - LogLine ("(appdomain method failed, external executable succeeded)"); - LogLine (e.ToString ()); - return true; - } - HandleFailure (filename, TestResult.ExecError, e.ToString ()); - return false; - } - - if (result is int && (int)result != 0) { - HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ()); - return false; - } - return true; - } - - void HandleFailure (string file, TestResult status, string extra) + public void HandleFailure (string file, TestResult status, string extra) { switch (status) { case TestResult.Success: success++; if (know_issues.Contains (file)) { - LogLine ("FIXED ISSUE"); + LogFileLine (file, "FIXED ISSUE"); return; } - LogLine ("OK"); + if (verbose) + LogFileLine (file, "OK"); return; case TestResult.CompileError: if (know_issues.Contains (file)) { - LogLine ("KNOWN ISSUE (Compilation error)"); + LogFileLine (file, "KNOWN ISSUE (Compilation error)"); know_issues.Remove (file); return; } - LogLine ("REGRESSION (SUCCESS -> COMPILATION ERROR)"); + LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)"); break; case TestResult.ExecError: if (know_issues.Contains (file)) { - LogLine ("KNOWN ISSUE (Execution error)"); + LogFileLine (file, "KNOWN ISSUE (Execution error)"); know_issues.Remove (file); return; } - LogLine ("REGRESSION (SUCCESS -> EXECUTION ERROR)"); + LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)"); break; case TestResult.XmlError: if (know_issues.Contains (file)) { - LogLine ("KNOWN ISSUE (Xml comparision error)"); + LogFileLine (file, "KNOWN ISSUE (Xml comparision error)"); know_issues.Remove (file); return; } - LogLine ("REGRESSION (SUCCESS -> DOCUMENTATION ERROR)"); + LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)"); break; case TestResult.LoadError: - LogLine ("REGRESSION (SUCCESS -> LOAD ERROR)"); + LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)"); + break; + + case TestResult.ILError: + if (!update_verif_file) { + LogFileLine (file, "IL REGRESSION: " + extra); + } + extra = null; break; } if (extra != null) - LogLine (extra); + LogLine ("{0}", extra); - regression.Add (file); + if (!regression.Contains (file)) + regression.Add (file); + } + + public override void Initialize () + { + if (verif_file != null) { +#if NET_2_0 + LoadVerificationData (verif_file); +#else + throw new NotSupportedException (); +#endif + } + + base.Initialize (); + } + + public override void CleanUp () + { + base.CleanUp (); + + if (update_verif_file) { +#if NET_2_0 + UpdateVerificationData (verif_file); +#else + throw new NotSupportedException (); +#endif + } + } + +#if NET_2_0 + void LoadVerificationData (string file) + { + LogLine ("Loading verification data from `{0}' ...", file); + + using (XmlReader r = XmlReader.Create (file)) { + r.ReadStartElement ("tests"); + verif_data = new Hashtable (); + + while (r.Read ()) { + if (r.Name != "test") + continue; + + string name = r.GetAttribute ("name"); + PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r); + verif_data.Add (name, tc); + } + } } + + void UpdateVerificationData (string file) + { + LogLine ("Updating verification data `{0}' ...", file); + + XmlWriterSettings s = new XmlWriterSettings (); + s.Indent = true; + using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) { + w.WriteStartDocument (); + w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test"); + w.WriteStartElement ("tests"); + foreach (PositiveTestCase tc in tests) { + if (tc.VerificationProvider != null) + tc.VerificationProvider.WriteCodeInfoTo (w); + } + w.WriteEndElement (); + } + } +#endif } class NegativeChecker: Checker @@ -577,14 +1023,14 @@ namespace TestRunner { Duplicate } - public NegativeChecker (ITester tester, string log_file, string issue_file, bool check_msg): - base (tester, log_file, issue_file) + public NegativeChecker (ITester tester, bool check_msg): + base (tester) { this.check_msg = check_msg; wrong_warning = new Hashtable (); } - protected override bool AnalyzeTestFile (ref int row, string line, + protected override bool AnalyzeTestFile (string file, ref int row, string line, ref string[] compiler_options, ref string[] dependencies) { @@ -593,9 +1039,9 @@ namespace TestRunner { int index = line.IndexOf (':'); if (index == -1 || index > 15) { - LogLine ("IGNORING: Wrong test file syntax (missing error mesage text)"); - ++ignored; - base.AnalyzeTestFile (ref row, line, ref compiler_options, + LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)"); + ++syntax_errors; + base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies); return false; } @@ -612,26 +1058,26 @@ namespace TestRunner { #if !NET_2_0 return true; #else - return AnalyzeTestFile(ref row, line, ref compiler_options, ref dependencies); + return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies); #endif } check_error_line = !filtered.StartsWith ("//Line:0"); if (!filtered.StartsWith ("//Line:")) { - LogLine ("IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax"); - ++ignored; + LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax"); + ++syntax_errors; return false; } } - if (!base.AnalyzeTestFile (ref row, line, ref compiler_options, ref dependencies)) + if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies)) return false; is_warning = false; if (compiler_options != null) { foreach (string s in compiler_options) { - if (s.EndsWith ("warnaserror")) + if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror")) is_warning = true; } } @@ -658,6 +1104,9 @@ namespace TestRunner { } catch (Exception e) { HandleFailure (filename, CompilerError.Missing); + if (e.InnerException != null) + e = e.InnerException; + Log (e.ToString ()); return false; } @@ -693,7 +1142,7 @@ namespace TestRunner { ArrayList ld = new ArrayList (); CompilerError result = CompilerError.Missing; while (line != null) { - if (ld.Contains (line)) { + if (ld.Contains (line) && result == CompilerError.Expected) { if (line.IndexOf ("Location of the symbol related to previous") == -1) return CompilerError.Duplicate; } @@ -704,8 +1153,13 @@ namespace TestRunner { if (check_msg) { int first = line.IndexOf (':'); int second = line.IndexOf (':', first + 1); - if (second == -1 || !check_error_line) + if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) { + if (check_error_line) { + second = line.IndexOf (':', second + 1); + } + } else if (second == -1 || !check_error_line) { second = first; + } string msg = line.Substring (second + 1).TrimEnd ('.').Trim (); if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) { @@ -733,39 +1187,41 @@ namespace TestRunner { switch (status) { case CompilerError.Expected: if (know_issues.Contains (file) || no_error_list.Contains (file)) { - LogLine ("FIXED ISSUE"); + LogFileLine (file, "FIXED ISSUE"); return true; } - LogLine ("OK"); + + if (verbose) + LogFileLine (file, "OK"); return true; case CompilerError.Wrong: if (know_issues.Contains (file)) { - LogLine ("KNOWN ISSUE (Wrong error reported)"); + LogFileLine (file, "KNOWN ISSUE (Wrong error reported)"); know_issues.Remove (file); return false; } if (no_error_list.Contains (file)) { - LogLine ("REGRESSION (NO ERROR -> WRONG ERROR CODE)"); + LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)"); no_error_list.Remove (file); } else { - LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)"); + LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)"); } break; case CompilerError.WrongMessage: if (know_issues.Contains (file)) { - LogLine ("KNOWN ISSUE (Wrong error message reported)"); + LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)"); know_issues.Remove (file); return false; } if (no_error_list.Contains (file)) { - LogLine ("REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)"); + LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)"); no_error_list.Remove (file); } else { - LogLine ("REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)"); + LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)"); LogLine ("Exp: {0}", expected_message); LogLine ("Was: {0}", error_message); } @@ -773,39 +1229,39 @@ namespace TestRunner { case CompilerError.Missing: if (no_error_list.Contains (file)) { - LogLine ("KNOWN ISSUE (No error reported)"); + LogFileLine (file, "KNOWN ISSUE (No error reported)"); no_error_list.Remove (file); return false; } if (know_issues.Contains (file)) { - LogLine ("REGRESSION (WRONG ERROR -> NO ERROR)"); + LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)"); know_issues.Remove (file); } else { - LogLine ("REGRESSION (CORRECT ERROR -> NO ERROR)"); + LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)"); } break; case CompilerError.MissingLocation: if (know_issues.Contains (file)) { - LogLine ("KNOWN ISSUE (Missing error location)"); + LogFileLine (file, "KNOWN ISSUE (Missing error location)"); know_issues.Remove (file); return false; } if (no_error_list.Contains (file)) { - LogLine ("REGRESSION (NO ERROR -> MISSING ERROR LOCATION)"); + LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)"); no_error_list.Remove (file); } else { - LogLine ("REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)"); + LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)"); } break; case CompilerError.Duplicate: // Will become an error soon - LogLine("WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES"); + LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES"); return true; } @@ -813,7 +1269,7 @@ namespace TestRunner { return false; } - public override void PrintSummary() + protected override void PrintSummary() { base.PrintSummary (); @@ -830,51 +1286,123 @@ namespace TestRunner { class Tester { - static int Main(string[] args) { - if (args.Length != 5) { - Console.Error.WriteLine ("Usage: TestRunner [negative|positive] test-pattern compiler know-issues log-file"); - return 1; - } + static int Main(string[] args) + { + string temp; - string mode = args[0].ToLower (); -#if NET_2_0 - string test_pattern = args [1] == "0" ? "*cs*.cs" : "*test-*.cs"; //args [1]; -#else - string test_pattern = args [1] == "0" ? "cs*.cs" : "test-*.cs"; //args [1]; -#endif - string mcs = args [2]; - string issue_file = args [3]; - string log_fname = args [4]; + if (GetOption ("help", args, false, out temp)) { + Usage (); + return 0; + } - string[] files = Directory.GetFiles (".", test_pattern); + string compiler; + if (!GetOption ("compiler", args, true, out compiler)) { + Usage (); + return 1; + } ITester tester; try { - Console.WriteLine ("Loading: " + mcs); - tester = new ReflectionTester (Assembly.LoadFile (mcs)); + Console.WriteLine ("Loading " + compiler + " ..."); + tester = new ReflectionTester (Assembly.LoadFile (compiler)); } catch (Exception) { +#if NET_2_1 + throw; +#else Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)"); - if (!File.Exists (mcs)) { + if (!File.Exists (compiler)) { Console.Error.WriteLine ("ERROR: Tested compiler was not found"); return 1; } - tester = new ProcessTester (mcs); + tester = new ProcessTester (compiler); +#endif + } + + string mode; + if (!GetOption ("mode", args, true, out mode)) { + Usage (); + return 1; } Checker checker; + bool positive; switch (mode) { - case "negative": - checker = new NegativeChecker (tester, log_fname, issue_file, true); + case "neg": + checker = new NegativeChecker (tester, true); + positive = false; break; - case "positive": - checker = new PositiveChecker (tester, log_fname, issue_file); + case "pos": + string iltest; + GetOption ("il", args, false, out iltest); + checker = new PositiveChecker (tester, iltest); + positive = true; + + if (iltest != null && GetOption ("update-il", args, false, out temp)) { + ((PositiveChecker) checker).UpdateVerificationDataFile = true; + } + break; default: - Console.Error.WriteLine ("You must specify testing mode (positive or negative)"); + Console.Error.WriteLine ("Invalid -mode argument"); return 1; } + + if (GetOption ("issues", args, true, out temp)) + checker.IssueFile = temp; + if (GetOption ("log", args, true, out temp)) + checker.LogFile = temp; + if (GetOption ("verbose", args, false, out temp)) + checker.Verbose = true; + if (GetOption ("safe-execution", args, false, out temp)) + checker.SafeExecution = true; + if (GetOption ("compiler-options", args, true, out temp)) { + string[] extra = temp.Split (' '); + checker.ExtraCompilerOptions = extra; + } + + string test_pattern; + if (!GetOption ("files", args, true, out test_pattern)) { + Usage (); + return 1; + } + + var files = new List (); + switch (test_pattern) { + case "v1": + files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs")); + break; + case "v2": + files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs")); + goto case "v1"; + case "v4": + files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs")); + goto case "v2"; + default: + files.AddRange (Directory.GetFiles (".", test_pattern)); + break; + } + + if (files.Count == 0) { + Console.Error.WriteLine ("No files matching `{0}' found", test_pattern); + return 2; + } + + checker.Initialize (); +/* + files.Sort ((a, b) => { + if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) { + if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal)) + return -1; + } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) { + if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal)) + return 1; + } + + return a.CompareTo (b); + }); +*/ foreach (string s in files) { string filename = Path.GetFileName (s); if (Char.IsUpper (filename, 0)) { // Windows hack @@ -887,11 +1415,54 @@ namespace TestRunner { checker.Do (filename); } - checker.PrintSummary (); + checker.CleanUp (); checker.Dispose (); return checker.ResultCode; } + + static bool GetOption (string opt, string[] args, bool req_arg, out string value) + { + opt = "-" + opt; + foreach (string a in args) { + if (a.StartsWith (opt)) { + int sep = a.IndexOf (':'); + if (sep > 0) { + value = a.Substring (sep + 1); + } else { + value = null; + if (req_arg) { + Console.Error.WriteLine ("Missing argument in option " + opt); + return false; + } + } + + return true; + } + } + + value = null; + return false; + } + + static void Usage () + { + Console.WriteLine ( + "Mono compiler tester, (C) 2009 Novell, Inc.\n" + + "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" + + " \n" + + " -compiler:FILE The file which will be used to compiler tests\n" + + " -compiler-options:OPTIONS Add global compiler options\n" + + " -il:IL-FILE XML file with expected IL details for each test\n" + + " -issues:FILE The list of expected failures\n" + + " -log:FILE Writes any output also to the file\n" + + " -help Lists all options\n" + + " -mode:[pos|neg] Specifies compiler test mode\n" + + " -safe-execution Runs compiled executables in separate app-domain\n" + + " -update-il Updates IL-FILE to match compiler output\n" + + " -verbose Prints more details during testing\n" + ); + } } }