//
//
-// Copyright (C) 2008 Novell, Inc (http://www.novell.com)
+// Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
using System.Text;
using System.Collections;
using System.Xml;
+using System.Collections.Generic;
namespace TestRunner {
}
#endif
- class TestCase
+ class TestCase : MarshalByRefObject
{
public readonly string FileName;
public readonly string[] CompilerOptions;
class PositiveTestCase : TestCase
{
- public class VerificationData
+ public class VerificationData : MarshalByRefObject
{
- public class MethodData
+ public class MethodData : MarshalByRefObject
{
public MethodData (MethodBase mi, int il_size)
{
public string Type;
public string MethodName;
public int ILSize;
+ public bool Checked;
}
- string test_file;
ArrayList methods;
+ public bool IsNewSet;
public VerificationData (string test_file)
{
+#if NET_2_0
this.test_file = test_file;
+#endif
}
-#if NET_2_0
+#if NET_2_0
+ string test_file;
+
public static VerificationData FromFile (string name, XmlReader r)
{
VerificationData tc = new VerificationData (name);
string type = null;
foreach (MethodData data in methods) {
+ if (!data.Checked)
+ continue;
+
if (type != data.Type) {
if (type != null)
w.WriteEndElement ();
w.WriteEndElement ();
w.WriteEndElement ();
}
- w.WriteEndElement ();
+
+ if (type != null)
+ w.WriteEndElement ();
w.WriteEndElement ();
}
public MethodData FindMethodData (string method_name, string declaring_type)
{
+ if (methods == null)
+ return null;
+
foreach (MethodData md in methods) {
if (md.MethodName == method_name && md.Type == declaring_type)
return md;
return null;
}
+
+ public void AddNewMethod (MethodBase mb, int il_size)
+ {
+ if (methods == null)
+ methods = new ArrayList ();
+
+ MethodData md = new MethodData (mb, il_size);
+ md.Checked = true;
+ methods.Add (md);
+ }
}
VerificationData verif_data;
{
}
+ public void CreateNewTest ()
+ {
+ verif_data = new VerificationData (FileName);
+ verif_data.IsNewSet = true;
+ }
+
public VerificationData VerificationProvider {
set {
verif_data = value;
return verif_data;
}
}
-
- public bool CompareIL (MethodBase mi, PositiveChecker checker)
- {
- if (verif_data == null)
- verif_data = new VerificationData (FileName);
-
- string m_name = mi.ToString ();
- VerificationData.MethodData md = verif_data.FindMethodData (m_name, mi.DeclaringType.ToString ());
- if (md == null) {
- checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError, m_name + " (new method?)");
- return false;
- }
-
-#if NET_2_0
- MethodBody body = mi.GetMethodBody ();
- int il_size = 0;
- if (body != null)
- il_size = body.GetILAsByteArray ().Length;
-
- if (md.ILSize == il_size)
- return true;
-
- checker.HandleFailure (FileName, PositiveChecker.TestResult.ILError,
- string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
- return false;
-#else
- throw new NotSupportedException ();
-#endif
- }
}
- class Checker: IDisposable
+ class Checker: MarshalByRefObject, IDisposable
{
protected ITester tester;
protected int success;
protected int syntax_errors;
string issue_file;
StreamWriter log_file;
+ protected string[] extra_compiler_options;
// protected string[] compiler_options;
// protected string[] dependencies;
protected ArrayList no_error_list = new ArrayList ();
protected bool verbose;
+ protected bool safe_execution;
int total_known_issues;
}
}
+ public bool SafeExecution {
+ set {
+ safe_execution = value;
+ }
+ }
+
+ public string[] ExtraCompilerOptions {
+ set {
+ extra_compiler_options = value;
+ }
+ }
+
protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
out string[] dependencies)
{
if (test_hash.Contains (filename))
return true;
+ if (verbose)
+ Log (filename + "...\t");
+
if (ignore_list.Contains (filename)) {
++ignored;
LogFileLine (filename, "NOT TESTED");
return false;
}
+ if (extra_compiler_options != null) {
+ if (compiler_options == null)
+ compiler_options = extra_compiler_options;
+ else {
+ string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
+ extra_compiler_options.CopyTo (new_options, 0);
+ compiler_options.CopyTo (new_options, extra_compiler_options.Length);
+ compiler_options = new_options;
+ }
+ }
+
TestCase test = CreateTestCase (filename, compiler_options, dependencies);
test_hash.Add (filename, test);
string[] test_args;
if (test.CompilerOptions != null) {
- test_args = new string [1 + test.CompilerOptions.Length];
+ test_args = new string [2 + test.CompilerOptions.Length];
test.CompilerOptions.CopyTo (test_args, 0);
} else {
- test_args = new string [1];
+ test_args = new string [2];
}
- test_args [test_args.Length - 1] = test.FileName;
+ test_args [test_args.Length - 2] = test.FileName;
+ test_args [test_args.Length - 1] = "-debug";
return tester.Invoke (test_args);
}
log_file.Write (msg, rest);
}
+ protected void LogLine (string msg)
+ {
+ Console.WriteLine (msg);
+ if (log_file != null)
+ log_file.WriteLine (msg);
+ }
+
protected void LogLine (string msg, params object [] rest)
{
Console.WriteLine (msg, rest);
log_file.WriteLine (msg, rest);
}
- protected void LogFileLine (string file, string msg, params object [] args)
+ public void LogFileLine (string file, string msg, params object [] args)
{
- string s = file + "...\t" + string.Format (msg, args);
+ string s = verbose ?
+ string.Format (msg, args) :
+ file + "...\t" + string.Format (msg, args);
+
Console.WriteLine (s);
if (log_file != null)
log_file.WriteLine (s);
set {
update_verif_file = value;
}
+ get {
+ return update_verif_file;
+ }
}
protected override bool GetExtraOptions(string file, out string[] compiler_options,
return true;
}
+ class DomainTester : MarshalByRefObject
+ {
+ public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
+ {
+ Assembly assembly = Assembly.LoadFile (file);
+
+ bool success = true;
+ Type[] types = assembly.GetTypes ();
+ foreach (Type t in types) {
+
+ // Skip interfaces
+ if (!t.IsClass && !t.IsValueType)
+ continue;
+
+ if (test.VerificationProvider == null) {
+ if (!checker.UpdateVerificationDataFile)
+ checker.LogFileLine (test.FileName, "Missing IL verification data");
+ test.CreateNewTest ();
+ }
+
+ foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
+ MethodBase mi = m as MethodBase;
+ if (mi == null)
+ continue;
+
+ if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
+ continue;
+
+ success &= CompareIL (mi, test, checker);
+ }
+ }
+
+ return success;
+ }
+
+ bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
+ {
+ string m_name = mi.ToString ();
+ string decl_type = mi.DeclaringType.ToString ();
+ PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
+
+ PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
+ if (md == null) {
+ data_provider.AddNewMethod (mi, GetILSize (mi));
+ if (!data_provider.IsNewSet) {
+ checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
+ return false;
+ }
+
+ return true;
+ }
+
+ if (md.Checked) {
+ checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
+ return false;
+ }
+
+ md.Checked = true;
+
+ int il_size = GetILSize (mi);
+ if (md.ILSize == il_size)
+ return true;
+
+ if (md.ILSize > il_size) {
+ checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size);
+ md.ILSize = il_size;
+ return true;
+ }
+
+ checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
+ string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
+
+ md.ILSize = il_size;
+
+ return false;
+ }
+
+ static int GetILSize (MethodBase mi)
+ {
+#if NET_2_0
+ MethodBody body = mi.GetMethodBody ();
+ if (body != null)
+ return body.GetILAsByteArray ().Length;
+#endif
+ return 0;
+ }
+
+ bool ExecuteFile (MethodInfo entry_point, string filename)
+ {
+ TextWriter stdout = Console.Out;
+ TextWriter stderr = Console.Error;
+ Console.SetOut (TextWriter.Null);
+ Console.SetError (TextWriter.Null);
+ ParameterInfo[] pi = entry_point.GetParameters ();
+ object[] args = pi.Length == 0 ? null : default_args;
+
+ object result = null;
+ try {
+ try {
+ result = entry_point.Invoke (null, args);
+ } finally {
+ Console.SetOut (stdout);
+ Console.SetError (stderr);
+ }
+ } catch (Exception e) {
+ throw new ApplicationException (e.ToString ());
+ }
+
+ if (result is int && (int) result != 0)
+ throw new ApplicationException ("Wrong return code: " + result.ToString ());
+
+ return true;
+ }
+
+ public bool Test (string file)
+ {
+ Assembly assembly = Assembly.LoadFile (file);
+ return ExecuteFile (assembly.EntryPoint, file);
+ }
+ }
+
protected override bool Check(TestCase test)
{
string filename = test.FileName;
return true;
}
- MethodInfo mi = null;
string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
// Enable .dll only tests (no execution required)
return true;
}
- Assembly assembly = null;
- try {
- assembly = Assembly.LoadFile (file);
- mi = assembly.EntryPoint;
- }
- catch (FileNotFoundException) {
- if (File.Exists (file)) {
- Console.WriteLine ("APPDOMAIN LIMIT REACHED");
- }
- }
- catch (Exception e) {
- HandleFailure (filename, TestResult.LoadError, e.ToString ());
- return false;
+ AppDomain domain = null;
+#if !NET_2_1
+ if (safe_execution) {
+ // Create a new AppDomain, with the current directory as the base.
+ AppDomainSetup setupInfo = new AppDomainSetup ();
+ setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
+ setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
+ domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
}
-
- if (!ExecuteFile (mi, file, filename))
- return false;
-
- if (doc_output != null) {
- string ref_file = filename.Replace (".cs", "-ref.xml");
+#endif
+ try {
+ DomainTester tester;
try {
#if !NET_2_1
- XmlComparer.Compare (ref_file, doc_output);
+ if (domain != null)
+ tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
+ else
#endif
- }
- catch (Exception e) {
- HandleFailure (filename, TestResult.XmlError, e.Message);
+ tester = new DomainTester ();
+
+ if (!tester.Test (file))
+ return false;
+
+ } catch (ApplicationException e) {
+ HandleFailure (filename, TestResult.ExecError, e.Message);
+ return false;
+ } catch (Exception e) {
+ HandleFailure (filename, TestResult.LoadError, e.ToString ());
return false;
}
- } else {
- if (verif_file != null) {
- PositiveTestCase pt = (PositiveTestCase) test;
- pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data [filename];
- if (!CheckILSize (assembly, pt))
+
+ if (doc_output != null) {
+ string ref_file = filename.Replace (".cs", "-ref.xml");
+ try {
+#if !NET_2_1
+ XmlComparer.Compare (ref_file, doc_output);
+#endif
+ } catch (Exception e) {
+ HandleFailure (filename, TestResult.XmlError, e.Message);
return false;
+ }
+ } else {
+ if (verif_file != null) {
+ PositiveTestCase pt = (PositiveTestCase) test;
+ pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
+
+ if (!tester.CheckILSize (pt, this, file))
+ return false;
+ }
}
+ } finally {
+ if (domain != null)
+ AppDomain.Unload (domain);
}
HandleFailure (filename, TestResult.Success, null);
return new PositiveTestCase (filename, options, deps);
}
- bool CheckILSize (Assembly assembly, PositiveTestCase test)
- {
- bool success = true;
- Type[] types = assembly.GetTypes ();
- foreach (Type t in types) {
- if (!t.IsClass && t.IsValueType)
- continue;
-
- foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
- MethodBase mi = m as MethodBase;
- if (mi == null)
- continue;
-
- if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
- continue;
-
- success &= test.CompareIL (mi, this);
- }
- }
-
- return success;
- }
-
-#if !NET_2_1
- int ExecFile (string exe_name, string filename)
- {
- if (mono == null)
- pi.FileName = exe_name;
- else
- pi.Arguments = exe_name;
-
- Process p = Process.Start (pi);
- p.WaitForExit ();
- return p.ExitCode;
- }
-#endif
-
- bool ExecuteFile (MethodInfo entry_point, string exe_name, string filename)
- {
- TextWriter stdout = Console.Out;
- TextWriter stderr = Console.Error;
- Console.SetOut (TextWriter.Null);
- Console.SetError (TextWriter.Null);
- ParameterInfo[] pi = entry_point.GetParameters ();
- object[] args = pi.Length == 0 ? null : default_args;
-
- object result = null;
- try {
- try {
- result = entry_point.Invoke (null, args);
- } finally {
- Console.SetOut (stdout);
- Console.SetError (stderr);
- }
- }
- catch (Exception e) {
-#if !NET_2_1
- int exit_code = ExecFile (exe_name, filename);
- if (exit_code == 0) {
- LogLine ("(appdomain method failed, external executable succeeded)");
- LogLine (e.ToString ());
- return true;
- }
-#endif
- HandleFailure (filename, TestResult.ExecError, e.ToString ());
- return false;
- }
-
- if (result is int && (int)result != 0) {
- HandleFailure (filename, TestResult.ExecError, "Wrong return code: " + result.ToString ());
- return false;
- }
- return true;
- }
-
public void HandleFailure (string file, TestResult status, string extra)
{
switch (status) {
break;
case TestResult.ILError:
- LogFileLine (file, "IL REGRESSION: " + extra);
+ if (!update_verif_file) {
+ LogFileLine (file, "IL REGRESSION: " + extra);
+ }
extra = null;
break;
}
if (extra != null)
LogLine ("{0}", extra);
- regression.Add (file);
+ if (!regression.Contains (file))
+ regression.Add (file);
}
public override void Initialize ()
#if NET_2_0
void LoadVerificationData (string file)
{
- LogLine ("Loading verification data {0} ...", file);
+ LogLine ("Loading verification data from `{0}' ...", file);
using (XmlReader r = XmlReader.Create (file)) {
r.ReadStartElement ("tests");
void UpdateVerificationData (string file)
{
- LogLine ("Updating verification data {0}...", file);
+ LogLine ("Updating verification data `{0}' ...", file);
XmlWriterSettings s = new XmlWriterSettings ();
s.Indent = true;
- using (XmlWriter w = XmlWriter.Create (file, s)) {
+ using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
w.WriteStartDocument ();
w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
w.WriteStartElement ("tests");
is_warning = false;
if (compiler_options != null) {
foreach (string s in compiler_options) {
- if (s.EndsWith ("warnaserror"))
+ if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
is_warning = true;
}
}
ArrayList ld = new ArrayList ();
CompilerError result = CompilerError.Missing;
while (line != null) {
- if (ld.Contains (line)) {
+ if (ld.Contains (line) && result == CompilerError.Expected) {
if (line.IndexOf ("Location of the symbol related to previous") == -1)
return CompilerError.Duplicate;
}
if (check_msg) {
int first = line.IndexOf (':');
int second = line.IndexOf (':', first + 1);
- if (second == -1 || !check_error_line)
+ if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
+ if (check_error_line) {
+ second = line.IndexOf (':', second + 1);
+ }
+ } else if (second == -1 || !check_error_line) {
second = first;
+ }
string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
}
Checker checker;
+ bool positive;
switch (mode) {
case "neg":
checker = new NegativeChecker (tester, true);
+ positive = false;
break;
case "pos":
string iltest;
GetOption ("il", args, false, out iltest);
checker = new PositiveChecker (tester, iltest);
+ positive = true;
if (iltest != null && GetOption ("update-il", args, false, out temp)) {
((PositiveChecker) checker).UpdateVerificationDataFile = true;
checker.LogFile = temp;
if (GetOption ("verbose", args, false, out temp))
checker.Verbose = true;
-
+ if (GetOption ("safe-execution", args, false, out temp))
+ checker.SafeExecution = true;
+ if (GetOption ("compiler-options", args, true, out temp)) {
+ string[] extra = temp.Split (' ');
+ checker.ExtraCompilerOptions = extra;
+ }
string test_pattern;
if (!GetOption ("files", args, true, out test_pattern)) {
return 1;
}
- string [] files = Directory.GetFiles (".", test_pattern);
- if (files.Length == 0) {
+ var files = new List<string> ();
+ switch (test_pattern) {
+ case "v1":
+ files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
+ break;
+ case "v2":
+ files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
+ goto case "v1";
+ case "v4":
+ files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
+ goto case "v2";
+ default:
+ files.AddRange (Directory.GetFiles (".", test_pattern));
+ break;
+ }
+
+ if (files.Count == 0) {
Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
return 2;
}
checker.Initialize ();
+/*
+ files.Sort ((a, b) => {
+ if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
+ if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
+ return -1;
+ } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
+ if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
+ return 1;
+ }
+ return a.CompareTo (b);
+ });
+*/
foreach (string s in files) {
string filename = Path.GetFileName (s);
if (Char.IsUpper (filename, 0)) { // Windows hack
static void Usage ()
{
Console.WriteLine (
- "Mono compiler tester, (C) 2008 Novell, Inc.\n" +
+ "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
"compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
" \n" +
" -compiler:FILE The file which will be used to compiler tests\n" +
+ " -compiler-options:OPTIONS Add global compiler options\n" +
" -il:IL-FILE XML file with expected IL details for each test\n" +
" -issues:FILE The list of expected failures\n" +
" -log:FILE Writes any output also to the file\n" +
" -help Lists all options\n" +
" -mode:[pos|neg] Specifies compiler test mode\n" +
+ " -safe-execution Runs compiled executables in separate app-domain\n" +
" -update-il Updates IL-FILE to match compiler output\n" +
" -verbose Prints more details during testing\n"
);
projects / mono.git / blobdiff