-using System;\r
-using System.Xml;\r
-using System.IO;\r
-using System.Collections;\r
-using System.ComponentModel;\r
-using System.Reflection;\r
-using System.Text;\r
-\r
-namespace MonoTests.stand_alone {\r
-#if NUNIT_SUPPORT
- using NUnit.Core;\r
+using System;
+using System.Xml;
+using System.IO;
+using System.Collections;
+using System.Text;
+
+namespace MonoTests.W3C_xmlconf {
+ using NUnit.Core;
using NUnit.Framework;\r
-#endif
-\r
- class AllTests: IDisposable {
- \r
- #region Command Line Options Handling\r
-\r
- class CommandLineOptionAttribute:Attribute{\r
- char _short;\r
- string _long; //FIXME: use long form, too\r
- public CommandLineOptionAttribute (char a_short, string a_long):base() {\r
- _short = a_short;\r
- _long = a_long;\r
- }\r
- \r
- public CommandLineOptionAttribute (char a_short):this (a_short, null) {\r
- }\r
-\r
- public override string ToString() {\r
- return _short.ToString();\r
- }\r
-\r
- public string Long {\r
- get {\r
- return _long;\r
- }\r
- }\r
- }\r
-\r
- static void PrintUsage () {\r
- Console.Error.WriteLine("Usage: xmlconf <flags>");\r
- Console.Error.WriteLine("\tFlags:");\r
- foreach (DictionaryEntry de in AllTests.GetOptions())\r
- Console.Error.WriteLine ("\t{0}\t{1}", de.Key, de.Value);\r
- }\r
-\r
- public static Hashtable GetOptions() {\r
- Hashtable h = new Hashtable();\r
\r
- foreach (FieldInfo i in typeof (AllTests).GetFields()) {\r
- //FIXME: handle long options, too\r
- string option = "-" + i.GetCustomAttributes(typeof(CommandLineOptionAttribute),\r
- true)[0].ToString();\r
- string descr = (i.GetCustomAttributes(typeof(DescriptionAttribute),\r
- true)[0] as DescriptionAttribute).Description;\r
- h[option] = descr;\r
- }\r
- return h;\r
- }\r
-\r
- public bool ParseOptions () {\r
- if (_args == null || _args.Length == 0)\r
- return true;\r
- if(_args[0].Length < 2 || _args[0][0] != '-') {\r
- PrintUsage();\r
- return false;\r
- }\r
- string options = _args[0].Substring(1); //FIXME: handle long options\r
- foreach (FieldInfo i in typeof (AllTests).GetFields (BindingFlags.NonPublic\r
- | BindingFlags.Instance)) {\r
- //FIXME: report if unknown options were passed\r
- object [] attrs = i.GetCustomAttributes(typeof(CommandLineOptionAttribute),true);\r
- if (attrs.Length == 0)\r
- continue;\r
- string option = attrs[0].ToString();\r
- if (options.IndexOf(option) == -1)\r
- continue;\r
- i.SetValue (this, true);\r
- }\r
- return true;\r
- }\r
- #endregion\r
-\r
- string [] _args;\r
-\r
- #region statistics fields\r
- int totalCount = 0;\r
- int performedCount = 0;\r
- int passedCount = 0;\r
- int failedCount = 0;\r
- int regressionsCount = 0; //failures not listed in knownFailures.lst\r
- int fixedCount = 0; //tested known to fail that passed\r
- #endregion\r
-\r
- #region test list fields\r
- ArrayList slowTests = new ArrayList ();\r
- ArrayList igroredTests = new ArrayList ();\r
- ArrayList knownFailures = new ArrayList ();\r
- ArrayList fixmeList = new ArrayList ();\r
- ArrayList netFailures = new ArrayList ();\r
- StreamWriter failedListWriter;\r
- StreamWriter fixedListWriter;\r
- StreamWriter slowNewListWriter;\r
- StreamWriter totalListWriter;
- #endregion\r
+ public abstract class BaseTests
+ {
+ TestSuite _suite;\r
- #region IDisposable Members
- public void Dispose()
- {
- if (failedListWriter != null)
- failedListWriter.Close ();
- if (fixedListWriter != null)
- fixedListWriter.Close ();
- if (slowNewListWriter != null)
- slowNewListWriter.Close ();
- if (totalListWriter != null)
- totalListWriter.Close ();
- failedListWriter = null;
- fixedListWriter = null;
- slowNewListWriter = null;
- totalListWriter = null;
- }
+ #region test list fields
+ protected readonly ArrayList ignoredTests = new ArrayList ();
+ protected readonly ArrayList knownFailures = new ArrayList ();
+ protected readonly ArrayList fixmeList = new ArrayList ();
+ protected readonly ArrayList netFailures = new ArrayList ();
#endregion
-\r
- #region command line option fields\r
- [CommandLineOption ('s')]\r
- [Description ("do run slow tests (skipped by default)")]\r
- bool runSlow = false;\r
-\r
- [CommandLineOption ('i')]\r
- [Description ("do run tests being ignored by default")]\r
- bool runIgnored = false;\r
- #endregion\r
-\r
- static int Main (string[] args)\r
- {\r
- using (AllTests tests = new AllTests (args)) {
- if (!tests.Run ())
- return 1;
- else
- return 0;
- }
- }\r
-\r
-#if NUNIT_SUPPORT
- TestSuite _suite;\r
-\r
- [Suite]\r
- static public TestSuite Suite {\r
- get {\r
- TestSuite suite = new TestSuite ("W3C_xmlconf");\r
- using (AllTests tests = new AllTests (suite)) {\r
- tests.Run ();\r
- }\r
- return suite;\r
- }\r
- }\r
-\r
- AllTests (TestSuite suite)\r
- : this () {\r
- _suite = suite;\r
- }\r
-#endif
-\r
- AllTests (string [] args)\r
- : this () {\r
- _args = args;\r
- }\r
-\r
- #region ReadStrings ()\r
- static void ReadStrings (ArrayList array, string filename)\r
- {\r
- if (!File.Exists (filename))\r
- return;\r
-\r
- using (StreamReader reader = new StreamReader (filename)) {\r
+
+ #region ReadStrings ()
+ static void ReadStrings (ArrayList array, string filename) {
+ if (!File.Exists (filename))
+ return;
+
+ using (StreamReader reader = new StreamReader (filename)) {
foreach (string s_ in reader.ReadToEnd ().Split ("\n".ToCharArray ())) {
string s = s_.Trim ();
if (s.Length > 0)
array.Add (s);
- }\r
- }\r
- }\r
- #endregion\r
-\r
- private AllTests ()\r
- {\r
- failedListWriter = new StreamWriter ("failed.lst", false);
- fixedListWriter = new StreamWriter ("fixed.lst", false);
- slowNewListWriter = new StreamWriter ("slow-new.lst", false);
- totalListWriter = new StreamWriter ("total.lst", false);
- ReadStrings (slowTests, "slow.lst");\r
- ReadStrings (igroredTests, "ignored.lst");\r
- ReadStrings (knownFailures, "knownFailures.lst");\r
- ReadStrings (fixmeList, "fixme.lst");\r
- ReadStrings (netFailures, "net-failed.lst");\r
- }\r
-\r
- bool Run ()\r
- {\r
- bool res = true;\r
- if (!ParseOptions ())\r
- return false;\r
-\r
- XmlDocument catalog = new XmlDocument ();\r
- catalog.Load ("xmlconf/xmlconf.xml");\r
- \r
- foreach (XmlElement test in catalog.SelectNodes ("//TEST")) {\r
- ++totalCount;\r
-\r
- string testId = test.GetAttribute ("ID");\r
- \r
- if (!runSlow && slowTests.Contains (testId)) {\r
- continue;\r
- }\r
-\r
- if (!runIgnored && igroredTests.Contains (testId)) {\r
- continue;\r
- }\r
-\r
- DateTime start = DateTime.Now;\r
- if (!PerformTest (test))\r
- res = false;\r
- TimeSpan span = DateTime.Now - start;\r
- if (span.TotalSeconds > 1) {\r
- if (slowTests.Contains (testId))\r
- continue;\r
- slowNewListWriter.WriteLine (testId);\r
- }\r
- }\r
-\r
- Console.Error.WriteLine ("\n\n*********");\r
- Console.Error.WriteLine ("Total:{0}", totalCount);\r
- Console.Error.WriteLine ("Performed:{0}", performedCount);\r
- Console.Error.WriteLine ("Passed:{0}", passedCount);\r
- Console.Error.WriteLine ("Failed:{0}", failedCount);\r
- Console.Error.WriteLine ("Regressions:{0}", regressionsCount);\r
- Console.Error.WriteLine ("Fixed:{0}\n", fixedCount);\r
-\r
- if (fixedCount > 0)\r
- Console.Error.WriteLine (@"\r
-\r
-ATTENTION!\r
-Delete the fixed tests (those listed in fixed.lst) from\r
-knownFailures.lst or fixme.lst, or we might miss\r
-regressions in the future.");\r
-\r
- if (regressionsCount > 0)\r
- Console.Error.WriteLine (@"\r
-\r
-ERROR!!! New regressions!\r
-If you see this message for the first time, your last changes had\r
-introduced new bugs! Before you commit, consider one of the following:\r
-\r
-1. Find and fix the bugs, so tests will pass again.\r
-2. Open new bugs in bugzilla and temporily add the tests to fixme.lst\r
-3. Write to devlist and confirm adding the new tests to knownFailures.lst");\r
-\r
- return res;\r
- }\r
-\r
- bool PerformTest (XmlElement test)\r
- {\r
- ++performedCount;\r
-\r
- string type = test.GetAttribute ("TYPE");\r
- if (type == "error")\r
- return true; //save time\r
-\r
- Uri baseUri = new Uri (test.BaseURI);\r
- Uri testUri = new Uri (baseUri, test.GetAttribute ("URI"));\r
-\r
- bool validatingPassed;\r
- bool nonValidatingPassed;\r
- try {\r
- XmlTextReader trd = new XmlTextReader (testUri.ToString ());\r
- new XmlDocument ().Load (trd);\r
- nonValidatingPassed = true;\r
- }\r
- catch (Exception) {\r
- nonValidatingPassed = false;\r
- }\r
-\r
- try {\r
- XmlTextReader rd = new XmlTextReader (testUri.ToString ());\r
- XmlValidatingReader vrd = new XmlValidatingReader (rd);\r
- new XmlDocument ().Load (vrd);\r
- validatingPassed = true;\r
- }\r
- catch (Exception) {\r
- validatingPassed = false;\r
- }\r
- bool res = isOK (type, nonValidatingPassed, validatingPassed);\r
- \r
- return Report (testUri, test, res, nonValidatingPassed, validatingPassed);\r
- }\r
-\r
- bool isOK (string type, bool nonValidatingPassed, bool validatingPassed)\r
- {\r
- switch (type) {\r
- case "valid":\r
- return nonValidatingPassed && validatingPassed;\r
- case "invalid":\r
- return nonValidatingPassed && !validatingPassed;\r
- case "not-wf":\r
- return !nonValidatingPassed && !validatingPassed;\r
- case "error":\r
- return true; //readers can optionally accept or reject errors\r
- default:\r
- throw new ArgumentException ("Bad test type", "type");\r
- }\r
- }\r
-\r
- bool Report (Uri testUri, XmlElement test, bool isok, bool nonValidatingPassed, bool validatingPassed)\r
- {\r
- string testId = test.GetAttribute ("ID");\r
-\r
-#if NUNIT_SUPPORT
- if (_suite != null) {\r
- StringBuilder sb = new StringBuilder();\r
- sb.Append (testUri.ToString ());\r
- sb.Append (test.InnerXml);\r
- _suite.Add (new PredefinedTest (testId, isok, sb.ToString ()));\r
- return true;\r
- }\r
-#endif
-\r
- totalListWriter.Write (testUri.ToString () + "\t");\r
- totalListWriter.Write (testId + "\t");
-\r
- if (isok) {\r
- ++passedCount;\r
- if (fixmeList.Contains (testId) || knownFailures.Contains (testId)) {\r
- ++fixedCount;\r
- fixedListWriter.WriteLine (testId);\r
- Console.Error.Write ("!");\r
- totalListWriter.WriteLine ("fixed");
- return true;\r
- }\r
- if (netFailures.Contains (testId)) {\r
- Console.Error.Write (",");\r
- totalListWriter.WriteLine (",");
- return true;\r
- }\r
-\r
- Console.Error.Write (".");\r
- totalListWriter.WriteLine (".");
- return true;\r
- }\r
-\r
- ++failedCount;\r
-\r
- if (netFailures.Contains (testId)) {\r
- Console.Error.Write ("K");\r
- totalListWriter.WriteLine ("dot net known failure");
- return true;\r
- }\r
- if (knownFailures.Contains (testId)) {\r
- Console.Error.Write ("k");\r
- totalListWriter.WriteLine ("known failure");
- return true;\r
- }\r
- if (fixmeList.Contains (testId)) {\r
- Console.Error.Write ("f");\r
- totalListWriter.WriteLine ("fixme");
- return true;\r
- }\r
-\r
- ++regressionsCount;\r
- Console.Error.Write ("E");\r
- totalListWriter.WriteLine ("regression");
- failedListWriter.Write ("*** Test failed:\t{0}\ttype:{1}\tnonValidatingPassed:{2},validatingPassed:{3}\t",\r
- testId, test.GetAttribute ("TYPE"), nonValidatingPassed, validatingPassed);\r
- failedListWriter.WriteLine (test.InnerXml);\r
- return false;\r
- }\r
+ }
+ reader.Close();
+ }
+ }
+ #endregion
+
+ protected BaseTests (TestSuite suite)
+ :this ()
+ {
+ _suite = suite;
+ }
+
+ private BaseTests ()
+ {
+ ReadStrings (ignoredTests, "ignored.lst");
+ ReadStrings (knownFailures, "knownFailures.lst");
+ ReadStrings (fixmeList, "fixme.lst");
+ ReadStrings (netFailures, "net-failed.lst");
+ }
+
+ protected void BuildSuite ()
+ {
+ XmlDocument catalog = new XmlDocument ();
+ catalog.Load ("xmlconf/xmlconf.xml");
+
+ foreach (XmlElement test in catalog.SelectNodes ("//TEST")) {
+ string testId = test.GetAttribute ("ID");
+
+ ProcessTest (testId, test);
+ }
+ }
+
+ protected virtual bool InverseResult {
+ get {return false;}
+ }
+
+ protected virtual void ProcessTest (string testId, XmlElement test)
+ {
+ if (ignoredTests.Contains (testId))
+ return;
+
+ if (netFailures.Contains (testId))
+ return;
+
+ _suite.Add (new TestFromCatalog (testId, test, InverseResult));
+ }
+ }
+
+ public class AllTests: BaseTests
+ {
+ [Suite]
+ static public TestSuite Suite{
+ get {
+ TestSuite suite = new TestSuite ("W3C_xmlconf.All");
+ AllTests tests = new AllTests (suite);
+ tests.BuildSuite ();
+ return suite;
+ }
+ }
+
+ AllTests (TestSuite suite)
+ : base (suite)
+ {
+ }
}\r
\r
- public class PredefinedTest: NUnit.Core.TestCase {\r
- bool _res;\r
- string _message;\r
- public PredefinedTest (string name, bool res, string message):base (null, name) {\r
- _res = res;\r
- _message = message;\r
- }\r
+ public class CleanTests : BaseTests\r
+ {
+ [Suite]
+ static public TestSuite Suite{
+ get {
+ TestSuite suite = new TestSuite ("W3C_xmlconf.Clean");
+ CleanTests tests = new CleanTests (suite);
+ tests.BuildSuite ();
+ return suite;
+ }
+ }
+
+ CleanTests (TestSuite suite)
+ : base (suite)
+ {
+ }
+
+ protected override void ProcessTest(string testId, XmlElement test)
+ {
+ if (knownFailures.Contains (testId) || fixmeList.Contains (testId))
+ return;
+
+ base.ProcessTest (testId, test);
+ }
+ }\r
\r
- public override void Run (TestCaseResult res) {\r
- if (_res)\r
- res.Success ();\r
- else\r
- res.Failure (_message, null);\r
- }\r
+ public class KnownFailureTests : BaseTests\r
+ {
+ [Suite]
+ static public TestSuite Suite{
+ get {
+ TestSuite suite = new TestSuite ("W3C_xmlconf.KnownFailures");
+ KnownFailureTests tests = new KnownFailureTests (suite);
+ tests.BuildSuite ();
+ return suite;
+ }
+ }
+
+ KnownFailureTests (TestSuite suite)
+ : base (suite)
+ {
+ }
+
+ protected override bool InverseResult {
+ get {return true;}
+ }
+
+ protected override void ProcessTest(string testId, XmlElement test)
+ {
+ if (!knownFailures.Contains (testId) && !fixmeList.Contains (testId))
+ return;
+
+ base.ProcessTest (testId, test);
+ }
}\r
-}\r
+\r
+ public class TestFromCatalog : NUnit.Core.TestCase
+ {
+ XmlElement _test;
+ string _errorString;
+ bool _inverseResult;
+
+ public TestFromCatalog (string testId, XmlElement test, bool inverseResult)
+ :base (null, testId)
+ {
+ _test = test;
+ _inverseResult = inverseResult;
+ }
+
+ bool TestNonValidating (string uri)
+ {
+ XmlTextReader trd = null;
+ try {
+ trd = new XmlTextReader (uri);
+ new XmlDocument ().Load (trd);
+ return true;
+ }
+ catch (Exception e) {
+ _errorString = e.ToString ();
+ return false;
+ }
+ finally {
+ if (trd != null)
+ trd.Close();
+ }
+ }
+
+ bool TestValidating (string uri)
+ {
+ XmlTextReader rd = null;
+ try {
+ rd = new XmlTextReader (uri);
+ XmlValidatingReader vrd = new XmlValidatingReader (rd);
+ new XmlDocument ().Load (vrd);
+ return true;
+ }
+ catch (Exception e) {
+ _errorString = e.ToString (); //rewrites existing, possibly, but it's ok
+ return false;
+ }
+ finally {
+ if (rd != null)
+ rd.Close();
+ }
+ }
+
+ public override void Run (TestCaseResult res)
+ {
+ string type = _test.GetAttribute ("TYPE");
+ if (type == "error")
+ res.Success ();
+
+ Uri baseUri = new Uri (_test.BaseURI);
+ Uri testUri = new Uri (baseUri, _test.GetAttribute ("URI"));
+
+ bool nonValidatingPassed = TestNonValidating (testUri.ToString ());
+ bool validatingPassed = TestValidating (testUri.ToString ());
+
+ bool isok = isOK (type, nonValidatingPassed, validatingPassed);
+ string message="";
+ if (_inverseResult) {
+ isok = !isok;
+ message = "The following test was FIXED:\n";
+ }
+
+ if (isok)
+ res.Success ();
+ else {
+ message += "type:"+type;
+ message += " non-validating passed:"+nonValidatingPassed.ToString();
+ message += " validating passed:"+validatingPassed.ToString();
+ message += " description:"+_test.InnerText;
+ res.Failure (message, _errorString);
+ }
+ }
+
+ static bool isOK (string type, bool nonValidatingPassed, bool validatingPassed)
+ {
+ switch (type) {
+ case "valid":
+ return nonValidatingPassed && validatingPassed;
+ case "invalid":
+ return nonValidatingPassed && !validatingPassed;
+ case "not-wf":
+ return !nonValidatingPassed && !validatingPassed;
+ case "error":
+ return true; //readers can optionally accept or reject errors
+ default:
+ throw new ArgumentException ("Wrong test type", "type");
+ }
+ }
+ }
+}
projects / mono.git / blobdiff