This directory contains XML W3C Conformance Test Suite. See http://www.w3.org/XML/Test/ for more details
+
+1. Legend of the output.
+
+symbol result description
+---------------------
+. pass test passed
+, pass test failsed on dot net, but passed on mono
+! pass test was fixed (was known as failing, now passed)
+K fail known as not working on dot net
+k fail known as not working on mono
+f fail listed in fixme.lst
+E fail ERROR: NEW REGRESSION(s)
+
+In case of E the xmlconf.exe exits with exit code 1 to signal regression(s).
+
+2. Special files.
+
+filename in/out description
+---------------------
+ignore.lst in tests to be ignored (please, add to README the reason for ignoring them)
+slow.lst in tests known to be slow (will be ignored if -s is not given)
+new-slow.lst out tests that took more than 1 sec to perform - consider adding them to slow.lst
+net-failed.lst in tests that known to fail on ms.net
+knownFailures.lst in known failures of mono
+fixme.lst in similar to knownFailures, but is not stored in svn (user-private list)
+failed.lst out new regressions list.
+fixed.lst out tests that should be deleted from knownFailures or fixme.lst
+
+3. Exit code
+
+0 Success - no new regressions
+1 new regressions
+2 invalid command line options
+
public override string ToString() {\r
return _short.ToString();\r
}\r
+\r
+ public string Long {\r
+ get {\r
+ return _long;\r
+ }\r
+ }\r
}\r
\r
static void PrintUsage () {\r
ArrayList slowTests = new ArrayList ();\r
ArrayList igroredTests = new ArrayList ();\r
ArrayList knownFailures = new ArrayList ();\r
+ ArrayList fixmeList = new ArrayList ();\r
+ ArrayList netFailures = new ArrayList ();\r
StreamWriter failedList;\r
StreamWriter fixedList;\r
+ StreamWriter slowNewList;\r
#endregion\r
\r
#region command line option fields\r
_args = args;\r
failedList = new StreamWriter ("failed.lst", false);\r
fixedList = new StreamWriter ("fixed.lst", false);\r
+ slowNewList = new StreamWriter ("slow-new.lst", false);\r
ReadStrings (slowTests, "slow.lst");\r
ReadStrings (igroredTests, "ignored.lst");\r
ReadStrings (knownFailures, "knownFailures.lst");\r
+ ReadStrings (fixmeList, "fixme.lst");\r
+ ReadStrings (netFailures, "net-failed.lst");\r
}\r
\r
bool Run ()\r
return false;\r
\r
XmlDocument catalog = new XmlDocument ();\r
- catalog.Load (@"xmlconf\xmlconf.xml");\r
+ catalog.Load ("xmlconf/xmlconf.xml");\r
\r
foreach (XmlElement test in catalog.SelectNodes ("//TEST")) {\r
++totalCount;\r
}\r
\r
DateTime start = DateTime.Now;\r
- res &= PerformTest (test); \r
+ res = PerformTest (test) && res;\r
TimeSpan span = DateTime.Now - start;\r
if (span.TotalSeconds > 1) {\r
if (slowTests.Contains (testId))\r
continue;\r
- using (StreamWriter wr = new StreamWriter("slow.lst", true))\r
- wr.WriteLine (testId);\r
+ slowNewList.WriteLine (testId);\r
+ slowNewList.Flush ();\r
}\r
}\r
\r
- Console.Error.WriteLine ("\n*********");\r
+ Console.Error.WriteLine ("\n\n*********");\r
Console.Error.WriteLine ("Total:{0}", totalCount);\r
Console.Error.WriteLine ("Performed:{0}", performedCount);\r
Console.Error.WriteLine ("Passed:{0}", passedCount);\r
Console.Error.WriteLine ("Failed:{0}", failedCount);\r
Console.Error.WriteLine ("Regressions:{0}", regressionsCount);\r
- Console.Error.WriteLine ("Fixed:{0}", fixedCount);\r
+ Console.Error.WriteLine ("Fixed:{0}\n", fixedCount);\r
+\r
+ if (fixedCount > 0)\r
+ Console.Error.WriteLine (@"ATTENTION!\r
+You must delete the fixed tests (those listed in fixed.lst) from\r
+knownFailures.lst or fixme.lst. If you don't do it, you can miss\r
+regressions in the future.");\r
+\r
+ if (regressionsCount > 0)\r
+ Console.Error.WriteLine (@"ERROR!!! New regressions!\r
+If you see this message for the first time, your last changes had\r
+introduced new bugs! Before you commit, you must do one of the following:\r
+\r
+1. Find and fix the bugs, so tests will pass again.\r
+2. Open new bugs in bugzilla and temporily add the tests to fixme.lst\r
+3. Write to devlist and confirm adding the new tests to knownFailures.lst");\r
\r
return res;\r
}\r
validatingPassed = false;\r
}\r
bool res = isOK (type, nonValidatingPassed, validatingPassed);\r
- Report (test, res, nonValidatingPassed, validatingPassed);\r
- return res;\r
+ \r
+ return Report (test, res, nonValidatingPassed, validatingPassed);\r
}\r
\r
bool isOK (string type, bool nonValidatingPassed, bool validatingPassed)\r
}\r
}\r
\r
- void Report (XmlElement test, bool isok, bool nonValidatingPassed, bool validatingPassed)\r
+ bool Report (XmlElement test, bool isok, bool nonValidatingPassed, bool validatingPassed)\r
{\r
string testId = test.GetAttribute ("ID");\r
\r
if (isok) {\r
++passedCount;\r
- if (knownFailures.Contains (testId)) {\r
+ if (fixmeList.Contains (testId) || knownFailures.Contains (testId)) {\r
++fixedCount;\r
fixedList.WriteLine (testId);\r
+ Console.Error.Write ("!");\r
+ return true;\r
+ }\r
+ if (netFailures.Contains (testId)) {\r
+ Console.Error.Write (",");\r
+ return true;\r
}\r
\r
Console.Error.Write (".");\r
- return;\r
+ return true;\r
}\r
\r
++failedCount;\r
\r
+ if (netFailures.Contains (testId)) {\r
+ Console.Error.Write ("K");\r
+ return true;\r
+ }\r
if (knownFailures.Contains (testId)) {\r
Console.Error.Write ("k");\r
- return;\r
+ return true;\r
+ }\r
+ if (fixmeList.Contains (testId)) {\r
+ Console.Error.Write ("f");\r
+ return true;\r
}\r
\r
++regressionsCount;\r
testId, test.GetAttribute ("TYPE"), nonValidatingPassed, validatingPassed);\r
failedList.WriteLine (test.InnerXml);\r
failedList.Flush ();\r
+ return false;\r
}\r
}\r
}\r
projects / mono.git / commitdiff