mcs/class/System.XML/Test/System.Xml/W3C/xmlconf.cs

   1 using System;\r
   2 using System.Xml;\r
   3 using System.IO;\r
   4 using System.Collections;\r
   5 using System.ComponentModel;\r
   6 using System.Reflection;\r
   7 \r
   8 namespace XmlConfTest {\r
   9         class XmlConfTest: IDisposable {
  10                 \r
  11                 #region Command Line Options Handling\r
  12 \r
  13                 class CommandLineOptionAttribute:Attribute{\r
  14                         char _short;\r
  15                         string _long; //FIXME: use long form, too\r
  16                         public CommandLineOptionAttribute (char a_short, string a_long):base() {\r
  17                                 _short = a_short;\r
  18                                 _long = a_long;\r
  19                         }\r
  20                         \r
  21                         public CommandLineOptionAttribute (char a_short):this (a_short, null) {\r
  22                         }\r
  23 \r
  24                         public override string ToString() {\r
  25                                 return _short.ToString();\r
  26                         }\r
  27 \r
  28                         public string Long {\r
  29                                 get {\r
  30                                         return _long;\r
  31                                 }\r
  32                         }\r
  33                 }\r
  34 \r
  35                 static void PrintUsage () {\r
  36                         Console.Error.WriteLine("Usage: xmlconf <flags>");\r
  37                         Console.Error.WriteLine("\tFlags:");\r
  38                         foreach (DictionaryEntry de in XmlConfTest.GetOptions())\r
  39                                 Console.Error.WriteLine ("\t{0}\t{1}", de.Key, de.Value);\r
  40                 }\r
  41 \r
  42                 public static Hashtable GetOptions() {\r
  43                         Hashtable h = new Hashtable();\r
  44 \r
  45                         foreach (FieldInfo i in typeof (XmlConfTest).GetFields()) {\r
  46                                 //FIXME: handle long options, too\r
  47                                 string option = "-" + i.GetCustomAttributes(typeof(CommandLineOptionAttribute),\r
  48                                         true)[0].ToString();\r
  49                                 string descr = (i.GetCustomAttributes(typeof(DescriptionAttribute),\r
  50                                         true)[0] as DescriptionAttribute).Description;\r
  51                                 h[option] = descr;\r
  52                         }\r
  53                         return h;\r
  54                 }\r
  55 \r
  56                 public bool ParseOptions () {\r
  57                         if (_args.Length < 1)\r
  58                                 return true;\r
  59                         if(_args[0].Length < 2 || _args[0][0] != '-') {\r
  60                                 PrintUsage();\r
  61                                 return false;\r
  62                         }\r
  63                         string options = _args[0].Substring(1); //FIXME: handle long options\r
  64                         foreach (FieldInfo i in typeof (XmlConfTest).GetFields (BindingFlags.NonPublic\r
  65                                 | BindingFlags.Instance)) {\r
  66                                 //FIXME: report if unknown options were passed\r
  67                                 object [] attrs = i.GetCustomAttributes(typeof(CommandLineOptionAttribute),true);\r
  68                                 if (attrs.Length == 0)\r
  69                                         continue;\r
  70                                 string option = attrs[0].ToString();\r
  71                                 if (options.IndexOf(option) == -1)\r
  72                                         continue;\r
  73                                 i.SetValue (this, true);\r
  74                         }\r
  75                         return true;\r
  76                 }\r
  77                 #endregion\r
  78 \r
  79                 string [] _args;\r
  80 \r
  81                 #region statistics fields\r
  82                 int totalCount = 0;\r
  83                 int performedCount = 0;\r
  84                 int passedCount = 0;\r
  85                 int failedCount = 0;\r
  86                 int regressionsCount = 0; //failures not listed in knownFailures.lst\r
  87                 int fixedCount = 0; //tested known to fail that passed\r
  88                 #endregion\r
  89 \r
  90                 #region test list fields\r
  91                 ArrayList slowTests = new ArrayList ();\r
  92                 ArrayList igroredTests = new ArrayList ();\r
  93                 ArrayList knownFailures = new ArrayList ();\r
  94                 ArrayList fixmeList = new ArrayList ();\r
  95                 ArrayList netFailures = new ArrayList ();\r
  96                 StreamWriter failedListWriter;\r
  97                 StreamWriter fixedListWriter;\r
  98                 StreamWriter slowNewListWriter;\r
  99                 StreamWriter totalListWriter;
 100                 #endregion\r
 101
 102                 #region IDisposable Members
 103                 public void Dispose()
 104                 {
 105                         if (failedListWriter != null)
 106                                 failedListWriter.Close ();
 107                         if (fixedListWriter != null)
 108                                 fixedListWriter.Close ();
 109                         if (slowNewListWriter != null)
 110                                 slowNewListWriter.Close ();
 111                         if (totalListWriter != null)
 112                                 totalListWriter.Close ();
 113                         failedListWriter = null;
 114                         fixedListWriter = null;
 115                         slowNewListWriter = null;
 116                         totalListWriter = null;
 117                 }
 118                 #endregion
 119 \r
 120                 #region command line option fields\r
 121                 [CommandLineOption ('s')]\r
 122                 [Description ("do run slow tests (skipped by default)")]\r
 123                 bool runSlow = false;\r
 124 \r
 125                 [CommandLineOption ('i')]\r
 126                 [Description ("do run tests being ignored by default")]\r
 127                 bool runIgnored = false;\r
 128                 #endregion\r
 129 \r
 130                 static int Main (string[] args)\r
 131                 {\r
 132                         using (XmlConfTest test = new XmlConfTest (args)) {
 133                                 if (!test.Run ())
 134                                         return 1;
 135                                 else
 136                                         return 0;
 137                         }
 138                 }\r
 139 \r
 140                 #region ReadStrings ()\r
 141                 static void ReadStrings (ArrayList array, string filename)\r
 142                 {\r
 143                         if (!File.Exists (filename))\r
 144                                 return;\r
 145 \r
 146                         using (StreamReader reader = new StreamReader (filename)) {\r
 147                                 foreach (string s_ in reader.ReadToEnd ().Split ("\n".ToCharArray ())) {
 148                                         string s = s_.Trim ();
 149                                         if (s.Length > 0)
 150                                                 array.Add (s);
 151                                 }\r
 152                         }\r
 153                 }\r
 154                 #endregion\r
 155 \r
 156                 XmlConfTest (string [] args)\r
 157                 {\r
 158                         _args = args;\r
 159                         failedListWriter = new StreamWriter ("failed.lst", false);
 160                         fixedListWriter = new StreamWriter ("fixed.lst", false);
 161                         slowNewListWriter = new StreamWriter ("slow-new.lst", false);
 162                         totalListWriter = new StreamWriter ("total.lst", false);
 163                         ReadStrings (slowTests, "slow.lst");\r
 164                         ReadStrings (igroredTests, "ignored.lst");\r
 165                         ReadStrings (knownFailures, "knownFailures.lst");\r
 166                         ReadStrings (fixmeList, "fixme.lst");\r
 167                         ReadStrings (netFailures, "net-failed.lst");\r
 168                 }\r
 169 \r
 170                 bool Run ()\r
 171                 {\r
 172                         bool res = true;\r
 173                         if (!ParseOptions ())\r
 174                                 return false;\r
 175 \r
 176                         XmlDocument catalog = new XmlDocument ();\r
 177                         catalog.Load ("xmlconf/xmlconf.xml");\r
 178                         \r
 179                         foreach (XmlElement test in catalog.SelectNodes ("//TEST")) {\r
 180                                 ++totalCount;\r
 181 \r
 182                                 string testId = test.GetAttribute ("ID");\r
 183                                 \r
 184                                 if (!runSlow && slowTests.Contains (testId)) {\r
 185                                         continue;\r
 186                                 }\r
 187 \r
 188                                 if (!runIgnored && igroredTests.Contains (testId)) {\r
 189                                         continue;\r
 190                                 }\r
 191 \r
 192                                 DateTime start = DateTime.Now;\r
 193                                 if (!PerformTest (test))\r
 194                                         res = false;\r
 195                                 TimeSpan span = DateTime.Now - start;\r
 196                                 if (span.TotalSeconds > 1) {\r
 197                                         if (slowTests.Contains (testId))\r
 198                                                 continue;\r
 199                                         slowNewListWriter.WriteLine (testId);\r
 200                                 }\r
 201                         }\r
 202 \r
 203                         Console.Error.WriteLine ("\n\n*********");\r
 204                         Console.Error.WriteLine ("Total:{0}", totalCount);\r
 205                         Console.Error.WriteLine ("Performed:{0}", performedCount);\r
 206                         Console.Error.WriteLine ("Passed:{0}", passedCount);\r
 207                         Console.Error.WriteLine ("Failed:{0}", failedCount);\r
 208                         Console.Error.WriteLine ("Regressions:{0}", regressionsCount);\r
 209                         Console.Error.WriteLine ("Fixed:{0}\n", fixedCount);\r
 210 \r
 211                         if (fixedCount > 0)\r
 212                                 Console.Error.WriteLine (@"\r
 213 \r
 214 ATTENTION!\r
 215 Delete the fixed tests (those listed in fixed.lst) from\r
 216 knownFailures.lst or fixme.lst, or we might miss\r
 217 regressions in the future.");\r
 218 \r
 219                         if (regressionsCount > 0)\r
 220                                 Console.Error.WriteLine (@"\r
 221 \r
 222 ERROR!!! New regressions!\r
 223 If you see this message for the first time, your last changes had\r
 224 introduced new bugs! Before you commit, consider one of the following:\r
 225 \r
 226 1. Find and fix the bugs, so tests will pass again.\r
 227 2. Open new bugs in bugzilla and temporily add the tests to fixme.lst\r
 228 3. Write to devlist and confirm adding the new tests to knownFailures.lst");\r
 229 \r
 230                         return res;\r
 231                 }\r
 232 \r
 233                 bool PerformTest (XmlElement test)\r
 234                 {\r
 235                         ++performedCount;\r
 236 \r
 237                         string type = test.GetAttribute ("TYPE");\r
 238                         if (type == "error")\r
 239                                 return true; //save time\r
 240 \r
 241                         Uri baseUri = new Uri (test.BaseURI);\r
 242                         Uri testUri = new Uri (baseUri, test.GetAttribute ("URI"));\r
 243                         bool validatingPassed;\r
 244                         bool nonValidatingPassed;\r
 245                         try {\r
 246                                 XmlTextReader trd = new XmlTextReader (testUri.ToString ());\r
 247                                 new XmlDocument ().Load (trd);\r
 248                                 nonValidatingPassed = true;\r
 249                         }\r
 250                         catch (Exception) {\r
 251                                 nonValidatingPassed = false;\r
 252                         }\r
 253 \r
 254                         try {\r
 255                                 XmlTextReader rd = new XmlTextReader (testUri.ToString ());\r
 256                                 XmlValidatingReader vrd = new XmlValidatingReader (rd);\r
 257                                 new XmlDocument ().Load (vrd);\r
 258                                 validatingPassed = true;\r
 259                         }\r
 260                         catch (Exception) {\r
 261                                 validatingPassed = false;\r
 262                         }\r
 263                         bool res = isOK (type, nonValidatingPassed, validatingPassed);\r
 264                         \r
 265                         return Report (test, res, nonValidatingPassed, validatingPassed);\r
 266                 }\r
 267 \r
 268                 bool isOK (string type, bool nonValidatingPassed, bool validatingPassed)\r
 269                 {\r
 270                         switch (type) {\r
 271                         case "valid":\r
 272                                 return nonValidatingPassed && validatingPassed;\r
 273                         case "invalid":\r
 274                                 return nonValidatingPassed && !validatingPassed;\r
 275                         case "not-wf":\r
 276                                 return !nonValidatingPassed && !validatingPassed;\r
 277                         case "error":\r
 278                                 return true; //readers can optionally accept or reject errors\r
 279                         default:\r
 280                                 throw new ArgumentException ("Bad test type", "type");\r
 281                         }\r
 282                 }\r
 283 \r
 284                 bool Report (XmlElement test, bool isok, bool nonValidatingPassed, bool validatingPassed)\r
 285                 {\r
 286                         string testId = test.GetAttribute ("ID");\r
 287                         totalListWriter.Write (testId + "\t");
 288 \r
 289                         if (isok) {\r
 290                                 ++passedCount;\r
 291                                 if (fixmeList.Contains (testId) || knownFailures.Contains (testId)) {\r
 292                                         ++fixedCount;\r
 293                                         fixedListWriter.WriteLine (testId);\r
 294                                         Console.Error.Write ("!");\r
 295                                         totalListWriter.WriteLine ("fixed");
 296                                         return true;\r
 297                                 }\r
 298                                 if (netFailures.Contains (testId)) {\r
 299                                         Console.Error.Write (",");\r
 300                                         totalListWriter.WriteLine (",");
 301                                         return true;\r
 302                                 }\r
 303 \r
 304                                 Console.Error.Write (".");\r
 305                                 totalListWriter.WriteLine (".");
 306                                 return true;\r
 307                         }\r
 308 \r
 309                         ++failedCount;\r
 310 \r
 311                         if (netFailures.Contains (testId)) {\r
 312                                 Console.Error.Write ("K");\r
 313                                 totalListWriter.WriteLine ("dot net known failure");
 314                                 return true;\r
 315                         }\r
 316                         if (knownFailures.Contains (testId)) {\r
 317                                 Console.Error.Write ("k");\r
 318                                 totalListWriter.WriteLine ("known failure");
 319                                 return true;\r
 320                         }\r
 321                         if (fixmeList.Contains (testId)) {\r
 322                                 Console.Error.Write ("f");\r
 323                                 totalListWriter.WriteLine ("fixme");
 324                                 return true;\r
 325                         }\r
 326 \r
 327                         ++regressionsCount;\r
 328                         Console.Error.Write ("E");\r
 329                         totalListWriter.WriteLine ("regression");
 330                         failedListWriter.Write ("*** Test failed:\t{0}\ttype:{1}\tnonValidatingPassed:{2},validatingPassed:{3}\t",\r
 331                                 testId, test.GetAttribute ("TYPE"), nonValidatingPassed, validatingPassed);\r
 332                         failedListWriter.WriteLine (test.InnerXml);\r
 333                         return false;\r
 334                 }\r
 335         }\r
 336 }\r