mcs/tools/monkeydoc/Monkeydoc/providers/ecma-provider.cs

   1 //
   2 // The ecmaspec provider is for ECMA specifications
   3 //
   4 // Authors:
   5 //      John Luke (jluke@cfl.rr.com)
   6 //      Ben Maurer (bmaurer@users.sourceforge.net)
   7 //
   8 // Use like this:
   9 //   mono assembler.exe --ecmaspec DIRECTORY --out name
  10 //
  11
  12 using System;
  13 using System.Linq;
  14 using System.IO;
  15 using System.Text;
  16 using System.Xml;
  17 using System.Xml.Linq;
  18 using System.Collections.Generic;
  19
  20 using Lucene.Net.Index;
  21 using Lucene.Net.Documents;
  22
  23 using Monkeydoc.Ecma;
  24 using Mono.Utilities;
  25
  26 namespace MonkeyDoc.Providers
  27 {
  28         public enum EcmaNodeType {
  29                 Invalid,
  30                 Namespace,
  31                 Type,
  32                 Member,
  33                 Meta, // A node that's here to serve as a header for other node
  34         }
  35
  36         public class EcmaProvider : Provider
  37         {
  38                 HashSet<string> directories = new HashSet<string> ();
  39
  40                 public EcmaProvider ()
  41                 {
  42                 }
  43
  44                 public EcmaProvider (string baseDir)
  45                 {
  46                         AddDirectory (baseDir);
  47                 }
  48
  49                 public void AddDirectory (string directory)
  50                 {
  51                         if (string.IsNullOrEmpty (directory))
  52                                 throw new ArgumentNullException ("directory");
  53
  54                         directories.Add (directory);
  55                 }
  56
  57                 public override void PopulateTree (Tree tree)
  58                 {
  59                         var root = tree.RootNode;
  60                         var storage = tree.HelpSource.Storage;
  61                         int resID = 0;
  62                         var nsSummaries = new Dictionary<string, XElement> ();
  63
  64                         foreach (var asm in directories) {
  65                                 var indexFilePath = Path.Combine (asm, "index.xml");
  66                                 if (!File.Exists (indexFilePath)) {
  67                                         Console.Error.WriteLine ("Warning: couldn't process directory `{0}' as it has no index.xml file", asm);
  68                                         continue;
  69                                 }
  70                                 using (var reader = XmlReader.Create (File.OpenRead (indexFilePath))) {
  71                                         reader.ReadToFollowing ("Types");
  72                                         var types = XElement.Load (reader.ReadSubtree ());
  73
  74                                         foreach (var ns in types.Elements ("Namespace")) {
  75                                                 var nsName = (string)ns.Attribute ("Name");
  76                                                 nsName = !string.IsNullOrEmpty (nsName) ? nsName : "global";
  77                                                 var nsNode = root.GetOrCreateNode (nsName, "N:" + nsName);
  78
  79                                                 XElement nsElements;
  80                                                 if (!nsSummaries.TryGetValue (nsName, out nsElements))
  81                                                         nsSummaries[nsName] = nsElements = new XElement ("elements",
  82                                                                                                          new XElement ("summary"),
  83                                                                                                          new XElement ("remarks"));
  84
  85                                                 foreach (var type in ns.Elements ("Type")) {
  86                                                         // Add the XML file corresponding to the type to our storage
  87                                                         var id = resID++;
  88                                                         var typeFilePath = Path.Combine (asm, nsName, Path.ChangeExtension (type.Attribute ("Name").Value, ".xml"));
  89                                                         if (!File.Exists (typeFilePath)) {
  90                                                                 Console.Error.WriteLine ("Warning: couldn't process type file `{0}' as it doesn't exist", typeFilePath);
  91                                                                 continue;
  92                                                         }
  93                                                         using (var file = File.OpenRead (typeFilePath))
  94                                                                 storage.Store (id.ToString (), file);
  95                                                         nsElements.Add (ExtractClassSummary (typeFilePath));
  96                                                         var typeDocument = XDocument.Load (typeFilePath);
  97
  98                                                         var typeCaption = ((string)(type.Attribute ("DisplayName") ?? type.Attribute ("Name"))).Replace ('+', '.');
  99                                                         var url = "ecma:" + id + '#' + typeCaption + '/';
 100                                                         typeCaption += " " + (string)type.Attribute ("Kind");
 101                                                         var typeNode = nsNode.CreateNode (typeCaption, url);
 102
 103                                                         // Add meta "Members" node
 104                                                         typeNode.CreateNode ("Members", "*");
 105                                                         var membersNode = typeDocument.Root.Element ("Members");
 106                                                         if (membersNode == null || !membersNode.Elements ().Any ())
 107                                                                 continue;
 108                                                         var members = membersNode
 109                                                                 .Elements ("Member")
 110                                                                 .ToLookup (m => m.Attribute ("MemberName").Value.StartsWith ("op_") ? "Operator" : m.Element ("MemberType").Value);
 111
 112                                                         foreach (var memberType in members) {
 113                                                                 // We pluralize the member type to get the caption and take the first letter as URL
 114                                                                 var node = typeNode.CreateNode (PluralizeMemberType (memberType.Key), memberType.Key[0].ToString ());
 115                                                                 var memberIndex = 0;
 116
 117                                                                 var isCtors = memberType.Key[0] == 'C';
 118
 119                                                                 // We do not escape much member name here
 120                                                                 foreach (var memberGroup in memberType.GroupBy (m => MakeMemberCaption (m, isCtors))) {
 121                                                                         if (memberGroup.Count () > 1) {
 122                                                                                 // Generate overload
 123                                                                                 var overloadCaption = MakeMemberCaption (memberGroup.First (), false);
 124                                                                                 var overloadNode = node.CreateNode (overloadCaption, overloadCaption);
 125                                                                                 foreach (var member in memberGroup)
 126                                                                                         overloadNode.CreateNode (MakeMemberCaption (member, true), (memberIndex++).ToString ());
 127                                                                                 overloadNode.Sort ();
 128                                                                         } else {
 129                                                                                 // We treat constructor differently by showing their argument list in all cases
 130                                                                                 node.CreateNode (MakeMemberCaption (memberGroup.First (), isCtors), (memberIndex++).ToString ());
 131                                                                         }
 132                                                                 }
 133                                                                 node.Sort ();
 134                                                         }
 135                                                 }
 136
 137                                                 nsNode.Sort ();
 138                                         }
 139                                         root.Sort ();
 140                                 }
 141                         }
 142
 143                         foreach (var summary in nsSummaries)
 144                                 storage.Store ("xml.summary." + summary.Key, summary.Value.ToString ());
 145
 146                         var masterSummary = new XElement ("elements",
 147                                                           directories
 148                                                           .SelectMany (d => Directory.EnumerateFiles (d, "ns-*.xml"))
 149                                                           .Select (ExtractNamespaceSummary));
 150                         storage.Store ("mastersummary.xml", masterSummary.ToString ());
 151                 }
 152
 153                 string PluralizeMemberType (string memberType)
 154                 {
 155                         switch (memberType) {
 156                         case "Property":
 157                                 return "Properties";
 158                         default:
 159                                 return memberType + "s";
 160                         }
 161                 }
 162
 163                 string MakeMemberCaption (XElement member, bool withArguments)
 164                 {
 165                         var caption = (string)member.Attribute ("MemberName");
 166                         // Use type name instead of .ctor for cosmetic sake
 167                         if (caption == ".ctor") {
 168                                 caption = (string)member.Ancestors ("Type").First ().Attribute ("Name");
 169                                 // If this is an inner type ctor, strip the parent type reference
 170                                 var plusIndex = caption.LastIndexOf ('+');
 171                                 if (plusIndex != -1)
 172                                         caption = caption.Substring (plusIndex + 1);
 173                         }
 174                         if (caption.StartsWith ("op_")) {
 175                                 string sig;
 176                                 caption = MakeOperatorSignature (member, out sig);
 177                                 caption = withArguments ? sig : caption;
 178                                 return caption;
 179                         }
 180                         if (withArguments) {
 181                                 var args = member.Element ("Parameters");
 182                                 caption += '(';
 183                                 if (args != null && args.Elements ("Parameter").Any ()) {
 184                                         caption += args.Elements ("Parameter")
 185                                                 .Select (p => (string)p.Attribute ("Type"))
 186                                                 .Aggregate ((p1, p2) => p1 + "," + p2);
 187                                 }
 188                                 caption += ')';
 189                         }
 190
 191                         return caption;
 192                 }
 193
 194                 XElement ExtractClassSummary (string typeFilePath)
 195                 {
 196                         using (var reader = XmlReader.Create (typeFilePath)) {
 197                                 reader.ReadToFollowing ("Type");
 198                                 var name = reader.GetAttribute ("Name");
 199                                 var fullName = reader.GetAttribute ("FullName");
 200                                 reader.ReadToFollowing ("AssemblyName");
 201                                 var assemblyName = reader.ReadElementString ();
 202                                 reader.ReadToFollowing ("summary");
 203                                 var summary = reader.ReadInnerXml ();
 204                                 reader.ReadToFollowing ("remarks");
 205                                 var remarks = reader.ReadInnerXml ();
 206
 207                                 return new XElement ("class",
 208                                                      new XAttribute ("name", name ?? string.Empty),
 209                                                      new XAttribute ("fullname", fullName ?? string.Empty),
 210                                                      new XAttribute ("assembly", assemblyName ?? string.Empty),
 211                                                      new XElement ("summary", new XCData (summary)),
 212                                                      new XElement ("remarks", new XCData (remarks)));
 213                         }
 214                 }
 215
 216                 XElement ExtractNamespaceSummary (string nsFile)
 217                 {
 218                         using (var reader = XmlReader.Create (nsFile)) {
 219                                 reader.ReadToFollowing ("Namespace");
 220                                 var name = reader.GetAttribute ("Name");
 221                                 reader.ReadToFollowing ("summary");
 222                                 var summary = reader.ReadInnerXml ();
 223                                 reader.ReadToFollowing ("remarks");
 224                                 var remarks = reader.ReadInnerXml ();
 225
 226                                 return new XElement ("namespace",
 227                                                      new XAttribute ("ns", name ?? string.Empty),
 228                                                      new XElement ("summary", new XCData (summary)),
 229                                                      new XElement ("remarks", new XCData (remarks)));
 230                         }
 231                 }
 232
 233                 public override void CloseTree (HelpSource hs, Tree tree)
 234                 {
 235                         AddImages (hs);
 236                         AddExtensionMethods (hs);
 237                 }
 238
 239                 void AddEcmaXml (HelpSource hs)
 240                 {
 241                         var xmls = directories
 242                                 .SelectMany (Directory.EnumerateDirectories) // Assemblies
 243                                 .SelectMany (Directory.EnumerateDirectories) // Namespaces
 244                                 .SelectMany (Directory.EnumerateFiles)
 245                                 .Where (f => f.EndsWith (".xml")); // Type XML files
 246
 247                         int resID = 0;
 248                         foreach (var xml in xmls)
 249                                 using (var file = File.OpenRead (xml))
 250                                         hs.Storage.Store ((resID++).ToString (), file);
 251                 }
 252
 253                 void AddImages (HelpSource hs)
 254                 {
 255                         var imgs = directories
 256                                 .SelectMany (Directory.EnumerateDirectories)
 257                                 .Select (d => Path.Combine (d, "_images"))
 258                                 .Where (Directory.Exists)
 259                                 .SelectMany (Directory.EnumerateFiles);
 260
 261                         foreach (var img in imgs)
 262                                 using (var file = File.OpenRead (img))
 263                                         hs.Storage.Store (Path.GetFileName (img), file);
 264                 }
 265
 266                 void AddExtensionMethods (HelpSource hs)
 267                 {
 268                         var extensionMethods = directories
 269                                 .SelectMany (Directory.EnumerateDirectories)
 270                                 .Select (d => Path.Combine (d, "index.xml"))
 271                                 .Where (File.Exists)
 272                                 .Select (f => {
 273                                         using (var file = File.OpenRead (f)) {
 274                                                 var reader = XmlReader.Create (file);
 275                                                 reader.ReadToFollowing ("ExtensionMethods");
 276                                                 return reader.ReadInnerXml ();
 277                                         }
 278                             })
 279                             .DefaultIfEmpty (string.Empty);
 280
 281                         hs.Storage.Store ("ExtensionMethods.xml",
 282                                           "<ExtensionMethods>" + extensionMethods.Aggregate (string.Concat) + "</ExtensionMethods>");
 283                 }
 284
 285                 IEnumerable<string> GetEcmaXmls ()
 286                 {
 287                         return directories
 288                                 .SelectMany (Directory.EnumerateDirectories) // Assemblies
 289                                 .SelectMany (Directory.EnumerateDirectories) // Namespaces
 290                                 .SelectMany (Directory.EnumerateFiles)
 291                                 .Where (f => f.EndsWith (".xml")); // Type XML files
 292                 }
 293
 294                 string MakeOperatorSignature (XElement member, out string memberSignature)
 295                 {
 296                         string name = (string)member.Attribute ("MemberName");
 297                         var nicename = name.Substring(3);
 298                         memberSignature = null;
 299
 300                         switch (name) {
 301                         // unary operators: no overloading possible     [ECMA-335 §10.3.1]
 302                         case "op_UnaryPlus":                    // static     R operator+       (T)
 303                         case "op_UnaryNegation":                // static     R operator-       (T)
 304                         case "op_LogicalNot":                   // static     R operator!       (T)
 305                         case "op_OnesComplement":               // static     R operator~       (T)
 306                         case "op_Increment":                    // static     R operator++      (T)
 307                         case "op_Decrement":                    // static     R operator--      (T)
 308                         case "op_True":                         // static  bool operator true   (T)
 309                         case "op_False":                        // static  bool operator false  (T)
 310                         case "op_AddressOf":                    // static     R operator&       (T)
 311                         case "op_PointerDereference":           // static     R operator*       (T)
 312                                 memberSignature = nicename;
 313                                 break;
 314                         // conversion operators: overloading based on parameter and return type [ECMA-335 §10.3.3]
 315                         case "op_Implicit":                    // static implicit operator R (T)
 316                         case "op_Explicit":                    // static explicit operator R (T)
 317                                 nicename = name.EndsWith ("Implicit") ? "ImplicitConversion" : "ExplicitConversion";
 318                                 string arg = (string)member.Element ("Parameters").Element ("Parameter").Attribute ("Type");
 319                                 string ret = (string)member.Element ("ReturnValue").Element ("ReturnType");
 320                                 memberSignature = arg + " to " + ret;
 321                                 break;
 322                         // binary operators: overloading is possible [ECMA-335 §10.3.2]
 323                         default:
 324                                 memberSignature =
 325                                         nicename + "("
 326                                         + string.Join (",", member.Element ("Parameters").Elements ("Parameter").Select (p => (string)p.Attribute ("Type")))
 327                                         + ")";
 328                                 break;
 329                         }
 330
 331                         return nicename;
 332                 }
 333         }
 334
 335         public class EcmaHelpSource : HelpSource
 336         {
 337                 const string EcmaPrefix = "ecma:";
 338                 EcmaUrlParser parser = new EcmaUrlParser ();
 339                 LRUCache<string, Node> cache = new LRUCache<string, Node> (4);
 340
 341                 public EcmaHelpSource (string base_file, bool create) : base (base_file, create)
 342                 {
 343                 }
 344
 345                 protected override string UriPrefix {
 346                         get {
 347                                 return EcmaPrefix;
 348                         }
 349                 }
 350
 351                 public override bool CanHandleUrl (string url)
 352                 {
 353                         if (url.Length > 2 && url[1] == ':') {
 354                                 switch (url[0]) {
 355                                 case 'T':
 356                                 case 'M':
 357                                 case 'C':
 358                                 case 'P':
 359                                 case 'E':
 360                                 case 'F':
 361                                 case 'N':
 362                                 case 'O':
 363                                         return true;
 364                                 }
 365                         }
 366                         return base.CanHandleUrl (url);
 367                 }
 368
 369                 // Clean the extra paramers in the id
 370                 public override Stream GetHelpStream (string id)
 371                 {
 372                         var idParts = id.Split ('?');
 373                         return base.GetHelpStream (idParts[0]);
 374                 }
 375
 376                 public override Stream GetCachedHelpStream (string id)
 377                 {
 378                         var idParts = id.Split ('?');
 379                         return base.GetCachedHelpStream (idParts[0]);
 380                 }
 381
 382                 public override DocumentType GetDocumentTypeForId (string id, out Dictionary<string, string> extraParams)
 383                 {
 384                         extraParams = null;
 385                         int interMark = id.LastIndexOf ('?');
 386                         if (interMark != -1)
 387                                 extraParams = id.Substring (interMark)
 388                                         .Split ('&')
 389                                         .Select (nvp => {
 390                                                 var eqIdx = nvp.IndexOf ('=');
 391                                                 return new { Key = nvp.Substring (0, eqIdx < 0 ? nvp.Length : eqIdx), Value = nvp.Substring (eqIdx + 1) };
 392                                         })
 393                                         .ToDictionary (kvp => kvp.Key, kvp => kvp.Value );
 394                         return DocumentType.EcmaXml;
 395                 }
 396
 397                 public override string GetPublicUrl (Node node)
 398                 {
 399                         string url = string.Empty;
 400                         var type = GetNodeType (node);
 401                         //Console.WriteLine ("GetPublicUrl {0} : {1} [{2}]", node.Element, node.Caption, type.ToString ());
 402                         switch (type) {
 403                         case EcmaNodeType.Namespace:
 404                                 return node.Element; // A namespace node has already a well formated internal url
 405                         case EcmaNodeType.Type:
 406                                 return MakeTypeNodeUrl (node);
 407                         case EcmaNodeType.Meta:
 408                                 return MakeTypeNodeUrl (GetNodeTypeParent (node)) + GenerateMetaSuffix (node);
 409                         case EcmaNodeType.Member:
 410                                 var typeChar = GetNodeMemberTypeChar (node);
 411                                 var parentNode = GetNodeTypeParent (node);
 412                                 var typeNode = MakeTypeNodeUrl (parentNode).Substring (2);
 413                                 return typeChar + ":" + typeNode + MakeMemberNodeUrl (typeChar, node);
 414                         default:
 415                                 return null;
 416                         }
 417                 }
 418
 419                 string MakeTypeNodeUrl (Node node)
 420                 {
 421                         // A Type node has a Element property of the form: 'ecma:{number}#{typename}/'
 422                         var hashIndex = node.Element.IndexOf ('#');
 423                         var typeName = node.Element.Substring (hashIndex + 1, node.Element.Length - hashIndex - 2);
 424                         return "T:" + node.Parent.Caption + '.' + typeName.Replace ('.', '+');
 425                 }
 426
 427                 string MakeMemberNodeUrl (char typeChar, Node node)
 428                 {
 429                         // We clean inner type ctor name which may contain the outer type name
 430                         var caption = node.Caption;
 431
 432                         // Sanitize constructor caption of inner types
 433                         if (typeChar == 'C') {
 434                                 int lastDot = -1;
 435                                 for (int i = 0; i < caption.Length && caption[i] != '('; i++)
 436                                         lastDot = caption[i] == '.' ? i : lastDot;
 437                                 return lastDot == -1 ? '.' + caption : caption.Substring (lastDot);
 438                         }
 439
 440                         /* We handle type conversion operator by checking if the name contains " to "
 441                          * (as in 'foo to bar') and we generate a corresponding conversion signature
 442                          */
 443                         if (typeChar == 'O' && caption.IndexOf (" to ") != -1) {
 444                                 var parts = caption.Split (' ');
 445                                 return "." + node.Parent.Caption + "(" + parts[0] + ", " + parts[2] + ")";
 446                         }
 447
 448                         /* The goal here is to treat method which are explicit interface definition
 449                          * such as 'void IDisposable.Dispose ()' for which the caption is a dot
 450                          * expression thus colliding with the ecma parser.
 451                          * If the first non-alpha character in the caption is a dot then we have an
 452                          * explicit member implementation (we assume the interface has namespace)
 453                          */
 454                         var firstNonAlpha = caption.FirstOrDefault (c => !char.IsLetterOrDigit (c));
 455                         if (firstNonAlpha == '.')
 456                                 return "$" + caption;
 457
 458                         return "." + caption;
 459                 }
 460
 461                 EcmaNodeType GetNodeType (Node node)
 462                 {
 463                         // We guess the node type by checking the depth level it's at in the tree
 464                         int level = GetNodeLevel (node);
 465                         switch (level) {
 466                         case 0:
 467                                 return EcmaNodeType.Namespace;
 468                         case 1:
 469                                 return EcmaNodeType.Type;
 470                         case 2:
 471                                 return EcmaNodeType.Meta;
 472                         case 3: // Here it's either a member or, in case of overload, a meta
 473                                 return node.IsLeaf ? EcmaNodeType.Member : EcmaNodeType.Meta;
 474                         case 4: // At this level, everything is necessarily a member
 475                                 return EcmaNodeType.Member;
 476                         default:
 477                                 return EcmaNodeType.Invalid;
 478                         }
 479                 }
 480
 481                 int GetNodeLevel (Node node)
 482                 {
 483                         int i = 0;
 484                         for (; !node.Element.StartsWith ("root:/", StringComparison.OrdinalIgnoreCase); i++) {
 485                                 //Console.WriteLine ("\tLevel {0} : {1} {2}", i, node.Element, node.Caption);
 486                                 node = node.Parent;
 487                         }
 488                         return i - 1;
 489                 }
 490
 491                 char GetNodeMemberTypeChar (Node node)
 492                 {
 493                         int level = GetNodeLevel (node);
 494                         // We try to reach the member group node depending on node nested level
 495                         switch (level) {
 496                         case 2:
 497                                 return node.Element[0];
 498                         case 3:
 499                                 return node.Parent.Element[0];
 500                         case 4:
 501                                 return node.Parent.Parent.Element[0];
 502                         default:
 503                                 throw new ArgumentException ("node", "Couldn't determine member type of node `" + node.Caption + "'");
 504                         }
 505                 }
 506
 507                 Node GetNodeTypeParent (Node node)
 508                 {
 509                         // Type nodes are always at level 2 so we just need to get there
 510                         while (node != null && node.Parent != null && !node.Parent.Parent.Element.StartsWith ("root:/", StringComparison.OrdinalIgnoreCase))
 511                                 node = node.Parent;
 512                         return node;
 513                 }
 514
 515                 string GenerateMetaSuffix (Node node)
 516                 {
 517                         string suffix = string.Empty;
 518                         // A meta node has always a type element to begin with
 519                         while (GetNodeType (node) != EcmaNodeType.Type) {
 520                                 suffix = '/' + node.Element + suffix;
 521                                 node = node.Parent;
 522                         }
 523                         return suffix;
 524                 }
 525
 526                 public override string GetInternalIdForUrl (string url, out Node node)
 527                 {
 528                         var id = string.Empty;
 529                         node = null;
 530
 531                         if (!url.StartsWith (EcmaPrefix, StringComparison.OrdinalIgnoreCase)) {
 532                                 node = MatchNode (url);
 533                                 if (node == null)
 534                                         return null;
 535                                 id = node.GetInternalUrl ();
 536                         }
 537
 538                         string hash;
 539                         id = GetInternalIdForInternalUrl (id, out hash);
 540
 541                         return id + GetArgs (hash, node);
 542                 }
 543
 544                 public string GetInternalIdForInternalUrl (string internalUrl, out string hash)
 545                 {
 546                         var id = internalUrl;
 547                         if (id.StartsWith (UriPrefix, StringComparison.OrdinalIgnoreCase))
 548                                 id = id.Substring (UriPrefix.Length);
 549                         else if (id.StartsWith ("N:", StringComparison.OrdinalIgnoreCase))
 550                                 id = "xml.summary." + id.Substring ("N:".Length);
 551
 552                         var hashIndex = id.IndexOf ('#');
 553                         hash = string.Empty;
 554                         if (hashIndex != -1) {
 555                                 hash = id.Substring (hashIndex + 1);
 556                                 id = id.Substring (0, hashIndex);
 557                         }
 558
 559                         return id;
 560                 }
 561
 562                 public override Node MatchNode (string url)
 563                 {
 564                         Node node = null;
 565                         if ((node = cache.Get (url)) == null) {
 566                                 node = InternalMatchNode (url);
 567                                 if (node != null)
 568                                         cache.Put (url, node);
 569                         }
 570                         return node;
 571                 }
 572
 573                 public Node InternalMatchNode (string url)
 574                 {
 575                         Node result = null;
 576                         EcmaDesc desc;
 577                         if (!parser.TryParse (url, out desc))
 578                                 return null;
 579
 580                         // Namespace search
 581                         Node currentNode = Tree.RootNode;
 582                         Node searchNode = new Node () { Caption = desc.Namespace };
 583                         int index = currentNode.Nodes.BinarySearch (searchNode, EcmaGenericNodeComparer.Instance);
 584                         if (index >= 0)
 585                                 result = currentNode.Nodes[index];
 586                         if (desc.DescKind == EcmaDesc.Kind.Namespace || index < 0)
 587                                 return result;
 588
 589                         // Type search
 590                         currentNode = result;
 591                         result = null;
 592                         searchNode.Caption = desc.ToCompleteTypeName ();
 593                         index = currentNode.Nodes.BinarySearch (searchNode, EcmaTypeNodeComparer.Instance);
 594                         if (index >= 0)
 595                                 result = currentNode.Nodes[index];
 596                         if ((desc.DescKind == EcmaDesc.Kind.Type && !desc.IsEtc) || index < 0)
 597                                 return result;
 598
 599                         // Member selection
 600                         currentNode = result;
 601                         result = null;
 602                         var caption = desc.IsEtc ? EtcKindToCaption (desc.Etc) : MemberKindToCaption (desc.DescKind);
 603                         currentNode = FindNodeForCaption (currentNode.Nodes, caption);
 604                         if (currentNode == null
 605                             || (desc.IsEtc && desc.DescKind == EcmaDesc.Kind.Type && string.IsNullOrEmpty (desc.EtcFilter)))
 606                                 return currentNode;
 607
 608                         // Member search
 609                         result = null;
 610                         var format = desc.DescKind == EcmaDesc.Kind.Constructor ? EcmaDesc.Format.WithArgs : EcmaDesc.Format.WithoutArgs;
 611                         searchNode.Caption = desc.ToCompleteMemberName (format);
 612                         index = currentNode.Nodes.BinarySearch (searchNode, EcmaGenericNodeComparer.Instance);
 613                         if (index < 0)
 614                                 return null;
 615                         result = currentNode.Nodes[index];
 616                         if (result.Nodes.Count == 0 || desc.IsEtc)
 617                                 return result;
 618
 619                         // Overloads search
 620                         currentNode = result;
 621                         searchNode.Caption = desc.ToCompleteMemberName (EcmaDesc.Format.WithArgs);
 622                         index = currentNode.Nodes.BinarySearch (searchNode, EcmaGenericNodeComparer.Instance);
 623                         if (index < 0)
 624                                 return result;
 625                         result = result.Nodes[index];
 626
 627                         return result;
 628                 }
 629
 630                 // This comparer returns the answer straight from caption comparison
 631                 class EcmaGenericNodeComparer : IComparer<Node>
 632                 {
 633                         public static readonly EcmaGenericNodeComparer Instance = new EcmaGenericNodeComparer ();
 634
 635                         public int Compare (Node n1, Node n2)
 636                         {
 637                                 return string.Compare (n1.Caption, n2.Caption, StringComparison.Ordinal);
 638                         }
 639                 }
 640
 641                 // This comparer take into account the space in the caption
 642                 class EcmaTypeNodeComparer : IComparer<Node>
 643                 {
 644                         public static readonly EcmaTypeNodeComparer Instance = new EcmaTypeNodeComparer ();
 645
 646                         public int Compare (Node n1, Node n2)
 647                         {
 648                                 int length1 = CaptionLength (n1.Caption);
 649                                 int length2 = CaptionLength (n2.Caption);
 650
 651                                 return string.Compare (n1.Caption, 0, n2.Caption, 0, Math.Max (length1, length2), StringComparison.Ordinal);
 652                         }
 653
 654                         int CaptionLength (string caption)
 655                         {
 656                                 var length = caption.LastIndexOf (' ');
 657                                 return length == -1 ? caption.Length : length;
 658                         }
 659                 }
 660
 661                 string EtcKindToCaption (char etc)
 662                 {
 663                         switch (etc) {
 664                         case 'M':
 665                                 return "Methods";
 666                         case 'P':
 667                                 return "Properties";
 668                         case 'C':
 669                                 return "Constructors";
 670                         case 'F':
 671                                 return "Fields";
 672                         case 'E':
 673                                 return "Events";
 674                         case 'O':
 675                                 return "Operators";
 676                         case '*':
 677                                 return "Members";
 678                         default:
 679                                 return null;
 680                         }
 681                 }
 682
 683                 string MemberKindToCaption (EcmaDesc.Kind kind)
 684                 {
 685                         switch (kind) {
 686                         case EcmaDesc.Kind.Method:
 687                                 return "Methods";
 688                         case EcmaDesc.Kind.Property:
 689                                 return "Properties";
 690                         case EcmaDesc.Kind.Constructor:
 691                                 return "Constructors";
 692                         case EcmaDesc.Kind.Field:
 693                                 return "Fields";
 694                         case EcmaDesc.Kind.Event:
 695                                 return "Events";
 696                         case EcmaDesc.Kind.Operator:
 697                                 return "Operators";
 698                         default:
 699                                 return null;
 700                         }
 701                 }
 702
 703                 Node FindNodeForCaption (List<Node> nodes, string caption)
 704                 {
 705                         foreach (var node in nodes)
 706                                 if (node.Caption.Equals (caption, StringComparison.OrdinalIgnoreCase))
 707                                         return node;
 708                         return null;
 709                 }
 710
 711                 string GetArgs (string hash, Node node)
 712                 {
 713                         var args = new Dictionary<string, string> ();
 714
 715                         args["source-id"] = SourceID.ToString ();
 716
 717                         if (node != null) {
 718                                 var nodeType = GetNodeType (node);
 719                                 switch (nodeType) {
 720                                 case EcmaNodeType.Namespace:
 721                                         args["show"] = "namespace";
 722                                         args["namespace"] = node.Element.Substring ("N:".Length);
 723                                         break;
 724                                 case EcmaNodeType.Type:
 725                                         args["show"] = "typeoverview";
 726                                         break;
 727                                 case EcmaNodeType.Member:
 728                                 case EcmaNodeType.Meta:
 729                                         switch (GetNodeMemberTypeChar (node)){
 730                                         case 'C':
 731                                                 args["membertype"] = "Constructor";
 732                                                 break;
 733                                         case 'M':
 734                                                 args["membertype"] = "Method";
 735                                                 break;
 736                                         case 'P':
 737                                                 args["membertype"] = "Property";
 738                                                 break;
 739                                         case 'F':
 740                                                 args["membertype"] = "Field";
 741                                                 break;
 742                                         case 'E':
 743                                                 args["membertype"] = "Event";
 744                                                 break;
 745                                         case 'O':
 746                                                 args["membertype"] = "Operator";
 747                                                 break;
 748                                         case 'X':
 749                                                 args["membertype"] = "ExtensionMethod";
 750                                                 break;
 751                                         case '*':
 752                                                 args["membertype"] = "All";
 753                                                 break;
 754                                         }
 755
 756                                         if (nodeType == EcmaNodeType.Meta) {
 757                                                 args["show"] = "members";
 758                                                 args["index"] = "all";
 759                                         } else {
 760                                                 args["show"] = "member";
 761                                                 args["index"] = node.Element;
 762                                         }
 763                                         break;
 764                                 }
 765                         }
 766
 767                         if (!string.IsNullOrEmpty (hash))
 768                                 args["hash"] = hash;
 769
 770                         return "?" + string.Join ("&", args.Select (kvp => kvp.Key == kvp.Value ? kvp.Key : kvp.Key + '=' + kvp.Value));
 771                 }
 772
 773                 public override void PopulateSearchableIndex (IndexWriter writer)
 774                 {
 775                         StringBuilder text = new StringBuilder ();
 776                         SearchableDocument searchDoc = new SearchableDocument ();
 777
 778                         foreach (Node ns_node in Tree.RootNode.Nodes) {
 779                                 foreach (Node type_node in ns_node.Nodes) {
 780                                         string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' '));
 781                                         string full = ns_node.Caption + "." + typename;
 782                                         string doc_tag = GetKindFromCaption (type_node.Caption);
 783                                         string url = "T:" + full;
 784                                         string rest, hash;
 785                                         var id = GetInternalIdForInternalUrl (type_node.GetInternalUrl (), out hash);
 786                                         var xdoc = XDocument.Load (GetHelpStream (id));
 787                                         if (xdoc == null)
 788                                                 continue;
 789                                         if (string.IsNullOrEmpty (doc_tag)) {
 790                                                 Console.WriteLine (type_node.Caption);
 791                                                 continue;
 792                                         }
 793
 794                                         // For classes, structures or interfaces add a doc for the overview and
 795                                         // add a doc for every constructor, method, event, ...
 796                                         // doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"
 797                                         if (doc_tag[0] == 'C' || doc_tag[0] == 'S' || doc_tag[0] == 'I') {
 798                                                 // Adds a doc for every overview of every type
 799                                                 SearchableDocument doc = searchDoc.Reset ();
 800                                                 doc.Title = type_node.Caption;
 801                                                 doc.HotText = typename;
 802                                                 doc.Url = url;
 803                                                 doc.FullTitle = full;
 804
 805                                                 var node_sel = xdoc.Root.Element ("Docs");
 806                                                 text.Clear ();
 807                                                 GetTextFromNode (node_sel, text);
 808                                                 doc.Text = text.ToString ();
 809
 810                                                 text.Clear ();
 811                                                 GetExamples (node_sel, text);
 812                                                 doc.Examples = text.ToString ();
 813
 814                                                 writer.AddDocument (doc.LuceneDoc);
 815                                                 var exportParsable = doc_tag[0] == 'C' && (ns_node.Caption.StartsWith ("MonoTouch") || ns_node.Caption.StartsWith ("MonoMac"));
 816
 817                                                 //Add docs for contructors, methods, etc.
 818                                                 foreach (Node c in type_node.Nodes) { // c = Constructors || Fields || Events || Properties || Methods || Operators
 819                                                         if (c.Element == "*")
 820                                                                 continue;
 821                                                         const float innerTypeBoost = 0.2f;
 822
 823                                                         IEnumerable<Node> ncnodes = c.Nodes;
 824                                                         // The rationale is that we need to properly handle method overloads
 825                                                         // so for those method node which have children, flatten them
 826                                                         if (c.Caption == "Methods") {
 827                                                                 ncnodes = ncnodes
 828                                                                         .Where (n => n.Nodes == null || n.Nodes.Count == 0)
 829                                                                         .Concat (ncnodes.Where (n => n.Nodes.Count > 0).SelectMany (n => n.Nodes));
 830                                                         } else if (c.Caption == "Operators") {
 831                                                                 ncnodes = ncnodes
 832                                                                         .Where (n => !n.Caption.EndsWith ("Conversion"))
 833                                                                         .Concat (ncnodes.Where (n => n.Caption.EndsWith ("Conversion")).SelectMany (n => n.Nodes));
 834                                                         }
 835
 836                                                         foreach (Node nc in ncnodes) {
 837                                                                 var docsNode = GetDocsFromCaption (xdoc, c.Caption[0] == 'C' ? ".ctor" : nc.Caption, c.Caption[0] == 'O');
 838
 839                                                                 SearchableDocument doc_nod = searchDoc.Reset ();
 840                                                                 doc_nod.Title = LargeName (nc) + " " + EtcKindToCaption (c.Caption[0]);
 841                                                                 doc_nod.FullTitle = ns_node.Caption + '.' + typename + "::" + nc.Caption;
 842                                                                 doc_nod.HotText = string.Empty;
 843
 844                                                                 /* Disable constructors hottext indexing as it's often "polluting" search queries
 845                                                                    because it has the same hottext than standard types */
 846                                                                 if (c.Caption != "Constructors") {
 847                                                                         //dont add the parameters to the hottext
 848                                                                         int ppos = nc.Caption.IndexOf ('(');
 849                                                                         doc_nod.HotText = ppos != -1 ? nc.Caption.Substring (0, ppos) : nc.Caption;
 850                                                                 }
 851
 852                                                                 var urlnc = nc.PublicUrl;
 853                                                                 doc_nod.Url = urlnc;
 854
 855                                                                 if (docsNode == null) {
 856                                                                         Console.Error.WriteLine ("Problem: {0}", urlnc);
 857                                                                         continue;
 858                                                                 }
 859
 860                                                                 text.Clear ();
 861                                                                 GetTextFromNode (docsNode, text);
 862                                                                 doc_nod.Text = text.ToString ();
 863
 864                                                                 text.Clear ();
 865                                                                 GetExamples (docsNode, text);
 866                                                                 doc_nod.Examples = text.ToString ();
 867
 868                                                                 Document lucene_doc = doc_nod.LuceneDoc;
 869                                                                 lucene_doc.Boost = innerTypeBoost;
 870                                                                 writer.AddDocument (lucene_doc);
 871
 872                                                                 // MonoTouch/Monomac specific parsing of [Export] attributes
 873                                                                 /*if (exportParsable) {
 874                                                                         try {
 875                                                                                 var exports =
 876                                                                                         xdoc.SelectNodes (string.Format ("/Type/Members/Member[@MemberName='{0}']/Attributes/Attribute/AttributeName[contains(text(), 'Foundation.Export')]", nc.Caption));
 877                                                                                 foreach (XmlNode exportNode in exports) {
 878                                                                                         var inner = exportNode.InnerText;
 879                                                                                         var parts = inner.Split ('"');
 880                                                                                         if (parts.Length != 3) {
 881                                                                                                 Console.WriteLine ("Export attribute not found or not usable in {0}", inner);
 882                                                                                                 continue;
 883                                                                                         }
 884
 885                                                                                         var export = parts[1];
 886                                                                                         var export_node = new SearchableDocument ();
 887                                                                                         export_node.Title = export + " Export";
 888                                                                                         export_node.FullTitle = string.Format ("{0}.{1}::{2}", ns_node.Caption, typename, export);
 889                                                                                         export_node.Url = urlnc;
 890                                                                                         export_node.HotText = export + ":";
 891                                                                                         export_node.Text = string.Empty;
 892                                                                                         export_node.Examples = string.Empty;
 893                                                                                         lucene_doc = export_node.LuceneDoc;
 894                                                                                         lucene_doc.SetBoost (innerTypeBoost);
 895                                                                                         writer.AddDocument (lucene_doc);
 896                                                                                 }
 897                                                                         } catch (Exception e){
 898                                                                                 Console.WriteLine ("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e);
 899                                                                         }
 900                                                                 }*/
 901                                                         }
 902                                                 }
 903                                         // doc_tag == "Enumeration"
 904                                         } else if (doc_tag[0] == 'E'){
 905                                                 var members = xdoc.Root.Element ("Members").Elements ("Member");
 906                                                 if (members == null)
 907                                                         continue;
 908
 909                                                 text.Clear ();
 910                                                 foreach (var member_node in members) {
 911                                                         string enum_value = (string)member_node.Attribute ("MemberName");
 912                                                         text.Append (enum_value);
 913                                                         text.Append (" ");
 914                                                         GetTextFromNode (member_node.Element ("Docs"), text);
 915                                                         text.AppendLine ();
 916                                                 }
 917
 918                                                 SearchableDocument doc = searchDoc.Reset ();
 919
 920                                                 text.Clear ();
 921                                                 GetExamples (xdoc.Root.Element ("Docs"), text);
 922                                                 doc.Examples = text.ToString ();
 923
 924                                                 doc.Title = type_node.Caption;
 925                                                 doc.HotText = (string)xdoc.Root.Attribute ("Name");
 926                                                 doc.FullTitle = full;
 927                                                 doc.Url = url;
 928                                                 doc.Text = text.ToString();
 929                                                 writer.AddDocument (doc.LuceneDoc);
 930                                         // doc_tag == "Delegate"
 931                                         } else if (doc_tag[0] == 'D'){
 932                                                 SearchableDocument doc = searchDoc.Reset ();
 933                                                 doc.Title = type_node.Caption;
 934                                                 doc.HotText = (string)xdoc.Root.Attribute ("Name");
 935                                                 doc.FullTitle = full;
 936                                                 doc.Url = url;
 937
 938                                                 var node_sel = xdoc.Root.Element ("Docs");
 939
 940                                                 text.Clear ();
 941                                                 GetTextFromNode (node_sel, text);
 942                                                 doc.Text = text.ToString();
 943
 944                                                 text.Clear ();
 945                                                 GetExamples (node_sel, text);
 946                                                 doc.Examples = text.ToString();
 947
 948                                                 writer.AddDocument (doc.LuceneDoc);
 949                                         }
 950                                 }
 951                         }
 952                 }
 953
 954                 string GetKindFromCaption (string s)
 955                 {
 956                         int p = s.LastIndexOf (' ');
 957                         if (p > 0)
 958                                 return s.Substring (p + 1);
 959                         return null;
 960                 }
 961
 962                 // Extract the interesting text from the docs node
 963                 void GetTextFromNode (XElement n, StringBuilder sb)
 964                 {
 965                         // Include the text content of the docs
 966                         sb.AppendLine (n.Value);
 967                         foreach (var tag in n.Descendants ())
 968                                 //include the url to which points the see tag and the name of the parameter
 969                                 if ((tag.Name.LocalName.Equals ("see", StringComparison.Ordinal) || tag.Name.LocalName.Equals ("paramref", StringComparison.Ordinal))
 970                                     && tag.HasAttributes)
 971                                         sb.AppendLine ((string)tag.Attributes ().First ());
 972                 }
 973
 974                 // Extract the code nodes from the docs
 975                 void GetExamples (XElement n, StringBuilder sb)
 976                 {
 977                         foreach (var code in n.Descendants ("code"))
 978                                 sb.Append ((string)code);
 979                 }
 980
 981                 // Extract a large name for the Node
 982                 static string LargeName (Node matched_node)
 983                 {
 984                         string[] parts = matched_node.GetInternalUrl ().Split('/', '#');
 985                         if (parts.Length == 3 && parts[2] != String.Empty) //List of Members, properties, events, ...
 986                                 return parts[1] + ": " + matched_node.Caption;
 987                         else if(parts.Length >= 4) //Showing a concrete Member, property, ...
 988                                 return parts[1] + "." + matched_node.Caption;
 989                         else
 990                                 return matched_node.Caption;
 991                 }
 992
 993                 XElement GetDocsFromCaption (XDocument xdoc, string caption, bool isOperator)
 994                 {
 995                         string name;
 996                         IList<string> args;
 997                         var doc = xdoc.Root.Element ("Members").Elements ("Member");
 998
 999                         if (isOperator) {
1000                                 // The first case are explicit and implicit conversion operators which are grouped specifically
1001                                 if (caption.IndexOf (" to ") != -1) {
1002                                         var convArgs = caption.Split (new[] { " to " }, StringSplitOptions.None);
1003                                         return doc
1004                                                 .First (n => (AttrEq (n, "MemberName", "op_Explicit") || AttrEq (n, "MemberName", "op_Implicit"))
1005                                                         && ((string)n.Element ("ReturnValue").Element ("ReturnType")).Equals (convArgs[1], StringComparison.Ordinal)
1006                                                         && AttrEq (n.Element ("Parameters").Element ("Parameter"), "Type", convArgs[0]))
1007                                                 .Element ("Docs");
1008                                 } else {
1009                                         return doc.First (m => AttrEq (m, "MemberName", "op_" + caption)).Element ("Docs");
1010                                 }
1011                         }
1012
1013                         TryParseCaption (caption, out name, out args);
1014
1015                         if (!string.IsNullOrEmpty (name)) // Filter member by name
1016                                 doc = doc.Where (m => AttrEq (m, "MemberName", name));
1017                         if (args != null && args.Count > 0) // Filter member by its argument list
1018                                 doc = doc.Where (m => m.Element ("Parameters").Elements ("Parameter").Attributes ("Type").Select (a => (string)a).SequenceEqual (args));
1019
1020                         return doc.First ().Element ("Docs");
1021                 }
1022
1023                 // A simple stack-based parser to detect single type definition separated by commas
1024                 IEnumerable<string> ExtractArguments (string rawArgList)
1025                 {
1026                         var sb = new System.Text.StringBuilder ();
1027                         int genericDepth = 0;
1028                         int arrayDepth = 0;
1029
1030                         for (int i = 0; i < rawArgList.Length; i++) {
1031                                 char c = rawArgList[i];
1032                                 switch (c) {
1033                                 case ',':
1034                                         if (genericDepth == 0 && arrayDepth == 0) {
1035                                                 yield return sb.ToString ();
1036                                                 sb.Clear ();
1037                                                 continue;
1038                                         }
1039                                         break;
1040                                 case '<':
1041                                         genericDepth++;
1042                                         break;
1043                                 case '>':
1044                                         genericDepth--;
1045                                         break;
1046                                 case '[':
1047                                         arrayDepth++;
1048                                         break;
1049                                 case ']':
1050                                         arrayDepth--;
1051                                         break;
1052                                 }
1053                                 sb.Append (c);
1054                         }
1055                         if (sb.Length > 0)
1056                                 yield return sb.ToString ();
1057                 }
1058
1059                 void TryParseCaption (string caption, out string name, out IList<string> argList)
1060                 {
1061                         name = null;
1062                         argList = null;
1063                         int parenIdx = caption.IndexOf ('(');
1064                         // In case of simple name, there is no need for processing
1065                         if (parenIdx == -1) {
1066                                 name = caption;
1067                                 return;
1068                         }
1069                         name = caption.Substring (0, parenIdx);
1070                         // Now we gather the argument list if there is any
1071                         var rawArgList = caption.Substring (parenIdx + 1, caption.Length - parenIdx - 2); // Only take what's inside the parens
1072                         if (string.IsNullOrEmpty (rawArgList))
1073                                 return;
1074
1075                         argList = ExtractArguments (rawArgList).Select (arg => arg.Trim ()).ToList ();
1076                 }
1077
1078                 bool AttrEq (XElement element, string attributeName, string expectedValue)
1079                 {
1080                         return ((string)element.Attribute (attributeName)).Equals (expectedValue, StringComparison.Ordinal);
1081                 }
1082         }
1083 }