mcs/class/monodoc/Monodoc/providers/ecma-provider.cs

   1 //
   2 // The ecmaspec provider is for ECMA specifications
   3 //
   4 // Authors:
   5 //      John Luke (jluke@cfl.rr.com)
   6 //      Ben Maurer (bmaurer@users.sourceforge.net)
   7 //
   8 // Use like this:
   9 //   mono assembler.exe --ecmaspec DIRECTORY --out name
  10 //
  11
  12 using System;
  13 using System.Linq;
  14 using System.IO;
  15 using System.Text;
  16 using System.Xml;
  17 using System.Xml.Linq;
  18 using System.Collections.Generic;
  19
  20 using Lucene.Net.Index;
  21 using Lucene.Net.Documents;
  22
  23 using Monodoc.Ecma;
  24 using Mono.Utilities;
  25
  26 namespace Monodoc.Providers
  27 {
  28         public interface IEcmaProviderFileSource {
  29                 XmlReader GetIndexReader(string path);
  30                 XDocument GetTypeDocument(string path);
  31                 XElement GetNamespaceElement(string path);
  32                 string GetTypeXmlPath(string basePath, string nsName, string typeName);
  33                 string GetNamespaceXmlPath(string basePath, string ns);
  34                 XElement ExtractNamespaceSummary (string path);
  35         }
  36
  37         internal class DefaultEcmaProviderFileSource : IEcmaProviderFileSource {
  38                 public static readonly IEcmaProviderFileSource Default = new DefaultEcmaProviderFileSource();
  39
  40                 public XmlReader GetIndexReader(string path) {
  41                         return XmlReader.Create (File.OpenRead (path));
  42                 }
  43
  44                 public XElement GetNamespaceElement(string path) {
  45                         return XElement.Load (path);
  46                 }
  47
  48                 public string GetTypeXmlPath(string basePath, string nsName, string typeName) {
  49                         string finalPath = Path.Combine (basePath, nsName, Path.ChangeExtension (typeName, ".xml"));
  50                         return finalPath;
  51                 }
  52
  53                 public XDocument GetTypeDocument(string path) {
  54                         return XDocument.Load (path);
  55                 }
  56
  57                 public string GetNamespaceXmlPath(string basePath, string ns) {
  58                         string finalPath = Path.Combine(basePath, String.Format("ns-{0}.xml", ns));
  59                         return finalPath;
  60                 }
  61
  62                 public XElement ExtractNamespaceSummary (string path)
  63                 {
  64                         using (var reader = XmlReader.Create (path)) {
  65                                 reader.ReadToFollowing ("Namespace");
  66                                 var name = reader.GetAttribute ("Name");
  67                                 var summary = reader.ReadToFollowing ("summary") ? XElement.Load (reader.ReadSubtree ()) : new XElement ("summary");
  68                                 var remarks = reader.ReadToFollowing ("remarks") ? XElement.Load (reader.ReadSubtree ()) : new XElement ("remarks");
  69
  70                                 return new XElement ("namespace",
  71                                                      new XAttribute ("ns", name ?? string.Empty),
  72                                                      summary,
  73                                                      remarks);
  74                         }
  75                 }
  76         }
  77
  78         public class EcmaProvider : Provider
  79         {
  80                 HashSet<string> directories = new HashSet<string> ();
  81                 IEcmaProviderFileSource fileSource;
  82
  83                 public EcmaProvider ()
  84                 {
  85                 }
  86
  87                 public EcmaProvider (string baseDir)
  88                 {
  89                         AddDirectory (baseDir);
  90                 }
  91
  92                 public IEcmaProviderFileSource FileSource {
  93                         get {
  94                                 if (fileSource == null) {
  95                                         fileSource = new DefaultEcmaProviderFileSource();
  96                                 }
  97                                 return fileSource;
  98                         }
  99                         set { fileSource = value; }
 100                 }
 101
 102                 public void AddDirectory (string directory)
 103                 {
 104                         if (string.IsNullOrEmpty (directory))
 105                                 throw new ArgumentNullException ("directory");
 106
 107                         directories.Add (directory);
 108                 }
 109
 110                 public override void PopulateTree (Tree tree)
 111                 {
 112                         var storage = tree.HelpSource.Storage;
 113                         var nsSummaries = new Dictionary<string, XElement> ();
 114                         int resID = 0;
 115
 116                         foreach (var asm in directories) {
 117                                 var indexFilePath = Path.Combine (asm, "index.xml");
 118                                 if (!File.Exists (indexFilePath)) {
 119                                         Console.Error.WriteLine ("Warning: couldn't process directory `{0}' as it has no index.xml file", asm);
 120                                         continue;
 121                                 }
 122
 123                                 EcmaDoc.PopulateTreeFromIndexFile (indexFilePath, EcmaHelpSource.EcmaPrefix, tree, storage, nsSummaries, _ => resID++.ToString (), FileSource);
 124                         }
 125
 126                         foreach (var summary in nsSummaries)
 127                                 storage.Store ("xml.summary." + summary.Key, summary.Value.ToString ());
 128
 129                         var masterSummary = new XElement ("elements",
 130                                                           directories
 131                                                           .SelectMany (d => Directory.EnumerateFiles (d, "ns-*.xml"))
 132                                                           .Select (FileSource.ExtractNamespaceSummary));
 133                         storage.Store ("mastersummary.xml", masterSummary.ToString ());
 134                 }
 135
 136
 137
 138                 public override void CloseTree (HelpSource hs, Tree tree)
 139                 {
 140                         AddImages (hs);
 141                         AddExtensionMethods (hs);
 142                 }
 143
 144                 void AddEcmaXml (HelpSource hs)
 145                 {
 146                         var xmls = directories
 147                                 .SelectMany (Directory.EnumerateDirectories) // Assemblies
 148                                 .SelectMany (Directory.EnumerateDirectories) // Namespaces
 149                                 .SelectMany (Directory.EnumerateFiles)
 150                                 .Where (f => f.EndsWith (".xml")); // Type XML files
 151
 152                         int resID = 0;
 153                         foreach (var xml in xmls)
 154                                 using (var file = File.OpenRead (xml))
 155                                         hs.Storage.Store ((resID++).ToString (), file);
 156                 }
 157
 158                 void AddImages (HelpSource hs)
 159                 {
 160                         var imgs = directories
 161                                 .SelectMany (Directory.EnumerateDirectories)
 162                                 .Select (d => Path.Combine (d, "_images"))
 163                                 .Where (Directory.Exists)
 164                                 .SelectMany (Directory.EnumerateFiles);
 165
 166                         foreach (var img in imgs)
 167                                 using (var file = File.OpenRead (img))
 168                                         hs.Storage.Store (Path.GetFileName (img), file);
 169                 }
 170
 171                 void AddExtensionMethods (HelpSource hs)
 172                 {
 173                         var extensionMethods = directories
 174                                 .SelectMany (Directory.EnumerateDirectories)
 175                                 .Select (d => Path.Combine (d, "index.xml"))
 176                                 .Where (File.Exists)
 177                                 .Select (f => {
 178                                         using (var file = File.OpenRead (f)) {
 179                                                 var reader = XmlReader.Create (file);
 180                                                 reader.ReadToFollowing ("ExtensionMethods");
 181                                                 return reader.ReadInnerXml ();
 182                                         }
 183                             })
 184                             .DefaultIfEmpty (string.Empty);
 185
 186                         hs.Storage.Store ("ExtensionMethods.xml",
 187                                           "<ExtensionMethods>" + extensionMethods.Aggregate (string.Concat) + "</ExtensionMethods>");
 188                 }
 189
 190                 IEnumerable<string> GetEcmaXmls ()
 191                 {
 192                         return directories
 193                                 .SelectMany (Directory.EnumerateDirectories) // Assemblies
 194                                 .SelectMany (Directory.EnumerateDirectories) // Namespaces
 195                                 .SelectMany (Directory.EnumerateFiles)
 196                                 .Where (f => f.EndsWith (".xml")); // Type XML files
 197                 }
 198         }
 199
 200         public class EcmaHelpSource : HelpSource
 201         {
 202                 internal const string EcmaPrefix = "ecma:";
 203                 LRUCache<string, Node> cache = new LRUCache<string, Node> (4);
 204
 205                 public EcmaHelpSource (string base_file, bool create) : base (base_file, create)
 206                 {
 207                 }
 208
 209                 protected EcmaHelpSource () : base ()
 210                 {
 211                 }
 212
 213                 protected override string UriPrefix {
 214                         get {
 215                                 return EcmaPrefix;
 216                         }
 217                 }
 218
 219                 public override bool CanHandleUrl (string url)
 220                 {
 221                         if (url.Length > 2 && url[1] == ':') {
 222                                 switch (url[0]) {
 223                                 case 'T':
 224                                 case 'M':
 225                                 case 'C':
 226                                 case 'P':
 227                                 case 'E':
 228                                 case 'F':
 229                                 case 'N':
 230                                 case 'O':
 231                                         return true;
 232                                 }
 233                         }
 234                         return base.CanHandleUrl (url);
 235                 }
 236
 237                 // Clean the extra paramers in the id
 238                 public override Stream GetHelpStream (string id)
 239                 {
 240                         var idParts = id.Split ('?');
 241                         var name = idParts[0];
 242                         if (name == "root:")
 243                                 name = "mastersummary.xml";
 244                         return base.GetHelpStream (name);
 245                 }
 246
 247                 public override Stream GetCachedHelpStream (string id)
 248                 {
 249                         var idParts = id.Split ('?');
 250                         return base.GetCachedHelpStream (idParts[0]);
 251                 }
 252
 253                 public override DocumentType GetDocumentTypeForId (string id)
 254                 {
 255                         return DocumentType.EcmaXml;
 256                 }
 257
 258                 public override string GetPublicUrl (Node node)
 259                 {
 260                         string url = string.Empty;
 261                         var type = EcmaDoc.GetNodeType (node);
 262                         //Console.WriteLine ("GetPublicUrl {0} : {1} [{2}]", node.Element, node.Caption, type.ToString ());
 263                         switch (type) {
 264                         case EcmaNodeType.Namespace:
 265                                 return node.Element; // A namespace node has already a well formated internal url
 266                         case EcmaNodeType.Type:
 267                                 return MakeTypeNodeUrl (node);
 268                         case EcmaNodeType.Meta:
 269                                 return MakeTypeNodeUrl (GetNodeTypeParent (node)) + GenerateMetaSuffix (node);
 270                         case EcmaNodeType.Member:
 271                                 var typeChar = EcmaDoc.GetNodeMemberTypeChar (node);
 272                                 var parentNode = GetNodeTypeParent (node);
 273                                 var typeNode = MakeTypeNodeUrl (parentNode).Substring (2);
 274                                 return typeChar + ":" + typeNode + MakeMemberNodeUrl (typeChar, node);
 275                         default:
 276                                 return null;
 277                         }
 278                 }
 279
 280                 string MakeTypeNodeUrl (Node node)
 281                 {
 282                         // A Type node has a Element property of the form: 'ecma:{number}#{typename}/'
 283                         var hashIndex = node.Element.IndexOf ('#');
 284                         var typeName = node.Element.Substring (hashIndex + 1, node.Element.Length - hashIndex - 2);
 285                         return "T:" + node.Parent.Caption + '.' + typeName.Replace ('.', '+');
 286                 }
 287
 288                 string MakeMemberNodeUrl (char typeChar, Node node)
 289                 {
 290                         // We clean inner type ctor name which may contain the outer type name
 291                         var caption = node.Caption;
 292
 293                         // Sanitize constructor caption of inner types
 294                         if (typeChar == 'C') {
 295                                 int lastDot = -1;
 296                                 for (int i = 0; i < caption.Length && caption[i] != '('; i++)
 297                                         lastDot = caption[i] == '.' ? i : lastDot;
 298                                 return lastDot == -1 ? '.' + caption : caption.Substring (lastDot);
 299                         }
 300
 301                         /* We handle type conversion operator by checking if the name contains " to "
 302                          * (as in 'foo to bar') and we generate a corresponding conversion signature
 303                          */
 304                         if (typeChar == 'O' && caption.IndexOf (" to ") != -1) {
 305                                 var parts = caption.Split (' ');
 306                                 return "." + node.Parent.Caption + "(" + parts[0] + ", " + parts[2] + ")";
 307                         }
 308
 309                         /* The goal here is to treat method which are explicit interface definition
 310                          * such as 'void IDisposable.Dispose ()' for which the caption is a dot
 311                          * expression thus colliding with the ecma parser.
 312                          * If the first non-alpha character in the caption is a dot then we have an
 313                          * explicit member implementation (we assume the interface has namespace)
 314                          */
 315                         var firstNonAlpha = caption.FirstOrDefault (c => !char.IsLetterOrDigit (c));
 316                         if (firstNonAlpha == '.')
 317                                 return "$" + caption;
 318
 319                         return "." + caption;
 320                 }
 321
 322                 Node GetNodeTypeParent (Node node)
 323                 {
 324                         // Type nodes are always at level 2 so we just need to get there
 325                         while (node != null && node.Parent != null
 326                                && !node.Parent.Parent.Element.StartsWith ("root:/", StringComparison.OrdinalIgnoreCase) && node.Parent.Parent.Parent != null)
 327                                 node = node.Parent;
 328                         return node;
 329                 }
 330
 331                 string GenerateMetaSuffix (Node node)
 332                 {
 333                         string suffix = string.Empty;
 334                         // A meta node has always a type element to begin with
 335                         while (EcmaDoc.GetNodeType (node) != EcmaNodeType.Type) {
 336                                 suffix = '/' + node.Element + suffix;
 337                                 node = node.Parent;
 338                         }
 339                         return suffix;
 340                 }
 341
 342                 public override string GetInternalIdForUrl (string url, out Node node, out Dictionary<string, string> context)
 343                 {
 344                         var id = string.Empty;
 345                         node = null;
 346                         context = null;
 347
 348                         if (!url.StartsWith (UriPrefix, StringComparison.OrdinalIgnoreCase)) {
 349                                 node = MatchNode (url);
 350                                 if (node == null)
 351                                         return null;
 352                                 id = node.GetInternalUrl ();
 353                         }
 354
 355                         string hash;
 356                         id = GetInternalIdForInternalUrl (id, out hash);
 357                         context = EcmaDoc.GetContextForEcmaNode (hash, SourceID.ToString (), node);
 358
 359                         return id;
 360                 }
 361
 362                 public string GetInternalIdForInternalUrl (string internalUrl, out string hash)
 363                 {
 364                         var id = internalUrl;
 365                         if (id.StartsWith (UriPrefix, StringComparison.OrdinalIgnoreCase))
 366                                 id = id.Substring (UriPrefix.Length);
 367                         else if (id.StartsWith ("N:", StringComparison.OrdinalIgnoreCase))
 368                                 id = "xml.summary." + id.Substring ("N:".Length);
 369
 370                         var hashIndex = id.IndexOf ('#');
 371                         hash = string.Empty;
 372                         if (hashIndex != -1) {
 373                                 hash = id.Substring (hashIndex + 1);
 374                                 id = id.Substring (0, hashIndex);
 375                         }
 376
 377                         return id;
 378                 }
 379
 380                 public override Node MatchNode (string url)
 381                 {
 382                         Node node = null;
 383                         if ((node = cache.Get (url)) == null) {
 384                                 node = EcmaDoc.MatchNodeWithEcmaUrl (url, Tree);
 385                                 if (node != null)
 386                                         cache.Put (url, node);
 387                         }
 388                         return node;
 389                 }
 390
 391                 public override void PopulateIndex (IndexMaker index_maker)
 392                 {
 393                         foreach (Node ns_node in Tree.RootNode.ChildNodes){
 394                                 foreach (Node type_node in ns_node.ChildNodes){
 395                                         string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' '));
 396                                         string full = ns_node.Caption + "." + typename;
 397
 398                                         string doc_tag = GetKindFromCaption (type_node.Caption);
 399                                         string url = type_node.PublicUrl;
 400
 401                                         //
 402                                         // Add MonoMac/MonoTouch [Export] attributes, those live only in classes
 403                                         //
 404                                         XDocument type_doc = null;
 405                                         ILookup<string, XElement> prematchedMembers = null;
 406                                         bool hasExports = doc_tag == "Class" && (ns_node.Caption.StartsWith ("MonoTouch") || ns_node.Caption.StartsWith ("MonoMac"));
 407                                         if (hasExports) {
 408                                                 try {
 409                                                         string rest, hash;
 410                                                         var id = GetInternalIdForInternalUrl (type_node.GetInternalUrl (), out hash);
 411                                                         type_doc = XDocument.Load (GetHelpStream (id));
 412                                                         prematchedMembers = type_doc.Root.Element ("Members").Elements ("Member").ToLookup (n => (string)n.Attribute ("MemberName"), n => n);
 413                                                 } catch (Exception e) {
 414                                                         Console.WriteLine ("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e);
 415                                                         hasExports = false;
 416                                                 }
 417                                         }
 418
 419                                         if (doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"){
 420                                                 index_maker.Add (type_node.Caption, typename, url);
 421                                                 index_maker.Add (full + " " + doc_tag, full, url);
 422
 423                                                 foreach (Node c in type_node.ChildNodes){
 424                                                         switch (c.Caption){
 425                                                         case "Constructors":
 426                                                                 index_maker.Add ("  constructors", typename+"0", url + "/C");
 427                                                                 break;
 428                                                         case "Fields":
 429                                                                 index_maker.Add ("  fields", typename+"1", url + "/F");
 430                                                                 break;
 431                                                         case "Events":
 432                                                                 index_maker.Add ("  events", typename+"2", url + "/E");
 433                                                                 break;
 434                                                         case "Properties":
 435                                                                 index_maker.Add ("  properties", typename+"3", url + "/P");
 436                                                                 break;
 437                                                         case "Methods":
 438                                                                 index_maker.Add ("  methods", typename+"4", url + "/M");
 439                                                                 break;
 440                                                         case "Operators":
 441                                                                 index_maker.Add ("  operators", typename+"5", url + "/O");
 442                                                                 break;
 443                                                         }
 444                                                 }
 445
 446                                                 //
 447                                                 // Now repeat, but use a different sort key, to make sure we come after
 448                                                 // the summary data above, start the counter at 6
 449                                                 //
 450                                                 string keybase = typename + "6.";
 451
 452                                                 foreach (Node c in type_node.ChildNodes){
 453                                                         var type = c.Caption[0];
 454
 455                                                         foreach (Node nc in c.ChildNodes) {
 456                                                                 string res = nc.Caption;
 457                                                                 string nurl = nc.PublicUrl;
 458
 459                                                                 // Process exports
 460                                                                 if (hasExports && (type == 'C' || type == 'M' || type == 'P')) {
 461                                                                         try {
 462                                                                                 var member = GetMemberFromCaption (type_doc, type == 'C' ? ".ctor" : res, false, prematchedMembers);
 463                                                                                 var exports = member.Descendants ("AttributeName").Where (a => a.Value.Contains ("Foundation.Export"));
 464                                                                                 foreach (var exportNode in exports) {
 465                                                                                         var parts = exportNode.Value.Split ('"');
 466                                                                                         if (parts.Length != 3) {
 467                                                                                                 Console.WriteLine ("Export attribute not found or not usable in {0}", exportNode);
 468                                                                                         } else {
 469                                                                                                 var export = parts[1];
 470                                                                                                 index_maker.Add (export + " selector", export, nurl);
 471                                                                                         }
 472                                                                                 }
 473                                                                         } catch (Exception e) {
 474                                                                                 Console.WriteLine ("Problem processing {0}/{1} for MonoTouch/MonoMac exports\n\n{2}", nurl, res, e);
 475                                                                         }
 476                                                                 }
 477
 478                                                                 switch (type){
 479                                                                 case 'C':
 480                                                                         break;
 481                                                                 case 'F':
 482                                                                         index_maker.Add (String.Format ("{0}.{1} field", typename, res),
 483                                                                                          keybase + res, nurl);
 484                                                                         index_maker.Add (String.Format ("{0} field", res), res, nurl);
 485                                                                         break;
 486                                                                 case 'E':
 487                                                                         index_maker.Add (String.Format ("{0}.{1} event", typename, res),
 488                                                                                          keybase + res, nurl);
 489                                                                         index_maker.Add (String.Format ("{0} event", res), res, nurl);
 490                                                                         break;
 491                                                                 case 'P':
 492                                                                         index_maker.Add (String.Format ("{0}.{1} property", typename, res),
 493                                                                                          keybase + res, nurl);
 494                                                                         index_maker.Add (String.Format ("{0} property", res), res, nurl);
 495                                                                         break;
 496                                                                 case 'M':
 497                                                                         index_maker.Add (String.Format ("{0}.{1} method", typename, res),
 498                                                                                          keybase + res, nurl);
 499                                                                         index_maker.Add (String.Format ("{0} method", res), res, nurl);
 500                                                                         break;
 501                                                                 case 'O':
 502                                                                         index_maker.Add (String.Format ("{0}.{1} operator", typename, res),
 503                                                                                          keybase + res, nurl);
 504                                                                         break;
 505                                                                 }
 506                                                         }
 507                                                 }
 508                                         } else if (doc_tag == "Enumeration"){
 509                                                 //
 510                                                 // Enumerations: add the enumeration values
 511                                                 //
 512                                                 index_maker.Add (type_node.Caption, typename, url);
 513                                                 index_maker.Add (full + " " + doc_tag, full, url);
 514
 515                                                 // Now, pull the values.
 516                                                 string rest, hash;
 517                                                 var id = GetInternalIdForInternalUrl (type_node.GetInternalUrl (), out hash);
 518                                                 var xdoc = XDocument.Load (GetHelpStream (id));
 519                                                 if (xdoc == null)
 520                                                         continue;
 521
 522                                                 var members = xdoc.Root.Element ("Members").Elements ("Members");
 523                                                 if (members == null)
 524                                                         continue;
 525
 526                                                 foreach (var member_node in members){
 527                                                         string enum_value = member_node.Attribute ("MemberName").Value;
 528                                                         string caption = enum_value + " value";
 529                                                         index_maker.Add (caption, caption, url);
 530                                                 }
 531                                         } else if (doc_tag == "Delegate"){
 532                                                 index_maker.Add (type_node.Caption, typename, url);
 533                                                 index_maker.Add (full + " " + doc_tag, full, url);
 534                                         }
 535                                 }
 536                         }
 537                 }
 538
 539
 540                 public override void PopulateSearchableIndex (IndexWriter writer)
 541                 {
 542                         StringBuilder text = new StringBuilder ();
 543                         SearchableDocument searchDoc = new SearchableDocument ();
 544
 545                         foreach (Node ns_node in Tree.RootNode.ChildNodes) {
 546                                 foreach (Node type_node in ns_node.ChildNodes) {
 547                                         string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' '));
 548                                         string full = ns_node.Caption + "." + typename;
 549                                         string url = type_node.PublicUrl;
 550                                         string doc_tag = GetKindFromCaption (type_node.Caption);
 551                                         string rest, hash;
 552                                         var id = GetInternalIdForInternalUrl (type_node.GetInternalUrl (), out hash);
 553                                         var xdoc = XDocument.Load (GetHelpStream (id));
 554                                         if (xdoc == null)
 555                                                 continue;
 556                                         if (string.IsNullOrEmpty (doc_tag))
 557                                                 continue;
 558
 559                                         // For classes, structures or interfaces add a doc for the overview and
 560                                         // add a doc for every constructor, method, event, ...
 561                                         // doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"
 562                                         if (doc_tag[0] == 'C' || doc_tag[0] == 'S' || doc_tag[0] == 'I') {
 563                                                 // Adds a doc for every overview of every type
 564                                                 SearchableDocument doc = searchDoc.Reset ();
 565                                                 doc.Title = type_node.Caption;
 566                                                 doc.HotText = typename;
 567                                                 doc.Url = url;
 568                                                 doc.FullTitle = full;
 569
 570                                                 var node_sel = xdoc.Root.Element ("Docs");
 571                                                 text.Clear ();
 572                                                 GetTextFromNode (node_sel, text);
 573                                                 doc.Text = text.ToString ();
 574
 575                                                 text.Clear ();
 576                                                 GetExamples (node_sel, text);
 577                                                 doc.Examples = text.ToString ();
 578
 579                                                 writer.AddDocument (doc.LuceneDoc);
 580                                                 var exportParsable = doc_tag[0] == 'C' && (ns_node.Caption.StartsWith ("MonoTouch") || ns_node.Caption.StartsWith ("MonoMac"));
 581
 582                                                 //Add docs for contructors, methods, etc.
 583                                                 foreach (Node c in type_node.ChildNodes) { // c = Constructors || Fields || Events || Properties || Methods || Operators
 584                                                         if (c.Element == "*")
 585                                                                 continue;
 586                                                         const float innerTypeBoost = 0.2f;
 587
 588                                                         IEnumerable<Node> ncnodes = c.ChildNodes;
 589                                                         // The rationale is that we need to properly handle method overloads
 590                                                         // so for those method node which have children, flatten them
 591                                                         if (c.Caption == "Methods") {
 592                                                                 ncnodes = ncnodes
 593                                                                         .Where (n => n.ChildNodes == null || n.ChildNodes.Count == 0)
 594                                                                         .Concat (ncnodes.Where (n => n.ChildNodes.Count > 0).SelectMany (n => n.ChildNodes));
 595                                                         } else if (c.Caption == "Operators") {
 596                                                                 ncnodes = ncnodes
 597                                                                         .Where (n => !n.Caption.EndsWith ("Conversion"))
 598                                                                         .Concat (ncnodes.Where (n => n.Caption.EndsWith ("Conversion")).SelectMany (n => n.ChildNodes));
 599                                                         }
 600
 601                                                         var prematchedMembers = xdoc.Root.Element ("Members").Elements ("Member").ToLookup (n => (string)n.Attribute ("MemberName"), n => n);
 602
 603                                                         foreach (Node nc in ncnodes) {
 604                                                                 XElement docsNode = null;
 605                                                                 try {
 606                                                                         docsNode = GetDocsFromCaption (xdoc, c.Caption[0] == 'C' ? ".ctor" : nc.Caption, c.Caption[0] == 'O', prematchedMembers);
 607                                                                 } catch {}
 608                                                                 if (docsNode == null) {
 609                                                                         Console.Error.WriteLine ("Problem: {0}", nc.PublicUrl);
 610                                                                         continue;
 611                                                                 }
 612
 613                                                                 SearchableDocument doc_nod = searchDoc.Reset ();
 614                                                                 doc_nod.Title = LargeName (nc) + " " + EcmaDoc.EtcKindToCaption (c.Caption[0]);
 615                                                                 doc_nod.FullTitle = ns_node.Caption + '.' + typename + "::" + nc.Caption;
 616                                                                 doc_nod.HotText = string.Empty;
 617
 618                                                                 /* Disable constructors hottext indexing as it's often "polluting" search queries
 619                                                                    because it has the same hottext than standard types */
 620                                                                 if (c.Caption != "Constructors") {
 621                                                                         //dont add the parameters to the hottext
 622                                                                         int ppos = nc.Caption.IndexOf ('(');
 623                                                                         doc_nod.HotText = ppos != -1 ? nc.Caption.Substring (0, ppos) : nc.Caption;
 624                                                                 }
 625
 626                                                                 var urlnc = nc.PublicUrl;
 627                                                                 doc_nod.Url = urlnc;
 628
 629                                                                 text.Clear ();
 630                                                                 GetTextFromNode (docsNode, text);
 631                                                                 doc_nod.Text = text.ToString ();
 632
 633                                                                 text.Clear ();
 634                                                                 GetExamples (docsNode, text);
 635                                                                 doc_nod.Examples = text.ToString ();
 636
 637                                                                 Document lucene_doc = doc_nod.LuceneDoc;
 638                                                                 lucene_doc.Boost = innerTypeBoost;
 639                                                                 writer.AddDocument (lucene_doc);
 640
 641                                                                 // Objective-C binding specific parsing of [Export] attributes
 642                                                                 if (exportParsable) {
 643                                                                         try {
 644                                                                                 var exports = docsNode.Parent.Elements ("Attributes").Elements ("Attribute").Elements ("AttributeName")
 645                                                                                         .Select (a => (string)a).Where (txt => txt.Contains ("Foundation.Export"));
 646
 647                                                                                 foreach (var exportNode in exports) {
 648                                                                                         var parts = exportNode.Split ('"');
 649                                                                                         if (parts.Length != 3) {
 650                                                                                                 Console.WriteLine ("Export attribute not found or not usable in {0}", exportNode);
 651                                                                                                 continue;
 652                                                                                         }
 653
 654                                                                                         var export = parts[1];
 655                                                                                         var export_node = searchDoc.Reset ();
 656                                                                                         export_node.Title = export + " Export";
 657                                                                                         export_node.FullTitle = ns_node.Caption + '.' + typename + "::" + export;
 658                                                                                         export_node.Url = urlnc;
 659                                                                                         export_node.HotText = export;
 660                                                                                         export_node.Text = string.Empty;
 661                                                                                         export_node.Examples = string.Empty;
 662                                                                                         lucene_doc = export_node.LuceneDoc;
 663                                                                                         lucene_doc.Boost = innerTypeBoost;
 664                                                                                         writer.AddDocument (lucene_doc);
 665                                                                                 }
 666                                                                         } catch (Exception e){
 667                                                                                 Console.WriteLine ("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e);
 668                                                                         }
 669                                                                 }
 670                                                         }
 671                                                 }
 672                                         // doc_tag == "Enumeration"
 673                                         } else if (doc_tag[0] == 'E'){
 674                                                 var members = xdoc.Root.Element ("Members").Elements ("Member");
 675                                                 if (members == null)
 676                                                         continue;
 677
 678                                                 text.Clear ();
 679                                                 foreach (var member_node in members) {
 680                                                         string enum_value = (string)member_node.Attribute ("MemberName");
 681                                                         text.Append (enum_value);
 682                                                         text.Append (" ");
 683                                                         GetTextFromNode (member_node.Element ("Docs"), text);
 684                                                         text.AppendLine ();
 685                                                 }
 686
 687                                                 SearchableDocument doc = searchDoc.Reset ();
 688
 689                                                 text.Clear ();
 690                                                 GetExamples (xdoc.Root.Element ("Docs"), text);
 691                                                 doc.Examples = text.ToString ();
 692
 693                                                 doc.Title = type_node.Caption;
 694                                                 doc.HotText = (string)xdoc.Root.Attribute ("Name");
 695                                                 doc.FullTitle = full;
 696                                                 doc.Url = url;
 697                                                 doc.Text = text.ToString();
 698                                                 writer.AddDocument (doc.LuceneDoc);
 699                                         // doc_tag == "Delegate"
 700                                         } else if (doc_tag[0] == 'D'){
 701                                                 SearchableDocument doc = searchDoc.Reset ();
 702                                                 doc.Title = type_node.Caption;
 703                                                 doc.HotText = (string)xdoc.Root.Attribute ("Name");
 704                                                 doc.FullTitle = full;
 705                                                 doc.Url = url;
 706
 707                                                 var node_sel = xdoc.Root.Element ("Docs");
 708
 709                                                 text.Clear ();
 710                                                 GetTextFromNode (node_sel, text);
 711                                                 doc.Text = text.ToString();
 712
 713                                                 text.Clear ();
 714                                                 GetExamples (node_sel, text);
 715                                                 doc.Examples = text.ToString();
 716
 717                                                 writer.AddDocument (doc.LuceneDoc);
 718                                         }
 719                                 }
 720                         }
 721                 }
 722
 723                 string GetKindFromCaption (string s)
 724                 {
 725                         int p = s.LastIndexOf (' ');
 726                         if (p > 0)
 727                                 return s.Substring (p + 1);
 728                         return null;
 729                 }
 730
 731                 // Extract the interesting text from the docs node
 732                 void GetTextFromNode (XElement n, StringBuilder sb)
 733                 {
 734                         // Include the text content of the docs
 735                         sb.AppendLine (n.Value);
 736                         foreach (var tag in n.Descendants ())
 737                                 //include the url to which points the see tag and the name of the parameter
 738                                 if ((tag.Name.LocalName.Equals ("see", StringComparison.Ordinal) || tag.Name.LocalName.Equals ("paramref", StringComparison.Ordinal))
 739                                     && tag.HasAttributes)
 740                                         sb.AppendLine ((string)tag.Attributes ().First ());
 741                 }
 742
 743                 // Extract the code nodes from the docs
 744                 void GetExamples (XElement n, StringBuilder sb)
 745                 {
 746                         foreach (var code in n.Descendants ("code"))
 747                                 sb.Append ((string)code);
 748                 }
 749
 750                 // Extract a large name for the Node
 751                 static string LargeName (Node matched_node)
 752                 {
 753                         string[] parts = matched_node.GetInternalUrl ().Split('/', '#');
 754                         if (parts.Length == 3 && parts[2] != String.Empty) //List of Members, properties, events, ...
 755                                 return parts[1] + ": " + matched_node.Caption;
 756                         else if(parts.Length >= 4) //Showing a concrete Member, property, ...
 757                                 return parts[1] + "." + matched_node.Caption;
 758                         else
 759                                 return matched_node.Caption;
 760                 }
 761
 762                 XElement GetMemberFromCaption (XDocument xdoc, string caption, bool isOperator, ILookup<string, XElement> prematchedMembers)
 763                 {
 764                         string name;
 765                         IList<string> args;
 766                         var doc = xdoc.Root.Element ("Members").Elements ("Member");
 767
 768                         if (isOperator) {
 769                                 // The first case are explicit and implicit conversion operators which are grouped specifically
 770                                 if (caption.IndexOf (" to ") != -1) {
 771                                         var convArgs = caption.Split (new[] { " to " }, StringSplitOptions.None);
 772                                         return doc
 773                                                 .First (n => (AttrEq (n, "MemberName", "op_Explicit") || AttrEq (n, "MemberName", "op_Implicit"))
 774                                                         && ((string)n.Element ("ReturnValue").Element ("ReturnType")).Equals (convArgs[1], StringComparison.Ordinal)
 775                                                         && AttrEq (n.Element ("Parameters").Element ("Parameter"), "Type", convArgs[0]));
 776                                 } else {
 777                                         return doc.First (m => AttrEq (m, "MemberName", "op_" + caption));
 778                                 }
 779                         }
 780
 781                         TryParseCaption (caption, out name, out args);
 782
 783                         if (!string.IsNullOrEmpty (name)) { // Filter member by name
 784                                 var prematched = prematchedMembers[name];
 785                                 doc = prematched.Any () ? prematched : doc.Where (m => AttrEq (m, "MemberName", name));
 786                         }
 787                         if (args != null && args.Count > 0) // Filter member by its argument list
 788                                 doc = doc.Where (m => m.Element ("Parameters").Elements ("Parameter").Attributes ("Type").Select (a => (string)a).SequenceEqual (args));
 789
 790                         return doc.First ();
 791                 }
 792
 793                 XElement GetDocsFromCaption (XDocument xdoc, string caption, bool isOperator, ILookup<string, XElement> prematchedMembers)
 794                 {
 795                         return GetMemberFromCaption (xdoc, caption, isOperator, prematchedMembers).Element ("Docs");
 796                 }
 797
 798                 // A simple stack-based parser to detect single type definition separated by commas
 799                 IEnumerable<string> ExtractArguments (string rawArgList)
 800                 {
 801                         var sb = new System.Text.StringBuilder ();
 802                         int genericDepth = 0;
 803                         int arrayDepth = 0;
 804
 805                         for (int i = 0; i < rawArgList.Length; i++) {
 806                                 char c = rawArgList[i];
 807                                 switch (c) {
 808                                 case ',':
 809                                         if (genericDepth == 0 && arrayDepth == 0) {
 810                                                 yield return sb.ToString ();
 811                                                 sb.Clear ();
 812                                                 continue;
 813                                         }
 814                                         break;
 815                                 case '<':
 816                                         genericDepth++;
 817                                         break;
 818                                 case '>':
 819                                         genericDepth--;
 820                                         break;
 821                                 case '[':
 822                                         arrayDepth++;
 823                                         break;
 824                                 case ']':
 825                                         arrayDepth--;
 826                                         break;
 827                                 }
 828                                 sb.Append (c);
 829                         }
 830                         if (sb.Length > 0)
 831                                 yield return sb.ToString ();
 832                 }
 833
 834                 void TryParseCaption (string caption, out string name, out IList<string> argList)
 835                 {
 836                         name = null;
 837                         argList = null;
 838                         int parenIdx = caption.IndexOf ('(');
 839                         // In case of simple name, there is no need for processing
 840                         if (parenIdx == -1) {
 841                                 name = caption;
 842                                 return;
 843                         }
 844                         name = caption.Substring (0, parenIdx);
 845                         // Now we gather the argument list if there is any
 846                         var rawArgList = caption.Substring (parenIdx + 1, caption.Length - parenIdx - 2); // Only take what's inside the parens
 847                         if (string.IsNullOrEmpty (rawArgList))
 848                                 return;
 849
 850                         argList = ExtractArguments (rawArgList).Select (arg => arg.Trim ()).ToList ();
 851                 }
 852
 853                 bool AttrEq (XElement element, string attributeName, string expectedValue)
 854                 {
 855                         return ((string)element.Attribute (attributeName)).Equals (expectedValue, StringComparison.Ordinal);
 856                 }
 857         }
 858 }