mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20 using System.Collections;
  21
  22 namespace Mono.CSharp
  23 {
  24         //
  25         // This class has to be used by parser only, it reuses token
  26         // details once a file is parsed
  27         //
  28         public class LocatedToken
  29         {
  30                 public int row, column;
  31                 public string value;
  32                 public SourceFile file;
  33
  34                 public LocatedToken ()
  35                 {
  36                 }
  37
  38                 public LocatedToken (string value, Location loc)
  39                 {
  40                         this.value = value;
  41                         file = loc.SourceFile;
  42                         row = loc.Row;
  43                         column = loc.Column;
  44                 }
  45
  46                 public override string ToString ()
  47                 {
  48                         return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  49                 }
  50
  51                 public Location Location
  52                 {
  53                         get { return new Location (file, row, column); }
  54                 }
  55
  56                 public string Value
  57                 {
  58                         get { return value; }
  59                 }
  60         }
  61
  62         /// <summary>
  63         ///    Tokenizer for C# source code.
  64         /// </summary>
  65         public class Tokenizer : yyParser.yyInput
  66         {
  67                 class KeywordEntry<T>
  68                 {
  69                         public readonly T Token;
  70                         public KeywordEntry<T> Next;
  71                         public readonly char[] Value;
  72
  73                         public KeywordEntry (string value, T token)
  74                         {
  75                                 this.Value = value.ToCharArray ();
  76                                 this.Token = token;
  77                         }
  78                 }
  79
  80                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  81                 {
  82                         readonly int length;
  83
  84                         public IdentifiersComparer (int length)
  85                         {
  86                                 this.length = length;
  87                         }
  88
  89                         public bool Equals (char[] x, char[] y)
  90                         {
  91                                 for (int i = 0; i < length; ++i)
  92                                         if (x [i] != y [i])
  93                                                 return false;
  94
  95                                 return true;
  96                         }
  97
  98                         public int GetHashCode (char[] obj)
  99                         {
 100                                 int h = 0;
 101                                 for (int i = 0; i < length; ++i)
 102                                         h = (h << 5) - h + obj [i];
 103
 104                                 return h;
 105                         }
 106                 }
 107
 108                 public class LocatedTokenBuffer
 109                 {
 110                         readonly LocatedToken[] buffer;
 111                         public int pos;
 112
 113                         public LocatedTokenBuffer ()
 114                         {
 115                                 buffer = new LocatedToken[0];
 116                         }
 117
 118                         public LocatedTokenBuffer (LocatedToken[] buffer)
 119                         {
 120                                 this.buffer = buffer ?? new LocatedToken[0];
 121                         }
 122
 123                         public LocatedToken Create (SourceFile file, int row, int column)
 124                         {
 125                                 return Create (null, file, row, column);
 126                         }
 127
 128                         public LocatedToken Create (string value, SourceFile file, int row, int column)
 129                         {
 130                                 //
 131                                 // TODO: I am not very happy about the logic but it's the best
 132                                 // what I could come up with for now.
 133                                 // Ideally we should be using just tiny buffer (256 elements) which
 134                                 // is enough to hold all details for currect stack and recycle elements
 135                                 // poped from the stack but there is a trick needed to recycle
 136                                 // them properly.
 137                                 //
 138                                 LocatedToken entry;
 139                                 if (pos >= buffer.Length) {
 140                                         entry = new LocatedToken ();
 141                                 } else {
 142                                         entry = buffer[pos];
 143                                         if (entry == null) {
 144                                                 entry = new LocatedToken ();
 145                                                 buffer[pos] = entry;
 146                                         }
 147
 148                                         ++pos;
 149                                 }
 150                                 entry.value = value;
 151                                 entry.file = file;
 152                                 entry.row = row;
 153                                 entry.column = column;
 154                                 return entry;
 155                         }
 156
 157                         //
 158                         // Used for token not required by expression evaluator
 159                         //
 160                         [Conditional ("FULL_AST")]
 161                         public void CreateOptional (SourceFile file, int row, int col, ref object token)
 162                         {
 163                                 token = Create (file, row, col);
 164                         }
 165                 }
 166
 167                 public enum PreprocessorDirective
 168                 {
 169                         Invalid = 0,
 170
 171                         Region = 1,
 172                         Endregion = 2,
 173                         If = 3 | RequiresArgument,
 174                         Endif = 4,
 175                         Elif = 5 | RequiresArgument,
 176                         Else = 6,
 177                         Define = 7 | RequiresArgument,
 178                         Undef = 8 | RequiresArgument,
 179                         Error = 9,
 180                         Warning = 10,
 181                         Pragma = 11 | CustomArgumentsParsing,
 182                         Line = 12 | CustomArgumentsParsing,
 183
 184                         CustomArgumentsParsing = 1 << 10,
 185                         RequiresArgument = 1 << 11
 186                 }
 187
 188                 readonly SeekableStreamReader reader;
 189                 readonly CompilationSourceFile source_file;
 190                 readonly CompilerContext context;
 191                 readonly Report Report;
 192
 193
 194                 SourceFile current_source;
 195                 Location hidden_block_start;
 196                 int ref_line = 1;
 197                 int line = 1;
 198                 int col = 0;
 199                 int previous_col;
 200                 int current_token;
 201                 readonly int tab_size;
 202                 bool handle_get_set = false;
 203                 bool handle_remove_add = false;
 204                 bool handle_where;
 205                 bool handle_typeof = false;
 206                 bool lambda_arguments_parsing;
 207                 List<Location> escaped_identifiers;
 208                 int parsing_generic_less_than;
 209                 readonly bool doc_processing;
 210                 readonly LocatedTokenBuffer ltb;
 211
 212                 //
 213                 // Used mainly for parser optimizations. Some expressions for instance
 214                 // can appear only in block (including initializer, base initializer)
 215                 // scope only
 216                 //
 217                 public int parsing_block;
 218                 internal bool query_parsing;
 219
 220                 //
 221                 // When parsing type only, useful for ambiguous nullable types
 222                 //
 223                 public int parsing_type;
 224
 225                 //
 226                 // Set when parsing generic declaration (type or method header)
 227                 //
 228                 public bool parsing_generic_declaration;
 229                 public bool parsing_generic_declaration_doc;
 230
 231                 //
 232                 // The value indicates that we have not reach any declaration or
 233                 // namespace yet
 234                 //
 235                 public int parsing_declaration;
 236
 237                 public bool parsing_attribute_section;
 238
 239                 public bool parsing_modifiers;
 240
 241                 //
 242                 // The special characters to inject on streams to run the unit parser
 243                 // in the special expression mode. Using private characters from
 244                 // Plane Sixteen (U+100000 to U+10FFFD)
 245                 //
 246                 // This character is only tested just before the tokenizer is about to report
 247                 // an error;   So on the regular operation mode, this addition will have no
 248                 // impact on the tokenizer's performance.
 249                 //
 250
 251                 public const int EvalStatementParserCharacter = 0x100000;
 252                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 253                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 254                 public const int DocumentationXref = 0x100003;
 255
 256                 const int UnicodeLS = 0x2028;
 257                 const int UnicodePS = 0x2029;
 258
 259                 //
 260                 // XML documentation buffer. The save point is used to divide
 261                 // comments on types and comments on members.
 262                 //
 263                 StringBuilder xml_comment_buffer;
 264
 265                 //
 266                 // See comment on XmlCommentState enumeration.
 267                 //
 268                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 269
 270                 //
 271                 // Whether tokens have been seen on this line
 272                 //
 273                 bool tokens_seen = false;
 274
 275                 //
 276                 // Set to true once the GENERATE_COMPLETION token has bee
 277                 // returned.   This helps produce one GENERATE_COMPLETION,
 278                 // as many COMPLETE_COMPLETION as necessary to complete the
 279                 // AST tree and one final EOF.
 280                 //
 281                 bool generated;
 282
 283                 //
 284                 // Whether a token has been seen on the file
 285                 // This is needed because `define' is not allowed to be used
 286                 // after a token has been seen.
 287                 //
 288                 bool any_token_seen;
 289
 290                 //
 291                 // Class variables
 292                 //
 293                 static readonly KeywordEntry<int>[][] keywords;
 294                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 295                 static readonly HashSet<string> keyword_strings;
 296                 static readonly NumberStyles styles;
 297                 static readonly NumberFormatInfo csharp_format_info;
 298
 299                 // Pragma arguments
 300                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 301                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 302                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 303                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 304                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 305                 static readonly char[] line_default = "default".ToCharArray ();
 306
 307                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 308
 309                 public bool PropertyParsing {
 310                         get { return handle_get_set; }
 311                         set { handle_get_set = value; }
 312                 }
 313
 314                 public bool EventParsing {
 315                         get { return handle_remove_add; }
 316                         set { handle_remove_add = value; }
 317                 }
 318
 319                 public bool ConstraintsParsing {
 320                         get { return handle_where; }
 321                         set { handle_where = value; }
 322                 }
 323
 324                 public bool TypeOfParsing {
 325                         get { return handle_typeof; }
 326                         set { handle_typeof = value; }
 327                 }
 328
 329                 public XmlCommentState doc_state {
 330                         get { return xml_doc_state; }
 331                         set {
 332                                 if (value == XmlCommentState.Allowed) {
 333                                         check_incorrect_doc_comment ();
 334                                         reset_doc_comment ();
 335                                 }
 336                                 xml_doc_state = value;
 337                         }
 338                 }
 339
 340                 //
 341                 // This is used to trigger completion generation on the parser
 342                 public bool CompleteOnEOF;
 343
 344                 void AddEscapedIdentifier (Location loc)
 345                 {
 346                         if (escaped_identifiers == null)
 347                                 escaped_identifiers = new List<Location> ();
 348
 349                         escaped_identifiers.Add (loc);
 350                 }
 351
 352                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 353                 {
 354                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 355                 }
 356
 357                 //
 358                 // Values for the associated token returned
 359                 //
 360                 internal int putback_char;      // Used by repl only
 361                 object val;
 362
 363                 //
 364                 // Pre-processor
 365                 //
 366                 const int TAKING        = 1;
 367                 const int ELSE_SEEN     = 4;
 368                 const int PARENT_TAKING = 8;
 369                 const int REGION        = 16;
 370
 371                 //
 372                 // pre-processor if stack state:
 373                 //
 374                 Stack<int> ifstack;
 375
 376                 public const int MaxIdentifierLength = 512;
 377                 public const int MaxNumberLength = 512;
 378
 379                 readonly char[] id_builder;
 380                 readonly Dictionary<char[], string>[] identifiers;
 381                 readonly char[] number_builder;
 382                 int number_pos;
 383
 384                 char[] value_builder = new char[64];
 385
 386                 public int Line {
 387                         get {
 388                                 return ref_line;
 389                         }
 390                 }
 391
 392                 //
 393                 // This is used when the tokenizer needs to save
 394                 // the current position as it needs to do some parsing
 395                 // on its own to deamiguate a token in behalf of the
 396                 // parser.
 397                 //
 398                 Stack<Position> position_stack = new Stack<Position> (2);
 399
 400                 class Position {
 401                         public int position;
 402                         public int line;
 403                         public int ref_line;
 404                         public int col;
 405                         public Location hidden;
 406                         public int putback_char;
 407                         public int previous_col;
 408                         public Stack<int> ifstack;
 409                         public int parsing_generic_less_than;
 410                         public int current_token;
 411                         public object val;
 412
 413                         public Position (Tokenizer t)
 414                         {
 415                                 position = t.reader.Position;
 416                                 line = t.line;
 417                                 ref_line = t.ref_line;
 418                                 col = t.col;
 419                                 hidden = t.hidden_block_start;
 420                                 putback_char = t.putback_char;
 421                                 previous_col = t.previous_col;
 422                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 423                                         // There is no simple way to clone Stack<T> all
 424                                         // methods reverse the order
 425                                         var clone = t.ifstack.ToArray ();
 426                                         Array.Reverse (clone);
 427                                         ifstack = new Stack<int> (clone);
 428                                 }
 429                                 parsing_generic_less_than = t.parsing_generic_less_than;
 430                                 current_token = t.current_token;
 431                                 val = t.val;
 432                         }
 433                 }
 434
 435                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 436                 {
 437                         this.source_file = file;
 438                         this.context = file.Compiler;
 439                         this.current_source = file.SourceFile;
 440                         this.identifiers = session.Identifiers;
 441                         this.id_builder = session.IDBuilder;
 442                         this.number_builder = session.NumberBuilder;
 443                         this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 444                         this.Report = report;
 445
 446                         reader = input;
 447
 448                         putback_char = -1;
 449
 450                         xml_comment_buffer = new StringBuilder ();
 451                         doc_processing = context.Settings.DocumentationFile != null;
 452
 453                         tab_size = context.Settings.TabSize;
 454                 }
 455
 456                 public void PushPosition ()
 457                 {
 458                         position_stack.Push (new Position (this));
 459                 }
 460
 461                 public void PopPosition ()
 462                 {
 463                         Position p = position_stack.Pop ();
 464
 465                         reader.Position = p.position;
 466                         ref_line = p.ref_line;
 467                         line = p.line;
 468                         col = p.col;
 469                         hidden_block_start = p.hidden;
 470                         putback_char = p.putback_char;
 471                         previous_col = p.previous_col;
 472                         ifstack = p.ifstack;
 473                         parsing_generic_less_than = p.parsing_generic_less_than;
 474                         current_token = p.current_token;
 475                         val = p.val;
 476                 }
 477
 478                 // Do not reset the position, ignore it.
 479                 public void DiscardPosition ()
 480                 {
 481                         position_stack.Pop ();
 482                 }
 483
 484                 static void AddKeyword (string kw, int token)
 485                 {
 486                         keyword_strings.Add (kw);
 487
 488                         AddKeyword (keywords, kw, token);
 489                 }
 490
 491                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 492                 {
 493                         AddKeyword (keywords_preprocessor, kw, directive);
 494                 }
 495
 496                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 497                 {
 498                         int length = kw.Length;
 499                         if (keywords[length] == null) {
 500                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 501                         }
 502
 503                         int char_index = kw[0] - '_';
 504                         var kwe = keywords[length][char_index];
 505                         if (kwe == null) {
 506                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 507                                 return;
 508                         }
 509
 510                         while (kwe.Next != null) {
 511                                 kwe = kwe.Next;
 512                         }
 513
 514                         kwe.Next = new KeywordEntry<T> (kw, token);
 515                 }
 516
 517                 //
 518                 // Class initializer
 519                 //
 520                 static Tokenizer ()
 521                 {
 522                         keyword_strings = new HashSet<string> ();
 523
 524                         // 11 is the length of the longest keyword for now
 525                         keywords = new KeywordEntry<int>[11][];
 526
 527                         AddKeyword ("__arglist", Token.ARGLIST);
 528                         AddKeyword ("__makeref", Token.MAKEREF);
 529                         AddKeyword ("__reftype", Token.REFTYPE);
 530                         AddKeyword ("__refvalue", Token.REFVALUE);
 531                         AddKeyword ("abstract", Token.ABSTRACT);
 532                         AddKeyword ("as", Token.AS);
 533                         AddKeyword ("add", Token.ADD);
 534                         AddKeyword ("base", Token.BASE);
 535                         AddKeyword ("bool", Token.BOOL);
 536                         AddKeyword ("break", Token.BREAK);
 537                         AddKeyword ("byte", Token.BYTE);
 538                         AddKeyword ("case", Token.CASE);
 539                         AddKeyword ("catch", Token.CATCH);
 540                         AddKeyword ("char", Token.CHAR);
 541                         AddKeyword ("checked", Token.CHECKED);
 542                         AddKeyword ("class", Token.CLASS);
 543                         AddKeyword ("const", Token.CONST);
 544                         AddKeyword ("continue", Token.CONTINUE);
 545                         AddKeyword ("decimal", Token.DECIMAL);
 546                         AddKeyword ("default", Token.DEFAULT);
 547                         AddKeyword ("delegate", Token.DELEGATE);
 548                         AddKeyword ("do", Token.DO);
 549                         AddKeyword ("double", Token.DOUBLE);
 550                         AddKeyword ("else", Token.ELSE);
 551                         AddKeyword ("enum", Token.ENUM);
 552                         AddKeyword ("event", Token.EVENT);
 553                         AddKeyword ("explicit", Token.EXPLICIT);
 554                         AddKeyword ("extern", Token.EXTERN);
 555                         AddKeyword ("false", Token.FALSE);
 556                         AddKeyword ("finally", Token.FINALLY);
 557                         AddKeyword ("fixed", Token.FIXED);
 558                         AddKeyword ("float", Token.FLOAT);
 559                         AddKeyword ("for", Token.FOR);
 560                         AddKeyword ("foreach", Token.FOREACH);
 561                         AddKeyword ("goto", Token.GOTO);
 562                         AddKeyword ("get", Token.GET);
 563                         AddKeyword ("if", Token.IF);
 564                         AddKeyword ("implicit", Token.IMPLICIT);
 565                         AddKeyword ("in", Token.IN);
 566                         AddKeyword ("int", Token.INT);
 567                         AddKeyword ("interface", Token.INTERFACE);
 568                         AddKeyword ("internal", Token.INTERNAL);
 569                         AddKeyword ("is", Token.IS);
 570                         AddKeyword ("lock", Token.LOCK);
 571                         AddKeyword ("long", Token.LONG);
 572                         AddKeyword ("namespace", Token.NAMESPACE);
 573                         AddKeyword ("new", Token.NEW);
 574                         AddKeyword ("null", Token.NULL);
 575                         AddKeyword ("object", Token.OBJECT);
 576                         AddKeyword ("operator", Token.OPERATOR);
 577                         AddKeyword ("out", Token.OUT);
 578                         AddKeyword ("override", Token.OVERRIDE);
 579                         AddKeyword ("params", Token.PARAMS);
 580                         AddKeyword ("private", Token.PRIVATE);
 581                         AddKeyword ("protected", Token.PROTECTED);
 582                         AddKeyword ("public", Token.PUBLIC);
 583                         AddKeyword ("readonly", Token.READONLY);
 584                         AddKeyword ("ref", Token.REF);
 585                         AddKeyword ("remove", Token.REMOVE);
 586                         AddKeyword ("return", Token.RETURN);
 587                         AddKeyword ("sbyte", Token.SBYTE);
 588                         AddKeyword ("sealed", Token.SEALED);
 589                         AddKeyword ("set", Token.SET);
 590                         AddKeyword ("short", Token.SHORT);
 591                         AddKeyword ("sizeof", Token.SIZEOF);
 592                         AddKeyword ("stackalloc", Token.STACKALLOC);
 593                         AddKeyword ("static", Token.STATIC);
 594                         AddKeyword ("string", Token.STRING);
 595                         AddKeyword ("struct", Token.STRUCT);
 596                         AddKeyword ("switch", Token.SWITCH);
 597                         AddKeyword ("this", Token.THIS);
 598                         AddKeyword ("throw", Token.THROW);
 599                         AddKeyword ("true", Token.TRUE);
 600                         AddKeyword ("try", Token.TRY);
 601                         AddKeyword ("typeof", Token.TYPEOF);
 602                         AddKeyword ("uint", Token.UINT);
 603                         AddKeyword ("ulong", Token.ULONG);
 604                         AddKeyword ("unchecked", Token.UNCHECKED);
 605                         AddKeyword ("unsafe", Token.UNSAFE);
 606                         AddKeyword ("ushort", Token.USHORT);
 607                         AddKeyword ("using", Token.USING);
 608                         AddKeyword ("virtual", Token.VIRTUAL);
 609                         AddKeyword ("void", Token.VOID);
 610                         AddKeyword ("volatile", Token.VOLATILE);
 611                         AddKeyword ("while", Token.WHILE);
 612                         AddKeyword ("partial", Token.PARTIAL);
 613                         AddKeyword ("where", Token.WHERE);
 614
 615                         // LINQ keywords
 616                         AddKeyword ("from", Token.FROM);
 617                         AddKeyword ("join", Token.JOIN);
 618                         AddKeyword ("on", Token.ON);
 619                         AddKeyword ("equals", Token.EQUALS);
 620                         AddKeyword ("select", Token.SELECT);
 621                         AddKeyword ("group", Token.GROUP);
 622                         AddKeyword ("by", Token.BY);
 623                         AddKeyword ("let", Token.LET);
 624                         AddKeyword ("orderby", Token.ORDERBY);
 625                         AddKeyword ("ascending", Token.ASCENDING);
 626                         AddKeyword ("descending", Token.DESCENDING);
 627                         AddKeyword ("into", Token.INTO);
 628
 629                         // Contextual async keywords
 630                         AddKeyword ("async", Token.ASYNC);
 631                         AddKeyword ("await", Token.AWAIT);
 632
 633                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 634
 635                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 636                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 637                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 638                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 639                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 640                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 641                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 642                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 643                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 644                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 645                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 646                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 647
 648                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 649                         styles = NumberStyles.Float;
 650                 }
 651
 652                 int GetKeyword (char[] id, int id_len)
 653                 {
 654                         //
 655                         // Keywords are stored in an array of arrays grouped by their
 656                         // length and then by the first character
 657                         //
 658                         if (id_len >= keywords.Length || keywords [id_len] == null)
 659                                 return -1;
 660
 661                         int first_index = id [0] - '_';
 662                         if (first_index > 'z' - '_')
 663                                 return -1;
 664
 665                         var kwe = keywords [id_len] [first_index];
 666                         if (kwe == null)
 667                                 return -1;
 668
 669                         int res;
 670                         do {
 671                                 res = kwe.Token;
 672                                 for (int i = 1; i < id_len; ++i) {
 673                                         if (id [i] != kwe.Value [i]) {
 674                                                 res = 0;
 675                                                 kwe = kwe.Next;
 676                                                 break;
 677                                         }
 678                                 }
 679                         } while (res == 0 && kwe != null);
 680
 681                         if (res == 0)
 682                                 return -1;
 683
 684                         int next_token;
 685                         switch (res) {
 686                         case Token.GET:
 687                         case Token.SET:
 688                                 if (!handle_get_set)
 689                                         res = -1;
 690                                 break;
 691                         case Token.REMOVE:
 692                         case Token.ADD:
 693                                 if (!handle_remove_add)
 694                                         res = -1;
 695                                 break;
 696                         case Token.EXTERN:
 697                                 if (parsing_declaration == 0)
 698                                         res = Token.EXTERN_ALIAS;
 699                                 break;
 700                         case Token.DEFAULT:
 701                                 if (peek_token () == Token.COLON) {
 702                                         token ();
 703                                         res = Token.DEFAULT_COLON;
 704                                 }
 705                                 break;
 706                         case Token.WHERE:
 707                                 if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 708                                         res = -1;
 709                                 break;
 710                         case Token.FROM:
 711                                 //
 712                                 // A query expression is any expression that starts with `from identifier'
 713                                 // followed by any token except ; , =
 714                                 //
 715                                 if (!query_parsing) {
 716                                         if (lambda_arguments_parsing || parsing_block == 0) {
 717                                                 res = -1;
 718                                                 break;
 719                                         }
 720
 721                                         PushPosition ();
 722                                         // HACK: to disable generics micro-parser, because PushPosition does not
 723                                         // store identifiers array
 724                                         parsing_generic_less_than = 1;
 725                                         switch (xtoken ()) {
 726                                         case Token.IDENTIFIER:
 727                                         case Token.INT:
 728                                         case Token.BOOL:
 729                                         case Token.BYTE:
 730                                         case Token.CHAR:
 731                                         case Token.DECIMAL:
 732                                         case Token.FLOAT:
 733                                         case Token.LONG:
 734                                         case Token.OBJECT:
 735                                         case Token.STRING:
 736                                         case Token.UINT:
 737                                         case Token.ULONG:
 738                                                 next_token = xtoken ();
 739                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 740                                                         goto default;
 741
 742                                                 res = Token.FROM_FIRST;
 743                                                 query_parsing = true;
 744                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 745                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 746                                                 break;
 747                                         case Token.VOID:
 748                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 749                                                 break;
 750                                         default:
 751                                                 PopPosition ();
 752                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 753                                                 // which has been overwritten before we grabbed the identifier
 754                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 755                                                 return -1;
 756                                         }
 757                                         PopPosition ();
 758                                 }
 759                                 break;
 760                         case Token.JOIN:
 761                         case Token.ON:
 762                         case Token.EQUALS:
 763                         case Token.SELECT:
 764                         case Token.GROUP:
 765                         case Token.BY:
 766                         case Token.LET:
 767                         case Token.ORDERBY:
 768                         case Token.ASCENDING:
 769                         case Token.DESCENDING:
 770                         case Token.INTO:
 771                                 if (!query_parsing)
 772                                         res = -1;
 773                                 break;
 774
 775                         case Token.USING:
 776                         case Token.NAMESPACE:
 777                                 // TODO: some explanation needed
 778                                 check_incorrect_doc_comment ();
 779                                 parsing_modifiers = false;
 780                                 break;
 781
 782                         case Token.PARTIAL:
 783                                 if (parsing_block > 0) {
 784                                         res = -1;
 785                                         break;
 786                                 }
 787
 788                                 // Save current position and parse next token.
 789                                 PushPosition ();
 790
 791                                 next_token = token ();
 792                                 bool ok = (next_token == Token.CLASS) ||
 793                                         (next_token == Token.STRUCT) ||
 794                                         (next_token == Token.INTERFACE) ||
 795                                         (next_token == Token.VOID);
 796
 797                                 PopPosition ();
 798
 799                                 if (ok) {
 800                                         if (next_token == Token.VOID) {
 801                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 802                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 803                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 804                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 805
 806                                         return res;
 807                                 }
 808
 809                                 if (next_token < Token.LAST_KEYWORD) {
 810                                         Report.Error (267, Location,
 811                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 812                                         return token ();
 813                                 }
 814
 815                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 816                                 // which has been overwritten before we grabbed the identifier
 817                                 id_builder[0] = 'p';
 818                                 id_builder[1] = 'a';
 819                                 id_builder[2] = 'r';
 820                                 id_builder[3] = 't';
 821                                 id_builder[4] = 'i';
 822                                 id_builder[5] = 'a';
 823                                 id_builder[6] = 'l';
 824                                 res = -1;
 825                                 break;
 826
 827                         case Token.ASYNC:
 828                                 if (parsing_modifiers) {
 829                                         //
 830                                         // Skip attributes section or constructor called async
 831                                         //
 832                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 833                                                 res = -1;
 834                                         } else {
 835                                                 // async is keyword
 836                                         }
 837                                 } else if (parsing_block > 0) {
 838                                         switch (peek_token ()) {
 839                                         case Token.DELEGATE:
 840                                         case Token.OPEN_PARENS_LAMBDA:
 841                                                 // async is keyword
 842                                                 break;
 843                                         case Token.IDENTIFIER:
 844                                                 PushPosition ();
 845                                                 xtoken ();
 846                                                 if (xtoken () != Token.ARROW) {
 847                                                         PopPosition ();
 848                                                         goto default;
 849                                                 }
 850
 851                                                 PopPosition ();
 852                                                 break;
 853                                         default:
 854                                                 // peek_token could overwrite id_buffer
 855                                                 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 856                                                 res = -1;
 857                                                 break;
 858                                         }
 859                                 } else {
 860                                         res = -1;
 861                                 }
 862
 863                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 864                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 865                                 }
 866
 867                                 break;
 868
 869                         case Token.AWAIT:
 870                                 if (parsing_block == 0)
 871                                         res = -1;
 872
 873                                 break;
 874                         }
 875
 876
 877                         return res;
 878                 }
 879
 880                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 881                 {
 882                         //
 883                         // Keywords are stored in an array of arrays grouped by their
 884                         // length and then by the first character
 885                         //
 886                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 887                                 return PreprocessorDirective.Invalid;
 888
 889                         int first_index = id[0] - '_';
 890                         if (first_index > 'z' - '_')
 891                                 return PreprocessorDirective.Invalid;
 892
 893                         var kwe = keywords_preprocessor[id_len][first_index];
 894                         if (kwe == null)
 895                                 return PreprocessorDirective.Invalid;
 896
 897                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 898                         do {
 899                                 res = kwe.Token;
 900                                 for (int i = 1; i < id_len; ++i) {
 901                                         if (id[i] != kwe.Value[i]) {
 902                                                 res = 0;
 903                                                 kwe = kwe.Next;
 904                                                 break;
 905                                         }
 906                                 }
 907                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 908
 909                         return res;
 910                 }
 911
 912                 public Location Location {
 913                         get {
 914                                 return new Location (current_source, ref_line, col);
 915                         }
 916                 }
 917
 918                 static bool is_identifier_start_character (int c)
 919                 {
 920                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 921                 }
 922
 923                 static bool is_identifier_part_character (char c)
 924                 {
 925                         if (c >= 'a' && c <= 'z')
 926                                 return true;
 927
 928                         if (c >= 'A' && c <= 'Z')
 929                                 return true;
 930
 931                         if (c == '_' || (c >= '0' && c <= '9'))
 932                                 return true;
 933
 934                         if (c < 0x80)
 935                                 return false;
 936
 937                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 938                 }
 939
 940                 public static bool IsKeyword (string s)
 941                 {
 942                         return keyword_strings.Contains (s);
 943                 }
 944
 945                 //
 946                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 947                 //
 948                 int TokenizeOpenParens ()
 949                 {
 950                         int ptoken;
 951                         current_token = -1;
 952
 953                         int bracket_level = 0;
 954                         bool is_type = false;
 955                         bool can_be_type = false;
 956
 957                         while (true) {
 958                                 ptoken = current_token;
 959                                 token ();
 960
 961                                 switch (current_token) {
 962                                 case Token.CLOSE_PARENS:
 963                                         token ();
 964
 965                                         //
 966                                         // Expression inside parens is lambda, (int i) =>
 967                                         //
 968                                         if (current_token == Token.ARROW)
 969                                                 return Token.OPEN_PARENS_LAMBDA;
 970
 971                                         //
 972                                         // Expression inside parens is single type, (int[])
 973                                         //
 974                                         if (is_type) {
 975                                                 if (current_token == Token.SEMICOLON)
 976                                                         return Token.OPEN_PARENS;
 977
 978                                                 return Token.OPEN_PARENS_CAST;
 979                                         }
 980
 981                                         //
 982                                         // Expression is possible cast, look at next token, (T)null
 983                                         //
 984                                         if (can_be_type) {
 985                                                 switch (current_token) {
 986                                                 case Token.OPEN_PARENS:
 987                                                 case Token.BANG:
 988                                                 case Token.TILDE:
 989                                                 case Token.IDENTIFIER:
 990                                                 case Token.LITERAL:
 991                                                 case Token.BASE:
 992                                                 case Token.CHECKED:
 993                                                 case Token.DELEGATE:
 994                                                 case Token.FALSE:
 995                                                 case Token.FIXED:
 996                                                 case Token.NEW:
 997                                                 case Token.NULL:
 998                                                 case Token.SIZEOF:
 999                                                 case Token.THIS:
1000                                                 case Token.THROW:
1001                                                 case Token.TRUE:
1002                                                 case Token.TYPEOF:
1003                                                 case Token.UNCHECKED:
1004                                                 case Token.UNSAFE:
1005                                                 case Token.DEFAULT:
1006                                                 case Token.AWAIT:
1007
1008                                                 //
1009                                                 // These can be part of a member access
1010                                                 //
1011                                                 case Token.INT:
1012                                                 case Token.UINT:
1013                                                 case Token.SHORT:
1014                                                 case Token.USHORT:
1015                                                 case Token.LONG:
1016                                                 case Token.ULONG:
1017                                                 case Token.DOUBLE:
1018                                                 case Token.FLOAT:
1019                                                 case Token.CHAR:
1020                                                 case Token.BYTE:
1021                                                 case Token.DECIMAL:
1022                                                 case Token.BOOL:
1023                                                         return Token.OPEN_PARENS_CAST;
1024                                                 }
1025                                         }
1026                                         return Token.OPEN_PARENS;
1027
1028                                 case Token.DOT:
1029                                 case Token.DOUBLE_COLON:
1030                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1031                                                 goto default;
1032
1033                                         continue;
1034
1035                                 case Token.IDENTIFIER:
1036                                 case Token.AWAIT:
1037                                         switch (ptoken) {
1038                                         case Token.DOT:
1039                                                 if (bracket_level == 0) {
1040                                                         is_type = false;
1041                                                         can_be_type = true;
1042                                                 }
1043
1044                                                 continue;
1045                                         case Token.OP_GENERICS_LT:
1046                                         case Token.COMMA:
1047                                         case Token.DOUBLE_COLON:
1048                                         case -1:
1049                                                 if (bracket_level == 0)
1050                                                         can_be_type = true;
1051                                                 continue;
1052                                         default:
1053                                                 can_be_type = is_type = false;
1054                                                 continue;
1055                                         }
1056
1057                                 case Token.OBJECT:
1058                                 case Token.STRING:
1059                                 case Token.BOOL:
1060                                 case Token.DECIMAL:
1061                                 case Token.FLOAT:
1062                                 case Token.DOUBLE:
1063                                 case Token.SBYTE:
1064                                 case Token.BYTE:
1065                                 case Token.SHORT:
1066                                 case Token.USHORT:
1067                                 case Token.INT:
1068                                 case Token.UINT:
1069                                 case Token.LONG:
1070                                 case Token.ULONG:
1071                                 case Token.CHAR:
1072                                 case Token.VOID:
1073                                         if (bracket_level == 0)
1074                                                 is_type = true;
1075                                         continue;
1076
1077                                 case Token.COMMA:
1078                                         if (bracket_level == 0) {
1079                                                 bracket_level = 100;
1080                                                 can_be_type = is_type = false;
1081                                         }
1082                                         continue;
1083
1084                                 case Token.OP_GENERICS_LT:
1085                                 case Token.OPEN_BRACKET:
1086                                         if (bracket_level++ == 0)
1087                                                 is_type = true;
1088                                         continue;
1089
1090                                 case Token.OP_GENERICS_GT:
1091                                 case Token.CLOSE_BRACKET:
1092                                         --bracket_level;
1093                                         continue;
1094
1095                                 case Token.INTERR_NULLABLE:
1096                                 case Token.STAR:
1097                                         if (bracket_level == 0)
1098                                                 is_type = true;
1099                                         continue;
1100
1101                                 case Token.REF:
1102                                 case Token.OUT:
1103                                         can_be_type = is_type = false;
1104                                         continue;
1105
1106                                 default:
1107                                         return Token.OPEN_PARENS;
1108                                 }
1109                         }
1110                 }
1111
1112                 public static bool IsValidIdentifier (string s)
1113                 {
1114                         if (s == null || s.Length == 0)
1115                                 return false;
1116
1117                         if (!is_identifier_start_character (s [0]))
1118                                 return false;
1119
1120                         for (int i = 1; i < s.Length; i ++)
1121                                 if (! is_identifier_part_character (s [i]))
1122                                         return false;
1123
1124                         return true;
1125                 }
1126
1127                 bool parse_less_than ()
1128                 {
1129                 start:
1130                         int the_token = token ();
1131                         if (the_token == Token.OPEN_BRACKET) {
1132                                 while (true) {
1133                                         the_token = token ();
1134                                         if (the_token == Token.EOF)
1135                                                 return true;
1136
1137                                         if (the_token == Token.CLOSE_BRACKET)
1138                                                 break;
1139                                 }
1140                                 the_token = token ();
1141                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1142                                 the_token = token ();
1143                         }
1144                         switch (the_token) {
1145                         case Token.IDENTIFIER:
1146                         case Token.OBJECT:
1147                         case Token.STRING:
1148                         case Token.BOOL:
1149                         case Token.DECIMAL:
1150                         case Token.FLOAT:
1151                         case Token.DOUBLE:
1152                         case Token.SBYTE:
1153                         case Token.BYTE:
1154                         case Token.SHORT:
1155                         case Token.USHORT:
1156                         case Token.INT:
1157                         case Token.UINT:
1158                         case Token.LONG:
1159                         case Token.ULONG:
1160                         case Token.CHAR:
1161                         case Token.VOID:
1162                                 break;
1163                         case Token.OP_GENERICS_GT:
1164                         case Token.IN:
1165                         case Token.OUT:
1166                                 return true;
1167
1168                         default:
1169                                 return false;
1170                         }
1171                 again:
1172                         the_token = token ();
1173
1174                         if (the_token == Token.OP_GENERICS_GT)
1175                                 return true;
1176                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1177                                 goto start;
1178                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1179                                 goto again;
1180                         else if (the_token == Token.OP_GENERICS_LT) {
1181                                 if (!parse_less_than ())
1182                                         return false;
1183                                 goto again;
1184                         } else if (the_token == Token.OPEN_BRACKET) {
1185                         rank_specifiers:
1186                                 the_token = token ();
1187                                 if (the_token == Token.CLOSE_BRACKET)
1188                                         goto again;
1189                                 else if (the_token == Token.COMMA)
1190                                         goto rank_specifiers;
1191                                 return false;
1192                         }
1193
1194                         return false;
1195                 }
1196
1197                 bool parse_generic_dimension (out int dimension)
1198                 {
1199                         dimension = 1;
1200
1201                 again:
1202                         int the_token = token ();
1203                         if (the_token == Token.OP_GENERICS_GT)
1204                                 return true;
1205                         else if (the_token == Token.COMMA) {
1206                                 dimension++;
1207                                 goto again;
1208                         }
1209
1210                         return false;
1211                 }
1212
1213                 public int peek_token ()
1214                 {
1215                         int the_token;
1216
1217                         PushPosition ();
1218                         the_token = token ();
1219                         PopPosition ();
1220
1221                         return the_token;
1222                 }
1223
1224                 //
1225                 // Tonizes `?' using custom disambiguous rules to return one
1226                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1227                 //
1228                 // Tricky expression looks like:
1229                 //
1230                 // Foo ? a = x ? b : c;
1231                 //
1232                 int TokenizePossibleNullableType ()
1233                 {
1234                         if (parsing_block == 0 || parsing_type > 0)
1235                                 return Token.INTERR_NULLABLE;
1236
1237                         int d = peek_char ();
1238                         if (d == '?') {
1239                                 get_char ();
1240                                 return Token.OP_COALESCING;
1241                         }
1242
1243                         switch (current_token) {
1244                         case Token.CLOSE_PARENS:
1245                         case Token.TRUE:
1246                         case Token.FALSE:
1247                         case Token.NULL:
1248                         case Token.LITERAL:
1249                                 return Token.INTERR;
1250                         }
1251
1252                         if (d != ' ') {
1253                                 if (d == ',' || d == ';' || d == '>')
1254                                         return Token.INTERR_NULLABLE;
1255                                 if (d == '*' || (d >= '0' && d <= '9'))
1256                                         return Token.INTERR;
1257                         }
1258
1259                         PushPosition ();
1260                         current_token = Token.NONE;
1261                         int next_token;
1262                         switch (xtoken ()) {
1263                         case Token.LITERAL:
1264                         case Token.TRUE:
1265                         case Token.FALSE:
1266                         case Token.NULL:
1267                         case Token.THIS:
1268                         case Token.NEW:
1269                                 next_token = Token.INTERR;
1270                                 break;
1271
1272                         case Token.SEMICOLON:
1273                         case Token.COMMA:
1274                         case Token.CLOSE_PARENS:
1275                         case Token.OPEN_BRACKET:
1276                         case Token.OP_GENERICS_GT:
1277                         case Token.INTERR:
1278                         case Token.OP_COALESCING:
1279                         case Token.COLON:
1280                                 next_token = Token.INTERR_NULLABLE;
1281                                 break;
1282
1283                         default:
1284                                 next_token = -1;
1285                                 break;
1286                         }
1287
1288                         if (next_token == -1) {
1289                                 switch (xtoken ()) {
1290                                 case Token.COMMA:
1291                                 case Token.SEMICOLON:
1292                                 case Token.OPEN_BRACE:
1293                                 case Token.CLOSE_PARENS:
1294                                 case Token.IN:
1295                                         next_token = Token.INTERR_NULLABLE;
1296                                         break;
1297
1298                                 case Token.COLON:
1299                                         next_token = Token.INTERR;
1300                                         break;
1301
1302                                 default:
1303                                         int ntoken;
1304                                         int interrs = 1;
1305                                         int colons = 0;
1306                                         int braces = 0;
1307                                         int parens = 0;
1308                                         //
1309                                         // All shorcuts failed, do it hard way
1310                                         //
1311                                         while ((ntoken = xtoken ()) != Token.EOF) {
1312                                                 switch (ntoken) {
1313                                                 case Token.OPEN_BRACE:
1314                                                         ++braces;
1315                                                         continue;
1316                                                 case Token.OPEN_PARENS:
1317                                                 case Token.OPEN_PARENS_CAST:
1318                                                 case Token.OPEN_PARENS_LAMBDA:
1319                                                         ++parens;
1320                                                         continue;
1321                                                 case Token.CLOSE_BRACE:
1322                                                         --braces;
1323                                                         continue;
1324                                                 case Token.CLOSE_PARENS:
1325                                                         if (parens > 0)
1326                                                                 --parens;
1327                                                         continue;
1328                                                 }
1329
1330                                                 if (braces != 0)
1331                                                         continue;
1332
1333                                                 if (ntoken == Token.SEMICOLON)
1334                                                         break;
1335
1336                                                 if (parens != 0)
1337                                                         continue;
1338
1339                                                 if (ntoken == Token.COLON) {
1340                                                         if (++colons == interrs)
1341                                                                 break;
1342                                                         continue;
1343                                                 }
1344
1345                                                 if (ntoken == Token.INTERR) {
1346                                                         ++interrs;
1347                                                         continue;
1348                                                 }
1349                                         }
1350
1351                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1352                                         break;
1353                                 }
1354                         }
1355
1356                         PopPosition ();
1357                         return next_token;
1358                 }
1359
1360                 bool decimal_digits (int c)
1361                 {
1362                         int d;
1363                         bool seen_digits = false;
1364
1365                         if (c != -1){
1366                                 if (number_pos == MaxNumberLength)
1367                                         Error_NumericConstantTooLong ();
1368                                 number_builder [number_pos++] = (char) c;
1369                         }
1370
1371                         //
1372                         // We use peek_char2, because decimal_digits needs to do a
1373                         // 2-character look-ahead (5.ToString for example).
1374                         //
1375                         while ((d = peek_char2 ()) != -1){
1376                                 if (d >= '0' && d <= '9'){
1377                                         if (number_pos == MaxNumberLength)
1378                                                 Error_NumericConstantTooLong ();
1379                                         number_builder [number_pos++] = (char) d;
1380                                         get_char ();
1381                                         seen_digits = true;
1382                                 } else
1383                                         break;
1384                         }
1385
1386                         return seen_digits;
1387                 }
1388
1389                 static bool is_hex (int e)
1390                 {
1391                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1392                 }
1393
1394                 static TypeCode real_type_suffix (int c)
1395                 {
1396                         switch (c){
1397                         case 'F': case 'f':
1398                                 return TypeCode.Single;
1399                         case 'D': case 'd':
1400                                 return TypeCode.Double;
1401                         case 'M': case 'm':
1402                                 return TypeCode.Decimal;
1403                         default:
1404                                 return TypeCode.Empty;
1405                         }
1406                 }
1407
1408                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1409                 {
1410                         bool is_unsigned = false;
1411                         bool is_long = false;
1412
1413                         if (c != -1){
1414                                 bool scanning = true;
1415                                 do {
1416                                         switch (c){
1417                                         case 'U': case 'u':
1418                                                 if (is_unsigned)
1419                                                         scanning = false;
1420                                                 is_unsigned = true;
1421                                                 get_char ();
1422                                                 break;
1423
1424                                         case 'l':
1425                                                 if (!is_unsigned){
1426                                                         //
1427                                                         // if we have not seen anything in between
1428                                                         // report this error
1429                                                         //
1430                                                         Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1431                                                 }
1432
1433                                                 goto case 'L';
1434
1435                                         case 'L':
1436                                                 if (is_long)
1437                                                         scanning = false;
1438                                                 is_long = true;
1439                                                 get_char ();
1440                                                 break;
1441
1442                                         default:
1443                                                 scanning = false;
1444                                                 break;
1445                                         }
1446                                         c = peek_char ();
1447                                 } while (scanning);
1448                         }
1449
1450                         if (is_long && is_unsigned){
1451                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1452                         }
1453
1454                         if (is_unsigned){
1455                                 // uint if possible, or ulong else.
1456
1457                                 if ((ul & 0xffffffff00000000) == 0)
1458                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1459                                 else
1460                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1461                         } else if (is_long){
1462                                 // long if possible, ulong otherwise
1463                                 if ((ul & 0x8000000000000000) != 0)
1464                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1465                                 else
1466                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1467                         } else {
1468                                 // int, uint, long or ulong in that order
1469                                 if ((ul & 0xffffffff00000000) == 0){
1470                                         uint ui = (uint) ul;
1471
1472                                         if ((ui & 0x80000000) != 0)
1473                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1474                                         else
1475                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1476                                 } else {
1477                                         if ((ul & 0x8000000000000000) != 0)
1478                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1479                                         else
1480                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1481                                 }
1482                         }
1483                 }
1484
1485                 //
1486                 // given `c' as the next char in the input decide whether
1487                 // we need to convert to a special type, and then choose
1488                 // the best representation for the integer
1489                 //
1490                 ILiteralConstant adjust_int (int c, Location loc)
1491                 {
1492                         try {
1493                                 if (number_pos > 9){
1494                                         ulong ul = (uint) (number_builder [0] - '0');
1495
1496                                         for (int i = 1; i < number_pos; i++){
1497                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1498                                         }
1499
1500                                         return integer_type_suffix (ul, c, loc);
1501                                 } else {
1502                                         uint ui = (uint) (number_builder [0] - '0');
1503
1504                                         for (int i = 1; i < number_pos; i++){
1505                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1506                                         }
1507
1508                                         return integer_type_suffix (ui, c, loc);
1509                                 }
1510                         } catch (OverflowException) {
1511                                 Error_NumericConstantTooLong ();
1512                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1513                         }
1514                         catch (FormatException) {
1515                                 Report.Error (1013, Location, "Invalid number");
1516                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1517                         }
1518                 }
1519
1520                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1521                 {
1522                         string s = new string (number_builder, 0, number_pos);
1523                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1524
1525                         switch (t){
1526                         case TypeCode.Decimal:
1527                                 try {
1528                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1529                                 } catch (OverflowException) {
1530                                         Report.Error (594, Location, error_details, "decimal");
1531                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1532                                 }
1533                         case TypeCode.Single:
1534                                 try {
1535                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1536                                 } catch (OverflowException) {
1537                                         Report.Error (594, Location, error_details, "float");
1538                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1539                                 }
1540                         default:
1541                                 try {
1542                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1543                                 } catch (OverflowException) {
1544                                         Report.Error (594, loc, error_details, "double");
1545                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1546                                 }
1547                         }
1548                 }
1549
1550                 ILiteralConstant handle_hex (Location loc)
1551                 {
1552                         int d;
1553                         ulong ul;
1554
1555                         get_char ();
1556                         while ((d = peek_char ()) != -1){
1557                                 if (is_hex (d)){
1558                                         number_builder [number_pos++] = (char) d;
1559                                         get_char ();
1560                                 } else
1561                                         break;
1562                         }
1563
1564                         string s = new String (number_builder, 0, number_pos);
1565
1566                         try {
1567                                 if (number_pos <= 8)
1568                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1569                                 else
1570                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1571
1572                                 return integer_type_suffix (ul, peek_char (), loc);
1573                         } catch (OverflowException){
1574                                 Error_NumericConstantTooLong ();
1575                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1576                         }
1577                         catch (FormatException) {
1578                                 Report.Error (1013, Location, "Invalid number");
1579                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1580                         }
1581                 }
1582
1583                 //
1584                 // Invoked if we know we have .digits or digits
1585                 //
1586                 int is_number (int c, bool dotLead)
1587                 {
1588                         ILiteralConstant res;
1589
1590 #if FULL_AST
1591                         int read_start = reader.Position - 1;
1592                         if (dotLead) {
1593                                 //
1594                                 // Caller did peek_char
1595                                 //
1596                                 --read_start;
1597                         }
1598 #endif
1599                         number_pos = 0;
1600                         var loc = Location;
1601
1602                         if (!dotLead){
1603                                 if (c == '0'){
1604                                         int peek = peek_char ();
1605
1606                                         if (peek == 'x' || peek == 'X') {
1607                                                 val = res = handle_hex (loc);
1608 #if FULL_AST
1609                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1610 #endif
1611
1612                                                 return Token.LITERAL;
1613                                         }
1614                                 }
1615                                 decimal_digits (c);
1616                                 c = peek_char ();
1617                         }
1618
1619                         //
1620                         // We need to handle the case of
1621                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1622                         //
1623                         bool is_real = false;
1624                         if (c == '.'){
1625                                 if (!dotLead)
1626                                         get_char ();
1627
1628                                 if (decimal_digits ('.')){
1629                                         is_real = true;
1630                                         c = peek_char ();
1631                                 } else {
1632                                         putback ('.');
1633                                         number_pos--;
1634                                         val = res = adjust_int (-1, loc);
1635
1636 #if FULL_AST
1637                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1638 #endif
1639                                         return Token.LITERAL;
1640                                 }
1641                         }
1642
1643                         if (c == 'e' || c == 'E'){
1644                                 is_real = true;
1645                                 get_char ();
1646                                 if (number_pos == MaxNumberLength)
1647                                         Error_NumericConstantTooLong ();
1648                                 number_builder [number_pos++] = (char) c;
1649                                 c = get_char ();
1650
1651                                 if (c == '+'){
1652                                         if (number_pos == MaxNumberLength)
1653                                                 Error_NumericConstantTooLong ();
1654                                         number_builder [number_pos++] = '+';
1655                                         c = -1;
1656                                 } else if (c == '-') {
1657                                         if (number_pos == MaxNumberLength)
1658                                                 Error_NumericConstantTooLong ();
1659                                         number_builder [number_pos++] = '-';
1660                                         c = -1;
1661                                 } else {
1662                                         if (number_pos == MaxNumberLength)
1663                                                 Error_NumericConstantTooLong ();
1664                                         number_builder [number_pos++] = '+';
1665                                 }
1666
1667                                 decimal_digits (c);
1668                                 c = peek_char ();
1669                         }
1670
1671                         var type = real_type_suffix (c);
1672                         if (type == TypeCode.Empty && !is_real) {
1673                                 res = adjust_int (c, loc);
1674                         } else {
1675                                 is_real = true;
1676
1677                                 if (type != TypeCode.Empty) {
1678                                         get_char ();
1679                                 }
1680
1681                                 res = adjust_real (type, loc);
1682                         }
1683
1684                         val = res;
1685
1686 #if FULL_AST
1687                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1688                         if (chars[chars.Length - 1] == '\r')
1689                                 Array.Resize (ref chars, chars.Length - 1);
1690                         res.ParsedValue = chars;
1691 #endif
1692
1693                         return Token.LITERAL;
1694                 }
1695
1696                 //
1697                 // Accepts exactly count (4 or 8) hex, no more no less
1698                 //
1699                 int getHex (int count, out int surrogate, out bool error)
1700                 {
1701                         int i;
1702                         int total = 0;
1703                         int c;
1704                         int top = count != -1 ? count : 4;
1705
1706                         get_char ();
1707                         error = false;
1708                         surrogate = 0;
1709                         for (i = 0; i < top; i++){
1710                                 c = get_char ();
1711
1712                                 if (c >= '0' && c <= '9')
1713                                         c = (int) c - (int) '0';
1714                                 else if (c >= 'A' && c <= 'F')
1715                                         c = (int) c - (int) 'A' + 10;
1716                                 else if (c >= 'a' && c <= 'f')
1717                                         c = (int) c - (int) 'a' + 10;
1718                                 else {
1719                                         error = true;
1720                                         return 0;
1721                                 }
1722
1723                                 total = (total * 16) + c;
1724                                 if (count == -1){
1725                                         int p = peek_char ();
1726                                         if (p == -1)
1727                                                 break;
1728                                         if (!is_hex ((char)p))
1729                                                 break;
1730                                 }
1731                         }
1732
1733                         if (top == 8) {
1734                                 if (total > 0x0010FFFF) {
1735                                         error = true;
1736                                         return 0;
1737                                 }
1738
1739                                 if (total >= 0x00010000) {
1740                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1741                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1742                                 }
1743                         }
1744
1745                         return total;
1746                 }
1747
1748                 int escape (int c, out int surrogate)
1749                 {
1750                         bool error;
1751                         int d;
1752                         int v;
1753
1754                         d = peek_char ();
1755                         if (c != '\\') {
1756                                 surrogate = 0;
1757                                 return c;
1758                         }
1759
1760                         switch (d){
1761                         case 'a':
1762                                 v = '\a'; break;
1763                         case 'b':
1764                                 v = '\b'; break;
1765                         case 'n':
1766                                 v = '\n'; break;
1767                         case 't':
1768                                 v = '\t'; break;
1769                         case 'v':
1770                                 v = '\v'; break;
1771                         case 'r':
1772                                 v = '\r'; break;
1773                         case '\\':
1774                                 v = '\\'; break;
1775                         case 'f':
1776                                 v = '\f'; break;
1777                         case '0':
1778                                 v = 0; break;
1779                         case '"':
1780                                 v = '"'; break;
1781                         case '\'':
1782                                 v = '\''; break;
1783                         case 'x':
1784                                 v = getHex (-1, out surrogate, out error);
1785                                 if (error)
1786                                         goto default;
1787                                 return v;
1788                         case 'u':
1789                         case 'U':
1790                                 return EscapeUnicode (d, out surrogate);
1791                         default:
1792                                 surrogate = 0;
1793                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1794                                 return d;
1795                         }
1796
1797                         get_char ();
1798                         surrogate = 0;
1799                         return v;
1800                 }
1801
1802                 int EscapeUnicode (int ch, out int surrogate)
1803                 {
1804                         bool error;
1805                         if (ch == 'U') {
1806                                 ch = getHex (8, out surrogate, out error);
1807                         } else {
1808                                 ch = getHex (4, out surrogate, out error);
1809                         }
1810
1811                         if (error)
1812                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1813
1814                         return ch;
1815                 }
1816
1817                 int get_char ()
1818                 {
1819                         int x;
1820                         if (putback_char != -1) {
1821                                 x = putback_char;
1822                                 putback_char = -1;
1823                         } else {
1824                                 x = reader.Read ();
1825                         }
1826
1827                         if (x <= 13) {
1828                                 if (x == '\r') {
1829                                         if (peek_char () == '\n') {
1830                                                 putback_char = -1;
1831                                         }
1832
1833                                         x = '\n';
1834                                         advance_line ();
1835                                 } else if (x == '\n') {
1836                                         advance_line ();
1837                                 } else {
1838                                         col++;
1839                                 }
1840                         } else if (x >= UnicodeLS && x <= UnicodePS) {
1841                                 advance_line ();
1842                         } else {
1843                                 col++;
1844                         }
1845
1846                         return x;
1847                 }
1848
1849                 void advance_line ()
1850                 {
1851                         line++;
1852                         ref_line++;
1853                         previous_col = col;
1854                         col = 0;
1855                 }
1856
1857                 int peek_char ()
1858                 {
1859                         if (putback_char == -1)
1860                                 putback_char = reader.Read ();
1861                         return putback_char;
1862                 }
1863
1864                 int peek_char2 ()
1865                 {
1866                         if (putback_char != -1)
1867                                 return putback_char;
1868                         return reader.Peek ();
1869                 }
1870
1871                 public void putback (int c)
1872                 {
1873                         if (putback_char != -1) {
1874                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
1875                         }
1876
1877                         if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
1878                                 // It won't happen though.
1879                                 line--;
1880                                 ref_line--;
1881                                 col = previous_col;
1882                         }
1883                         else
1884                                 col--;
1885                         putback_char = c;
1886                 }
1887
1888                 public bool advance ()
1889                 {
1890                         return peek_char () != -1 || CompleteOnEOF;
1891                 }
1892
1893                 public Object Value {
1894                         get {
1895                                 return val;
1896                         }
1897                 }
1898
1899                 public Object value ()
1900                 {
1901                         return val;
1902                 }
1903
1904                 public int token ()
1905                 {
1906                         current_token = xtoken ();
1907                         return current_token;
1908                 }
1909
1910                 int TokenizePreprocessorIdentifier (out int c)
1911                 {
1912                         // skip over white space
1913                         do {
1914                                 c = get_char ();
1915                         } while (c == ' ' || c == '\t');
1916
1917
1918                         int pos = 0;
1919                         while (c != -1 && c >= 'a' && c <= 'z') {
1920                                 id_builder[pos++] = (char) c;
1921                                 c = get_char ();
1922                                 if (c == '\\') {
1923                                         int peek = peek_char ();
1924                                         if (peek == 'U' || peek == 'u') {
1925                                                 int surrogate;
1926                                                 c = EscapeUnicode (c, out surrogate);
1927                                                 if (surrogate != 0) {
1928                                                         if (is_identifier_part_character ((char) c)) {
1929                                                                 id_builder[pos++] = (char) c;
1930                                                         }
1931                                                         c = surrogate;
1932                                                 }
1933                                         }
1934                                 }
1935                         }
1936
1937                         return pos;
1938                 }
1939
1940                 PreprocessorDirective get_cmd_arg (out string arg)
1941                 {
1942                         int c;
1943
1944                         tokens_seen = false;
1945                         arg = "";
1946
1947                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1948
1949                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1950                                 return cmd;
1951
1952                         // skip over white space
1953                         while (c == ' ' || c == '\t')
1954                                 c = get_char ();
1955
1956                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1957                         int pos = 0;
1958
1959                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
1960                                 if (c == '\\' && has_identifier_argument >= 0) {
1961                                         if (has_identifier_argument != 0) {
1962                                                 has_identifier_argument = 1;
1963
1964                                                 int peek = peek_char ();
1965                                                 if (peek == 'U' || peek == 'u') {
1966                                                         int surrogate;
1967                                                         c = EscapeUnicode (c, out surrogate);
1968                                                         if (surrogate != 0) {
1969                                                                 if (is_identifier_part_character ((char) c)) {
1970                                                                         if (pos == value_builder.Length)
1971                                                                                 Array.Resize (ref value_builder, pos * 2);
1972
1973                                                                         value_builder[pos++] = (char) c;
1974                                                                 }
1975                                                                 c = surrogate;
1976                                                         }
1977                                                 }
1978                                         } else {
1979                                                 has_identifier_argument = -1;
1980                                         }
1981                                 } else if (c == '/' && peek_char () == '/') {
1982                                         //
1983                                         // Eat single-line comments
1984                                         //
1985                                         get_char ();
1986                                         ReadToEndOfLine ();
1987                                         break;
1988                                 }
1989
1990                                 if (pos == value_builder.Length)
1991                                         Array.Resize (ref value_builder, pos * 2);
1992
1993                                 value_builder[pos++] = (char) c;
1994                                 c = get_char ();
1995                         }
1996
1997                         if (pos != 0) {
1998                                 if (pos > MaxIdentifierLength)
1999                                         arg = new string (value_builder, 0, pos);
2000                                 else
2001                                         arg = InternIdentifier (value_builder, pos);
2002
2003                                 // Eat any trailing whitespaces
2004                                 arg = arg.Trim (simple_whitespaces);
2005                         }
2006
2007                         return cmd;
2008                 }
2009
2010                 //
2011                 // Handles the #line directive
2012                 //
2013                 bool PreProcessLine ()
2014                 {
2015                         Location loc = Location;
2016
2017                         int c;
2018
2019                         int length = TokenizePreprocessorIdentifier (out c);
2020                         if (length == line_default.Length) {
2021                                 if (!IsTokenIdentifierEqual (line_default))
2022                                         return false;
2023
2024                                 current_source = source_file.SourceFile;
2025                                 if (!hidden_block_start.IsNull) {
2026                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
2027                                         hidden_block_start = Location.Null;
2028                                 }
2029
2030                                 ref_line = line;
2031                                 return true;
2032                         }
2033
2034                         if (length == line_hidden.Length) {
2035                                 if (!IsTokenIdentifierEqual (line_hidden))
2036                                         return false;
2037
2038                                 if (hidden_block_start.IsNull)
2039                                         hidden_block_start = loc;
2040
2041                                 return true;
2042                         }
2043
2044                         if (length != 0 || c < '0' || c > '9') {
2045                                 //
2046                                 // Eat any remaining characters to continue parsing on next line
2047                                 //
2048                                 ReadToEndOfLine ();
2049                                 return false;
2050                         }
2051
2052                         int new_line = TokenizeNumber (c);
2053                         if (new_line < 1) {
2054                                 //
2055                                 // Eat any remaining characters to continue parsing on next line
2056                                 //
2057                                 ReadToEndOfLine ();
2058                                 return new_line != 0;
2059                         }
2060
2061                         c = get_char ();
2062                         if (c == ' ') {
2063                                 // skip over white space
2064                                 do {
2065                                         c = get_char ();
2066                                 } while (c == ' ' || c == '\t');
2067                         } else if (c == '"') {
2068                                 c = 0;
2069                         }
2070
2071                         if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
2072                                 //
2073                                 // Eat any remaining characters to continue parsing on next line
2074                                 //
2075                                 ReadToEndOfLine ();
2076
2077                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2078                                 return true;
2079                         }
2080
2081                         string new_file_name = null;
2082                         if (c == '"') {
2083                                 new_file_name = TokenizeFileName (ref c);
2084
2085                                 // skip over white space
2086                                 while (c == ' ' || c == '\t') {
2087                                         c = get_char ();
2088                                 }
2089                         }
2090
2091                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2092
2093                         } else if (c == '/') {
2094                                 ReadSingleLineComment ();
2095                         } else {
2096                                 //
2097                                 // Eat any remaining characters to continue parsing on next line
2098                                 //
2099                                 ReadToEndOfLine ();
2100
2101                                 Error_EndLineExpected ();
2102                                 return true;
2103                         }
2104
2105                         if (new_file_name != null) {
2106                                 current_source = context.LookupFile (source_file, new_file_name);
2107                                 source_file.AddIncludeFile (current_source);
2108                         }
2109
2110                         if (!hidden_block_start.IsNull) {
2111                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2112                                 hidden_block_start = Location.Null;
2113                         }
2114
2115                         ref_line = new_line;
2116                         return true;
2117                 }
2118
2119                 //
2120                 // Handles #define and #undef
2121                 //
2122                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2123                 {
2124                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2125                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2126                                 return;
2127                         }
2128
2129                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2130                                 Error_EndLineExpected ();
2131                                 return;
2132                         }
2133
2134                         if (!is_identifier_start_character (ident [0]))
2135                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2136
2137                         foreach (char c in ident.Substring (1)){
2138                                 if (!is_identifier_part_character (c)){
2139                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2140                                         return;
2141                                 }
2142                         }
2143
2144                         if (!caller_is_taking)
2145                                 return;
2146
2147                         if (is_define) {
2148                                 //
2149                                 // #define ident
2150                                 //
2151                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2152                                         return;
2153
2154                                 source_file.AddDefine (ident);
2155                         } else {
2156                                 //
2157                                 // #undef ident
2158                                 //
2159                                 source_file.AddUndefine (ident);
2160                         }
2161                 }
2162
2163                 byte read_hex (out bool error)
2164                 {
2165                         int total;
2166                         int c = get_char ();
2167
2168                         if ((c >= '0') && (c <= '9'))
2169                                 total = (int) c - (int) '0';
2170                         else if ((c >= 'A') && (c <= 'F'))
2171                                 total = (int) c - (int) 'A' + 10;
2172                         else if ((c >= 'a') && (c <= 'f'))
2173                                 total = (int) c - (int) 'a' + 10;
2174                         else {
2175                                 error = true;
2176                                 return 0;
2177                         }
2178
2179                         total *= 16;
2180                         c = get_char ();
2181
2182                         if ((c >= '0') && (c <= '9'))
2183                                 total += (int) c - (int) '0';
2184                         else if ((c >= 'A') && (c <= 'F'))
2185                                 total += (int) c - (int) 'A' + 10;
2186                         else if ((c >= 'a') && (c <= 'f'))
2187                                 total += (int) c - (int) 'a' + 10;
2188                         else {
2189                                 error = true;
2190                                 return 0;
2191                         }
2192
2193                         error = false;
2194                         return (byte) total;
2195                 }
2196
2197                 //
2198                 // Parses #pragma checksum
2199                 //
2200                 bool ParsePragmaChecksum ()
2201                 {
2202                         //
2203                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2204                         //
2205                         // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
2206                         //
2207                         int c = get_char ();
2208
2209                         if (c != '"')
2210                                 return false;
2211
2212                         string file_name = TokenizeFileName (ref c);
2213
2214                         // TODO: Any white-spaces count
2215                         if (c != ' ')
2216                                 return false;
2217
2218                         SourceFile file = context.LookupFile (source_file, file_name);
2219
2220                         if (get_char () != '"' || get_char () != '{')
2221                                 return false;
2222
2223                         bool error;
2224                         byte[] guid_bytes = new byte [16];
2225                         int i = 0;
2226
2227                         for (; i < 4; i++) {
2228                                 guid_bytes [i] = read_hex (out error);
2229                                 if (error)
2230                                         return false;
2231                         }
2232
2233                         if (get_char () != '-')
2234                                 return false;
2235
2236                         for (; i < 10; i++) {
2237                                 guid_bytes [i] = read_hex (out error);
2238                                 if (error)
2239                                         return false;
2240
2241                                 guid_bytes [i++] = read_hex (out error);
2242                                 if (error)
2243                                         return false;
2244
2245                                 if (get_char () != '-')
2246                                         return false;
2247                         }
2248
2249                         for (; i < 16; i++) {
2250                                 guid_bytes [i] = read_hex (out error);
2251                                 if (error)
2252                                         return false;
2253                         }
2254
2255                         if (get_char () != '}' || get_char () != '"')
2256                                 return false;
2257
2258                         // TODO: Any white-spaces count
2259                         c = get_char ();
2260                         if (c != ' ')
2261                                 return false;
2262
2263                         if (get_char () != '"')
2264                                 return false;
2265
2266                         // Any length of checksum
2267                         List<byte> checksum_bytes = new List<byte> (16);
2268
2269                         var checksum_location = Location;
2270                         c = peek_char ();
2271                         while (c != '"' && c != -1) {
2272                                 checksum_bytes.Add (read_hex (out error));
2273                                 if (error)
2274                                         return false;
2275
2276                                 c = peek_char ();
2277                         }
2278
2279                         if (c == '/') {
2280                                 ReadSingleLineComment ();
2281                         } else if (get_char () != '"') {
2282                                 return false;
2283                         }
2284
2285                         if (context.Settings.GenerateDebugInfo) {
2286                                 var chsum = checksum_bytes.ToArray ();
2287
2288                                 if (file.HasChecksum) {
2289                                         if (!ArrayComparer.IsEqual (file.Checksum, chsum)) {
2290                                                 // TODO: Report.SymbolRelatedToPreviousError
2291                                                 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name);
2292                                         }
2293                                 }
2294
2295                                 file.SetChecksum (guid_bytes, chsum);
2296                                 current_source.AutoGenerated = true;
2297                         }
2298
2299                         return true;
2300                 }
2301
2302                 bool IsTokenIdentifierEqual (char[] identifier)
2303                 {
2304                         for (int i = 0; i < identifier.Length; ++i) {
2305                                 if (identifier[i] != id_builder[i])
2306                                         return false;
2307                         }
2308
2309                         return true;
2310                 }
2311
2312                 int TokenizeNumber (int value)
2313                 {
2314                         number_pos = 0;
2315
2316                         decimal_digits (value);
2317                         uint ui = (uint) (number_builder[0] - '0');
2318
2319                         try {
2320                                 for (int i = 1; i < number_pos; i++) {
2321                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2322                                 }
2323
2324                                 return (int) ui;
2325                         } catch (OverflowException) {
2326                                 Error_NumericConstantTooLong ();
2327                                 return -1;
2328                         }
2329                 }
2330
2331                 string TokenizeFileName (ref int c)
2332                 {
2333                         var string_builder = new StringBuilder ();
2334                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2335                                 c = get_char ();
2336                                 if (c == '"') {
2337                                         c = get_char ();
2338                                         break;
2339                                 }
2340
2341                                 string_builder.Append ((char) c);
2342                         }
2343
2344                         if (string_builder.Length == 0) {
2345                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2346                         }
2347
2348
2349                         return string_builder.ToString ();
2350                 }
2351
2352                 int TokenizePragmaNumber (ref int c)
2353                 {
2354                         number_pos = 0;
2355
2356                         int number;
2357
2358                         if (c >= '0' && c <= '9') {
2359                                 number = TokenizeNumber (c);
2360
2361                                 c = get_char ();
2362
2363                                 // skip over white space
2364                                 while (c == ' ' || c == '\t')
2365                                         c = get_char ();
2366
2367                                 if (c == ',') {
2368                                         c = get_char ();
2369                                 }
2370
2371                                 // skip over white space
2372                                 while (c == ' ' || c == '\t')
2373                                         c = get_char ();
2374                         } else {
2375                                 number = -1;
2376                                 if (c == '/') {
2377                                         ReadSingleLineComment ();
2378                                 } else {
2379                                         Report.Warning (1692, 1, Location, "Invalid number");
2380
2381                                         // Read everything till the end of the line or file
2382                                         ReadToEndOfLine ();
2383                                 }
2384                         }
2385
2386                         return number;
2387                 }
2388
2389                 void ReadToEndOfLine ()
2390                 {
2391                         int c;
2392                         do {
2393                                 c = get_char ();
2394                         } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
2395                 }
2396
2397                 void ReadSingleLineComment ()
2398                 {
2399                         if (peek_char () != '/')
2400                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2401
2402                         // Read everything till the end of the line or file
2403                         ReadToEndOfLine ();
2404                 }
2405
2406                 /// <summary>
2407                 /// Handles #pragma directive
2408                 /// </summary>
2409                 void ParsePragmaDirective ()
2410                 {
2411                         int c;
2412                         int length = TokenizePreprocessorIdentifier (out c);
2413                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2414                                 length = TokenizePreprocessorIdentifier (out c);
2415
2416                                 //
2417                                 // #pragma warning disable
2418                                 // #pragma warning restore
2419                                 //
2420                                 if (length == pragma_warning_disable.Length) {
2421                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2422                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2423                                                 // skip over white space
2424                                                 while (c == ' ' || c == '\t')
2425                                                         c = get_char ();
2426
2427                                                 var loc = Location;
2428
2429                                                 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
2430                                                         if (c == '/')
2431                                                                 ReadSingleLineComment ();
2432
2433                                                         //
2434                                                         // Disable/Restore all warnings
2435                                                         //
2436                                                         if (disable) {
2437                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2438                                                         } else {
2439                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2440                                                         }
2441                                                 } else {
2442                                                         //
2443                                                         // Disable/Restore a warning or group of warnings
2444                                                         //
2445                                                         int code;
2446                                                         do {
2447                                                                 code = TokenizePragmaNumber (ref c);
2448                                                                 if (code > 0) {
2449                                                                         if (disable) {
2450                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2451                                                                         } else {
2452                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2453                                                                         }
2454                                                                 }
2455                                                         } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
2456                                                 }
2457
2458                                                 return;
2459                                         }
2460                                 }
2461
2462                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2463
2464                                 // Eat any remaining characters on the line
2465                                 ReadToEndOfLine ();
2466
2467                                 return;
2468                         }
2469
2470                         //
2471                         // #pragma checksum
2472                         //
2473                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2474                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2475                                         Report.Warning (1695, 1, Location,
2476                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2477                                 }
2478
2479                                 return;
2480                         }
2481
2482                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2483                 }
2484
2485                 bool eval_val (string s)
2486                 {
2487                         if (s == "true")
2488                                 return true;
2489                         if (s == "false")
2490                                 return false;
2491
2492                         return source_file.IsConditionalDefined (s);
2493                 }
2494
2495                 bool pp_primary (ref string s)
2496                 {
2497                         s = s.Trim ();
2498                         int len = s.Length;
2499
2500                         if (len > 0){
2501                                 char c = s [0];
2502
2503                                 if (c == '('){
2504                                         s = s.Substring (1);
2505                                         bool val = pp_expr (ref s, false);
2506                                         if (s.Length > 0 && s [0] == ')'){
2507                                                 s = s.Substring (1);
2508                                                 return val;
2509                                         }
2510                                         Error_InvalidDirective ();
2511                                         return false;
2512                                 }
2513
2514                                 if (is_identifier_start_character (c)){
2515                                         int j = 1;
2516
2517                                         while (j < len){
2518                                                 c = s [j];
2519
2520                                                 if (is_identifier_part_character (c)){
2521                                                         j++;
2522                                                         continue;
2523                                                 }
2524                                                 bool v = eval_val (s.Substring (0, j));
2525                                                 s = s.Substring (j);
2526                                                 return v;
2527                                         }
2528                                         bool vv = eval_val (s);
2529                                         s = "";
2530                                         return vv;
2531                                 }
2532                         }
2533                         Error_InvalidDirective ();
2534                         return false;
2535                 }
2536
2537                 bool pp_unary (ref string s)
2538                 {
2539                         s = s.Trim ();
2540                         int len = s.Length;
2541
2542                         if (len > 0){
2543                                 if (s [0] == '!'){
2544                                         if (len > 1 && s [1] == '='){
2545                                                 Error_InvalidDirective ();
2546                                                 return false;
2547                                         }
2548                                         s = s.Substring (1);
2549                                         return ! pp_primary (ref s);
2550                                 } else
2551                                         return pp_primary (ref s);
2552                         } else {
2553                                 Error_InvalidDirective ();
2554                                 return false;
2555                         }
2556                 }
2557
2558                 bool pp_eq (ref string s)
2559                 {
2560                         bool va = pp_unary (ref s);
2561
2562                         s = s.Trim ();
2563                         int len = s.Length;
2564                         if (len > 0){
2565                                 if (s [0] == '='){
2566                                         if (len > 2 && s [1] == '='){
2567                                                 s = s.Substring (2);
2568                                                 return va == pp_unary (ref s);
2569                                         } else {
2570                                                 Error_InvalidDirective ();
2571                                                 return false;
2572                                         }
2573                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2574                                         s = s.Substring (2);
2575
2576                                         return va != pp_unary (ref s);
2577
2578                                 }
2579                         }
2580
2581                         return va;
2582
2583                 }
2584
2585                 bool pp_and (ref string s)
2586                 {
2587                         bool va = pp_eq (ref s);
2588
2589                         s = s.Trim ();
2590                         int len = s.Length;
2591                         if (len > 0){
2592                                 if (s [0] == '&'){
2593                                         if (len > 2 && s [1] == '&'){
2594                                                 s = s.Substring (2);
2595                                                 return (va & pp_and (ref s));
2596                                         } else {
2597                                                 Error_InvalidDirective ();
2598                                                 return false;
2599                                         }
2600                                 }
2601                         }
2602                         return va;
2603                 }
2604
2605                 //
2606                 // Evaluates an expression for `#if' or `#elif'
2607                 //
2608                 bool pp_expr (ref string s, bool isTerm)
2609                 {
2610                         bool va = pp_and (ref s);
2611                         s = s.Trim ();
2612                         int len = s.Length;
2613                         if (len > 0){
2614                                 char c = s [0];
2615
2616                                 if (c == '|'){
2617                                         if (len > 2 && s [1] == '|'){
2618                                                 s = s.Substring (2);
2619                                                 return va | pp_expr (ref s, isTerm);
2620                                         } else {
2621                                                 Error_InvalidDirective ();
2622                                                 return false;
2623                                         }
2624                                 }
2625                                 if (isTerm) {
2626                                         Error_EndLineExpected ();
2627                                         return false;
2628                                 }
2629                         }
2630
2631                         return va;
2632                 }
2633
2634                 bool eval (string s)
2635                 {
2636                         bool v = pp_expr (ref s, true);
2637                         s = s.Trim ();
2638                         if (s.Length != 0){
2639                                 return false;
2640                         }
2641
2642                         return v;
2643                 }
2644
2645                 void Error_NumericConstantTooLong ()
2646                 {
2647                         Report.Error (1021, Location, "Integral constant is too large");
2648                 }
2649
2650                 void Error_InvalidDirective ()
2651                 {
2652                         Report.Error (1517, Location, "Invalid preprocessor directive");
2653                 }
2654
2655                 void Error_UnexpectedDirective (string extra)
2656                 {
2657                         Report.Error (
2658                                 1028, Location,
2659                                 "Unexpected processor directive ({0})", extra);
2660                 }
2661
2662                 void Error_TokensSeen ()
2663                 {
2664                         Report.Error (1032, Location,
2665                                 "Cannot define or undefine preprocessor symbols after first token in file");
2666                 }
2667
2668                 void Eror_WrongPreprocessorLocation ()
2669                 {
2670                         Report.Error (1040, Location,
2671                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2672                 }
2673
2674                 void Error_EndLineExpected ()
2675                 {
2676                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2677                 }
2678
2679                 //
2680                 // Raises a warning when tokenizer found documentation comment
2681                 // on unexpected place
2682                 //
2683                 void WarningMisplacedComment (Location loc)
2684                 {
2685                         if (doc_state != XmlCommentState.Error) {
2686                                 doc_state = XmlCommentState.Error;
2687                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2688                         }
2689                 }
2690
2691                 //
2692                 // if true, then the code continues processing the code
2693                 // if false, the code stays in a loop until another directive is
2694                 // reached.
2695                 // When caller_is_taking is false we ignore all directives except the ones
2696                 // which can help us to identify where the #if block ends
2697                 bool ParsePreprocessingDirective (bool caller_is_taking)
2698                 {
2699                         string arg;
2700                         bool region_directive = false;
2701
2702                         var directive = get_cmd_arg (out arg);
2703
2704                         //
2705                         // The first group of pre-processing instructions is always processed
2706                         //
2707                         switch (directive) {
2708                         case PreprocessorDirective.Region:
2709                                 region_directive = true;
2710                                 arg = "true";
2711                                 goto case PreprocessorDirective.If;
2712
2713                         case PreprocessorDirective.Endregion:
2714                                 if (ifstack == null || ifstack.Count == 0){
2715                                         Error_UnexpectedDirective ("no #region for this #endregion");
2716                                         return true;
2717                                 }
2718                                 int pop = ifstack.Pop ();
2719
2720                                 if ((pop & REGION) == 0)
2721                                         Report.Error (1027, Location, "Expected `#endif' directive");
2722
2723                                 return caller_is_taking;
2724
2725                         case PreprocessorDirective.If:
2726                                 if (ifstack == null)
2727                                         ifstack = new Stack<int> (2);
2728
2729                                 int flags = region_directive ? REGION : 0;
2730                                 if (ifstack.Count == 0){
2731                                         flags |= PARENT_TAKING;
2732                                 } else {
2733                                         int state = ifstack.Peek ();
2734                                         if ((state & TAKING) != 0) {
2735                                                 flags |= PARENT_TAKING;
2736                                         }
2737                                 }
2738
2739                                 if (eval (arg) && caller_is_taking) {
2740                                         ifstack.Push (flags | TAKING);
2741                                         return true;
2742                                 }
2743                                 ifstack.Push (flags);
2744                                 return false;
2745
2746                         case PreprocessorDirective.Endif:
2747                                 if (ifstack == null || ifstack.Count == 0){
2748                                         Error_UnexpectedDirective ("no #if for this #endif");
2749                                         return true;
2750                                 } else {
2751                                         pop = ifstack.Pop ();
2752
2753                                         if ((pop & REGION) != 0)
2754                                                 Report.Error (1038, Location, "#endregion directive expected");
2755
2756                                         if (arg.Length != 0) {
2757                                                 Error_EndLineExpected ();
2758                                         }
2759
2760                                         if (ifstack.Count == 0)
2761                                                 return true;
2762
2763                                         int state = ifstack.Peek ();
2764                                         return (state & TAKING) != 0;
2765                                 }
2766
2767                         case PreprocessorDirective.Elif:
2768                                 if (ifstack == null || ifstack.Count == 0){
2769                                         Error_UnexpectedDirective ("no #if for this #elif");
2770                                         return true;
2771                                 } else {
2772                                         int state = ifstack.Pop ();
2773
2774                                         if ((state & REGION) != 0) {
2775                                                 Report.Error (1038, Location, "#endregion directive expected");
2776                                                 return true;
2777                                         }
2778
2779                                         if ((state & ELSE_SEEN) != 0){
2780                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2781                                                 return true;
2782                                         }
2783
2784                                         if ((state & TAKING) != 0) {
2785                                                 ifstack.Push (0);
2786                                                 return false;
2787                                         }
2788
2789                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2790                                                 ifstack.Push (state | TAKING);
2791                                                 return true;
2792                                         }
2793
2794                                         ifstack.Push (state);
2795                                         return false;
2796                                 }
2797
2798                         case PreprocessorDirective.Else:
2799                                 if (ifstack == null || ifstack.Count == 0){
2800                                         Error_UnexpectedDirective ("no #if for this #else");
2801                                         return true;
2802                                 } else {
2803                                         int state = ifstack.Peek ();
2804
2805                                         if ((state & REGION) != 0) {
2806                                                 Report.Error (1038, Location, "#endregion directive expected");
2807                                                 return true;
2808                                         }
2809
2810                                         if ((state & ELSE_SEEN) != 0){
2811                                                 Error_UnexpectedDirective ("#else within #else");
2812                                                 return true;
2813                                         }
2814
2815                                         ifstack.Pop ();
2816
2817                                         if (arg.Length != 0) {
2818                                                 Error_EndLineExpected ();
2819                                                 return true;
2820                                         }
2821
2822                                         bool ret = false;
2823                                         if ((state & PARENT_TAKING) != 0) {
2824                                                 ret = (state & TAKING) == 0;
2825
2826                                                 if (ret)
2827                                                         state |= TAKING;
2828                                                 else
2829                                                         state &= ~TAKING;
2830                                         }
2831
2832                                         ifstack.Push (state | ELSE_SEEN);
2833
2834                                         return ret;
2835                                 }
2836                         case PreprocessorDirective.Define:
2837                                 if (any_token_seen){
2838                                         if (caller_is_taking)
2839                                                 Error_TokensSeen ();
2840                                         return caller_is_taking;
2841                                 }
2842                                 PreProcessDefinition (true, arg, caller_is_taking);
2843                                 return caller_is_taking;
2844
2845                         case PreprocessorDirective.Undef:
2846                                 if (any_token_seen){
2847                                         if (caller_is_taking)
2848                                                 Error_TokensSeen ();
2849                                         return caller_is_taking;
2850                                 }
2851                                 PreProcessDefinition (false, arg, caller_is_taking);
2852                                 return caller_is_taking;
2853
2854                         case PreprocessorDirective.Invalid:
2855                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2856                                 return true;
2857                         }
2858
2859                         //
2860                         // These are only processed if we are in a `taking' block
2861                         //
2862                         if (!caller_is_taking)
2863                                 return false;
2864
2865                         switch (directive){
2866                         case PreprocessorDirective.Error:
2867                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2868                                 return true;
2869
2870                         case PreprocessorDirective.Warning:
2871                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2872                                 return true;
2873
2874                         case PreprocessorDirective.Pragma:
2875                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2876                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2877                                 }
2878
2879                                 ParsePragmaDirective ();
2880                                 return true;
2881
2882                         case PreprocessorDirective.Line:
2883                                 Location loc = Location;
2884                                 if (!PreProcessLine ())
2885                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
2886
2887                                 return caller_is_taking;
2888                         }
2889
2890                         throw new NotImplementedException (directive.ToString ());
2891                 }
2892
2893                 private int consume_string (bool quoted)
2894                 {
2895                         int c;
2896                         int pos = 0;
2897                         Location start_location = Location;
2898                         if (quoted)
2899                                 start_location = start_location - 1;
2900
2901 #if FULL_AST
2902                         int reader_pos = reader.Position;
2903 #endif
2904
2905                         while (true){
2906                                 // Cannot use get_char because of \r in quoted strings
2907                                 if (putback_char != -1) {
2908                                         c = putback_char;
2909                                         putback_char = -1;
2910                                 } else {
2911                                         c = reader.Read ();
2912                                 }
2913
2914                                 if (c == '"') {
2915                                         ++col;
2916
2917                                         if (quoted && peek_char () == '"') {
2918                                                 if (pos == value_builder.Length)
2919                                                         Array.Resize (ref value_builder, pos * 2);
2920
2921                                                 value_builder[pos++] = (char) c;
2922                                                 get_char ();
2923                                                 continue;
2924                                         }
2925
2926                                         string s;
2927                                         if (pos == 0)
2928                                                 s = string.Empty;
2929                                         else if (pos <= 4)
2930                                                 s = InternIdentifier (value_builder, pos);
2931                                         else
2932                                                 s = new string (value_builder, 0, pos);
2933
2934                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2935                                         val = res;
2936 #if FULL_AST
2937                                         res.ParsedValue = quoted ?
2938                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2939                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2940 #endif
2941
2942                                         return Token.LITERAL;
2943                                 }
2944
2945                                 if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2946                                         if (!quoted) {
2947                                                 Report.Error (1010, Location, "Newline in constant");
2948
2949                                                 advance_line ();
2950
2951                                                 // Don't add \r to string literal
2952                                                 if (pos > 1 && value_builder [pos - 1] == '\r')
2953                                                         --pos;
2954
2955                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
2956                                                 return Token.LITERAL;
2957                                         }
2958
2959                                         advance_line ();
2960                                 } else if (c == '\\' && !quoted) {
2961                                         ++col;
2962                                         int surrogate;
2963                                         c = escape (c, out surrogate);
2964                                         if (c == -1)
2965                                                 return Token.ERROR;
2966                                         if (surrogate != 0) {
2967                                                 if (pos == value_builder.Length)
2968                                                         Array.Resize (ref value_builder, pos * 2);
2969
2970                                                 value_builder[pos++] = (char) c;
2971                                                 c = surrogate;
2972                                         }
2973                                 } else if (c == -1) {
2974                                         Report.Error (1039, Location, "Unterminated string literal");
2975                                         return Token.EOF;
2976                                 } else {
2977                                         ++col;
2978                                 }
2979
2980                                 if (pos == value_builder.Length)
2981                                         Array.Resize (ref value_builder, pos * 2);
2982
2983                                 value_builder[pos++] = (char) c;
2984                         }
2985                 }
2986
2987                 private int consume_identifier (int s)
2988                 {
2989                         int res = consume_identifier (s, false);
2990
2991                         if (doc_state == XmlCommentState.Allowed)
2992                                 doc_state = XmlCommentState.NotAllowed;
2993
2994                         return res;
2995                 }
2996
2997                 int consume_identifier (int c, bool quoted)
2998                 {
2999                         //
3000                         // This method is very performance sensitive. It accounts
3001                         // for approximately 25% of all parser time
3002                         //
3003
3004                         int pos = 0;
3005                         int column = col;
3006                         if (quoted)
3007                                 --column;
3008
3009                         if (c == '\\') {
3010                                 int surrogate;
3011                                 c = escape (c, out surrogate);
3012                                 if (surrogate != 0) {
3013                                         id_builder [pos++] = (char) c;
3014                                         c = surrogate;
3015                                 }
3016                         }
3017
3018                         id_builder [pos++] = (char) c;
3019
3020                         try {
3021                                 while (true) {
3022                                         c = reader.Read ();
3023
3024                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
3025                                                 id_builder [pos++] = (char) c;
3026                                                 continue;
3027                                         }
3028
3029                                         if (c < 0x80) {
3030                                                 if (c == '\\') {
3031                                                         int surrogate;
3032                                                         c = escape (c, out surrogate);
3033                                                         if (is_identifier_part_character ((char) c))
3034                                                                 id_builder[pos++] = (char) c;
3035
3036                                                         if (surrogate != 0) {
3037                                                                 c = surrogate;
3038                                                         }
3039
3040                                                         continue;
3041                                                 }
3042                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
3043                                                 id_builder [pos++] = (char) c;
3044                                                 continue;
3045                                         }
3046
3047                                         putback_char = c;
3048                                         break;
3049                                 }
3050                         } catch (IndexOutOfRangeException) {
3051                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
3052                                 --pos;
3053                                 col += pos;
3054                         }
3055
3056                         col += pos - 1;
3057
3058                         //
3059                         // Optimization: avoids doing the keyword lookup
3060                         // on uppercase letters
3061                         //
3062                         if (id_builder [0] >= '_' && !quoted) {
3063                                 int keyword = GetKeyword (id_builder, pos);
3064                                 if (keyword != -1) {
3065                                         val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column);
3066                                         return keyword;
3067                                 }
3068                         }
3069
3070                         string s = InternIdentifier (id_builder, pos);
3071                         val = ltb.Create (s, current_source, ref_line, column);
3072                         if (quoted && parsing_attribute_section)
3073                                 AddEscapedIdentifier (((LocatedToken) val).Location);
3074
3075                         return Token.IDENTIFIER;
3076                 }
3077
3078                 string InternIdentifier (char[] charBuffer, int length)
3079                 {
3080                         //
3081                         // Keep identifiers in an array of hashtables to avoid needless
3082                         // allocations
3083                         //
3084                         var identifiers_group = identifiers[length];
3085                         string s;
3086                         if (identifiers_group != null) {
3087                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3088                                         return s;
3089                                 }
3090                         } else {
3091                                 // TODO: this should be number of files dependant
3092                                 // corlib compilation peaks at 1000 and System.Core at 150
3093                                 int capacity = length > 20 ? 10 : 100;
3094                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3095                                 identifiers[length] = identifiers_group;
3096                         }
3097
3098                         char[] chars = new char[length];
3099                         Array.Copy (charBuffer, chars, length);
3100
3101                         s = new string (charBuffer, 0, length);
3102                         identifiers_group.Add (chars, s);
3103                         return s;
3104                 }
3105
3106                 public int xtoken ()
3107                 {
3108                         int d, c;
3109
3110                         // Whether we have seen comments on the current line
3111                         bool comments_seen = false;
3112                         while ((c = get_char ()) != -1) {
3113                                 switch (c) {
3114                                 case '\t':
3115                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3116                                         continue;
3117
3118                                 case ' ':
3119                                 case '\f':
3120                                 case '\v':
3121                                 case 0xa0:
3122                                 case 0:
3123                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3124                                         continue;
3125
3126 /*                              This is required for compatibility with .NET
3127                                 case 0xEF:
3128                                         if (peek_char () == 0xBB) {
3129                                                 PushPosition ();
3130                                                 get_char ();
3131                                                 if (get_char () == 0xBF)
3132                                                         continue;
3133                                                 PopPosition ();
3134                                         }
3135                                         break;
3136 */
3137                                 case '\\':
3138                                         tokens_seen = true;
3139                                         return consume_identifier (c);
3140
3141                                 case '{':
3142                                         val = ltb.Create (current_source, ref_line, col);
3143                                         return Token.OPEN_BRACE;
3144                                 case '}':
3145                                         val = ltb.Create (current_source, ref_line, col);
3146                                         return Token.CLOSE_BRACE;
3147                                 case '[':
3148                                         // To block doccomment inside attribute declaration.
3149                                         if (doc_state == XmlCommentState.Allowed)
3150                                                 doc_state = XmlCommentState.NotAllowed;
3151
3152                                         val = ltb.Create (current_source, ref_line, col);
3153
3154                                         if (parsing_block == 0 || lambda_arguments_parsing)
3155                                                 return Token.OPEN_BRACKET;
3156
3157                                         int next = peek_char ();
3158                                         switch (next) {
3159                                         case ']':
3160                                         case ',':
3161                                                 return Token.OPEN_BRACKET;
3162
3163                                         case ' ':
3164                                         case '\f':
3165                                         case '\v':
3166                                         case '\r':
3167                                         case '\n':
3168                                         case UnicodeLS:
3169                                         case UnicodePS:
3170                                         case '/':
3171                                                 next = peek_token ();
3172                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3173                                                         return Token.OPEN_BRACKET;
3174
3175                                                 return Token.OPEN_BRACKET_EXPR;
3176                                         default:
3177                                                 return Token.OPEN_BRACKET_EXPR;
3178                                         }
3179                                 case ']':
3180                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3181                                         return Token.CLOSE_BRACKET;
3182                                 case '(':
3183                                         val = ltb.Create (current_source, ref_line, col);
3184                                         //
3185                                         // An expression versions of parens can appear in block context only
3186                                         //
3187                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3188
3189                                                 //
3190                                                 // Optmize most common case where we know that parens
3191                                                 // is not special
3192                                                 //
3193                                                 switch (current_token) {
3194                                                 case Token.IDENTIFIER:
3195                                                 case Token.IF:
3196                                                 case Token.FOR:
3197                                                 case Token.FOREACH:
3198                                                 case Token.TYPEOF:
3199                                                 case Token.WHILE:
3200                                                 case Token.SWITCH:
3201                                                 case Token.USING:
3202                                                 case Token.DEFAULT:
3203                                                 case Token.DELEGATE:
3204                                                 case Token.OP_GENERICS_GT:
3205                                                         return Token.OPEN_PARENS;
3206                                                 }
3207
3208                                                 // Optimize using peek
3209                                                 int xx = peek_char ();
3210                                                 switch (xx) {
3211                                                 case '(':
3212                                                 case '\'':
3213                                                 case '"':
3214                                                 case '0':
3215                                                 case '1':
3216                                                         return Token.OPEN_PARENS;
3217                                                 }
3218
3219                                                 lambda_arguments_parsing = true;
3220                                                 PushPosition ();
3221                                                 d = TokenizeOpenParens ();
3222                                                 PopPosition ();
3223                                                 lambda_arguments_parsing = false;
3224                                                 return d;
3225                                         }
3226
3227                                         return Token.OPEN_PARENS;
3228                                 case ')':
3229                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3230                                         return Token.CLOSE_PARENS;
3231                                 case ',':
3232                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3233                                         return Token.COMMA;
3234                                 case ';':
3235                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3236                                         return Token.SEMICOLON;
3237                                 case '~':
3238                                         val = ltb.Create (current_source, ref_line, col);
3239                                         return Token.TILDE;
3240                                 case '?':
3241                                         val = ltb.Create (current_source, ref_line, col);
3242                                         return TokenizePossibleNullableType ();
3243                                 case '<':
3244                                         val = ltb.Create (current_source, ref_line, col);
3245                                         if (parsing_generic_less_than++ > 0)
3246                                                 return Token.OP_GENERICS_LT;
3247
3248                                         return TokenizeLessThan ();
3249
3250                                 case '>':
3251                                         val = ltb.Create (current_source, ref_line, col);
3252                                         d = peek_char ();
3253
3254                                         if (d == '='){
3255                                                 get_char ();
3256                                                 return Token.OP_GE;
3257                                         }
3258
3259                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3260                                                 parsing_generic_less_than--;
3261                                                 return Token.OP_GENERICS_GT;
3262                                         }
3263
3264                                         if (d == '>') {
3265                                                 get_char ();
3266                                                 d = peek_char ();
3267
3268                                                 if (d == '=') {
3269                                                         get_char ();
3270                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3271                                                 }
3272                                                 return Token.OP_SHIFT_RIGHT;
3273                                         }
3274
3275                                         return Token.OP_GT;
3276
3277                                 case '+':
3278                                         val = ltb.Create (current_source, ref_line, col);
3279                                         d = peek_char ();
3280                                         if (d == '+') {
3281                                                 d = Token.OP_INC;
3282                                         } else if (d == '=') {
3283                                                 d = Token.OP_ADD_ASSIGN;
3284                                         } else {
3285                                                 return Token.PLUS;
3286                                         }
3287                                         get_char ();
3288                                         return d;
3289
3290                                 case '-':
3291                                         val = ltb.Create (current_source, ref_line, col);
3292                                         d = peek_char ();
3293                                         if (d == '-') {
3294                                                 d = Token.OP_DEC;
3295                                         } else if (d == '=')
3296                                                 d = Token.OP_SUB_ASSIGN;
3297                                         else if (d == '>')
3298                                                 d = Token.OP_PTR;
3299                                         else {
3300                                                 return Token.MINUS;
3301                                         }
3302                                         get_char ();
3303                                         return d;
3304
3305                                 case '!':
3306                                         val = ltb.Create (current_source, ref_line, col);
3307                                         if (peek_char () == '='){
3308                                                 get_char ();
3309                                                 return Token.OP_NE;
3310                                         }
3311                                         return Token.BANG;
3312
3313                                 case '=':
3314                                         val = ltb.Create (current_source, ref_line, col);
3315                                         d = peek_char ();
3316                                         if (d == '='){
3317                                                 get_char ();
3318                                                 return Token.OP_EQ;
3319                                         }
3320                                         if (d == '>'){
3321                                                 get_char ();
3322                                                 return Token.ARROW;
3323                                         }
3324
3325                                         return Token.ASSIGN;
3326
3327                                 case '&':
3328                                         val = ltb.Create (current_source, ref_line, col);
3329                                         d = peek_char ();
3330                                         if (d == '&'){
3331                                                 get_char ();
3332                                                 return Token.OP_AND;
3333                                         }
3334                                         if (d == '='){
3335                                                 get_char ();
3336                                                 return Token.OP_AND_ASSIGN;
3337                                         }
3338                                         return Token.BITWISE_AND;
3339
3340                                 case '|':
3341                                         val = ltb.Create (current_source, ref_line, col);
3342                                         d = peek_char ();
3343                                         if (d == '|'){
3344                                                 get_char ();
3345                                                 return Token.OP_OR;
3346                                         }
3347                                         if (d == '='){
3348                                                 get_char ();
3349                                                 return Token.OP_OR_ASSIGN;
3350                                         }
3351                                         return Token.BITWISE_OR;
3352
3353                                 case '*':
3354                                         val = ltb.Create (current_source, ref_line, col);
3355                                         if (peek_char () == '='){
3356                                                 get_char ();
3357                                                 return Token.OP_MULT_ASSIGN;
3358                                         }
3359                                         return Token.STAR;
3360
3361                                 case '/':
3362                                         d = peek_char ();
3363                                         if (d == '='){
3364                                                 val = ltb.Create (current_source, ref_line, col);
3365                                                 get_char ();
3366                                                 return Token.OP_DIV_ASSIGN;
3367                                         }
3368
3369                                         // Handle double-slash comments.
3370                                         if (d == '/'){
3371                                                 get_char ();
3372                                                 if (doc_processing) {
3373                                                         if (peek_char () == '/') {
3374                                                                 get_char ();
3375                                                                 // Don't allow ////.
3376                                                                 if ((d = peek_char ()) != '/') {
3377                                                                         if (doc_state == XmlCommentState.Allowed)
3378                                                                                 handle_one_line_xml_comment ();
3379                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3380                                                                                 WarningMisplacedComment (Location - 3);
3381                                                                 }
3382                                                         } else {
3383                                                                 if (xml_comment_buffer.Length > 0)
3384                                                                         doc_state = XmlCommentState.NotAllowed;
3385                                                         }
3386                                                 }
3387
3388                                                 ReadToEndOfLine ();
3389
3390                                                 any_token_seen |= tokens_seen;
3391                                                 tokens_seen = false;
3392                                                 comments_seen = false;
3393                                                 continue;
3394                                         } else if (d == '*'){
3395                                                 get_char ();
3396                                                 bool docAppend = false;
3397                                                 if (doc_processing && peek_char () == '*') {
3398                                                         get_char ();
3399                                                         // But when it is /**/, just do nothing.
3400                                                         if (peek_char () == '/') {
3401                                                                 get_char ();
3402                                                                 continue;
3403                                                         }
3404                                                         if (doc_state == XmlCommentState.Allowed)
3405                                                                 docAppend = true;
3406                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3407                                                                 WarningMisplacedComment (Location - 2);
3408                                                         }
3409                                                 }
3410
3411                                                 int current_comment_start = 0;
3412                                                 if (docAppend) {
3413                                                         current_comment_start = xml_comment_buffer.Length;
3414                                                         xml_comment_buffer.Append (Environment.NewLine);
3415                                                 }
3416
3417                                                 while ((d = get_char ()) != -1){
3418                                                         if (d == '*' && peek_char () == '/'){
3419                                                                 get_char ();
3420                                                                 comments_seen = true;
3421                                                                 break;
3422                                                         }
3423                                                         if (docAppend)
3424                                                                 xml_comment_buffer.Append ((char) d);
3425
3426                                                         if (d == '\n' || d == UnicodeLS || d == UnicodePS){
3427                                                                 any_token_seen |= tokens_seen;
3428                                                                 tokens_seen = false;
3429                                                                 //
3430                                                                 // Reset 'comments_seen' just to be consistent.
3431                                                                 // It doesn't matter either way, here.
3432                                                                 //
3433                                                                 comments_seen = false;
3434                                                         }
3435                                                 }
3436                                                 if (!comments_seen)
3437                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3438
3439                                                 if (docAppend)
3440                                                         update_formatted_doc_comment (current_comment_start);
3441                                                 continue;
3442                                         }
3443                                         val = ltb.Create (current_source, ref_line, col);
3444                                         return Token.DIV;
3445
3446                                 case '%':
3447                                         val = ltb.Create (current_source, ref_line, col);
3448                                         if (peek_char () == '='){
3449                                                 get_char ();
3450                                                 return Token.OP_MOD_ASSIGN;
3451                                         }
3452                                         return Token.PERCENT;
3453
3454                                 case '^':
3455                                         val = ltb.Create (current_source, ref_line, col);
3456                                         if (peek_char () == '='){
3457                                                 get_char ();
3458                                                 return Token.OP_XOR_ASSIGN;
3459                                         }
3460                                         return Token.CARRET;
3461
3462                                 case ':':
3463                                         val = ltb.Create (current_source, ref_line, col);
3464                                         if (peek_char () == ':') {
3465                                                 get_char ();
3466                                                 return Token.DOUBLE_COLON;
3467                                         }
3468                                         return Token.COLON;
3469
3470                                 case '0': case '1': case '2': case '3': case '4':
3471                                 case '5': case '6': case '7': case '8': case '9':
3472                                         tokens_seen = true;
3473                                         return is_number (c, false);
3474
3475                                 case '\n': // white space
3476                                 case UnicodeLS:
3477                                 case UnicodePS:
3478                                         any_token_seen |= tokens_seen;
3479                                         tokens_seen = false;
3480                                         comments_seen = false;
3481                                         continue;
3482
3483                                 case '.':
3484                                         tokens_seen = true;
3485                                         d = peek_char ();
3486                                         if (d >= '0' && d <= '9')
3487                                                 return is_number (c, true);
3488
3489                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3490                                         return Token.DOT;
3491
3492                                 case '#':
3493                                         if (tokens_seen || comments_seen) {
3494                                                 Eror_WrongPreprocessorLocation ();
3495                                                 return Token.ERROR;
3496                                         }
3497
3498                                         if (ParsePreprocessingDirective (true))
3499                                                 continue;
3500
3501                                         bool directive_expected = false;
3502                                         while ((c = get_char ()) != -1) {
3503                                                 if (col == 1) {
3504                                                         directive_expected = true;
3505                                                 } else if (!directive_expected) {
3506                                                         // TODO: Implement comment support for disabled code and uncomment this code
3507 //                                                      if (c == '#') {
3508 //                                                              Eror_WrongPreprocessorLocation ();
3509 //                                                              return Token.ERROR;
3510 //                                                      }
3511                                                         continue;
3512                                                 }
3513
3514                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
3515                                                         continue;
3516
3517                                                 if (c == '#') {
3518                                                         if (ParsePreprocessingDirective (false))
3519                                                                 break;
3520                                                 }
3521                                                 directive_expected = false;
3522                                         }
3523
3524                                         if (c != -1) {
3525                                                 tokens_seen = false;
3526                                                 continue;
3527                                         }
3528
3529                                         return Token.EOF;
3530
3531                                 case '"':
3532                                         return consume_string (false);
3533
3534                                 case '\'':
3535                                         return TokenizeBackslash ();
3536
3537                                 case '@':
3538                                         c = get_char ();
3539                                         if (c == '"') {
3540                                                 tokens_seen = true;
3541                                                 return consume_string (true);
3542                                         }
3543
3544                                         if (is_identifier_start_character (c)){
3545                                                 return consume_identifier (c, true);
3546                                         }
3547
3548                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3549                                         return Token.ERROR;
3550
3551                                 case EvalStatementParserCharacter:
3552                                         return Token.EVAL_STATEMENT_PARSER;
3553                                 case EvalCompilationUnitParserCharacter:
3554                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3555                                 case EvalUsingDeclarationsParserCharacter:
3556                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3557                                 case DocumentationXref:
3558                                         return Token.DOC_SEE;
3559                                 }
3560
3561                                 if (is_identifier_start_character (c)) {
3562                                         tokens_seen = true;
3563                                         return consume_identifier (c);
3564                                 }
3565
3566                                 if (char.IsWhiteSpace ((char) c))
3567                                         continue;
3568
3569                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3570                         }
3571
3572                         if (CompleteOnEOF){
3573                                 if (generated)
3574                                         return Token.COMPLETE_COMPLETION;
3575
3576                                 generated = true;
3577                                 return Token.GENERATE_COMPLETION;
3578                         }
3579
3580
3581                         return Token.EOF;
3582                 }
3583
3584                 int TokenizeBackslash ()
3585                 {
3586 #if FULL_AST
3587                         int read_start = reader.Position;
3588 #endif
3589                         Location start_location = Location;
3590                         int c = get_char ();
3591                         tokens_seen = true;
3592                         if (c == '\'') {
3593                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3594                                 Report.Error (1011, start_location, "Empty character literal");
3595                                 return Token.LITERAL;
3596                         }
3597
3598                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3599                                 Report.Error (1010, start_location, "Newline in constant");
3600                                 return Token.ERROR;
3601                         }
3602
3603                         int d;
3604                         c = escape (c, out d);
3605                         if (c == -1)
3606                                 return Token.ERROR;
3607                         if (d != 0)
3608                                 throw new NotImplementedException ();
3609
3610                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3611                         val = res;
3612                         c = get_char ();
3613
3614                         if (c != '\'') {
3615                                 Report.Error (1012, start_location, "Too many characters in character literal");
3616
3617                                 // Try to recover, read until newline or next "'"
3618                                 while ((c = get_char ()) != -1) {
3619                                         if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
3620                                                 break;
3621                                 }
3622                         }
3623
3624 #if FULL_AST
3625                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3626 #endif
3627
3628                         return Token.LITERAL;
3629                 }
3630
3631                 int TokenizeLessThan ()
3632                 {
3633                         int d;
3634                         if (handle_typeof) {
3635                                 PushPosition ();
3636                                 if (parse_generic_dimension (out d)) {
3637                                         val = d;
3638                                         DiscardPosition ();
3639                                         return Token.GENERIC_DIMENSION;
3640                                 }
3641                                 PopPosition ();
3642                         }
3643
3644                         // Save current position and parse next token.
3645                         PushPosition ();
3646                         if (parse_less_than ()) {
3647                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3648                                         d = Token.OP_GENERICS_LT_DECL;
3649                                 } else {
3650                                         d = Token.OP_GENERICS_LT;
3651                                 }
3652                                 PopPosition ();
3653                                 return d;
3654                         }
3655
3656                         PopPosition ();
3657                         parsing_generic_less_than = 0;
3658
3659                         d = peek_char ();
3660                         if (d == '<') {
3661                                 get_char ();
3662                                 d = peek_char ();
3663
3664                                 if (d == '=') {
3665                                         get_char ();
3666                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3667                                 }
3668                                 return Token.OP_SHIFT_LEFT;
3669                         }
3670
3671                         if (d == '=') {
3672                                 get_char ();
3673                                 return Token.OP_LE;
3674                         }
3675                         return Token.OP_LT;
3676                 }
3677
3678                 //
3679                 // Handles one line xml comment
3680                 //
3681                 private void handle_one_line_xml_comment ()
3682                 {
3683                         int c;
3684                         while ((c = peek_char ()) == ' ')
3685                                 get_char (); // skip heading whitespaces.
3686                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3687                                 xml_comment_buffer.Append ((char) get_char ());
3688                         }
3689                         if (c == '\r' || c == '\n')
3690                                 xml_comment_buffer.Append (Environment.NewLine);
3691                 }
3692
3693                 //
3694                 // Remove heading "*" in Javadoc-like xml documentation.
3695                 //
3696                 private void update_formatted_doc_comment (int current_comment_start)
3697                 {
3698                         int length = xml_comment_buffer.Length - current_comment_start;
3699                         string [] lines = xml_comment_buffer.ToString (
3700                                 current_comment_start,
3701                                 length).Replace ("\r", "").Split ('\n');
3702
3703                         // The first line starts with /**, thus it is not target
3704                         // for the format check.
3705                         for (int i = 1; i < lines.Length; i++) {
3706                                 string s = lines [i];
3707                                 int idx = s.IndexOf ('*');
3708                                 string head = null;
3709                                 if (idx < 0) {
3710                                         if (i < lines.Length - 1)
3711                                                 return;
3712                                         head = s;
3713                                 } else
3714                                         head = s.Substring (0, idx);
3715                                 foreach (char c in head)
3716                                         if (c != ' ')
3717                                                 return;
3718                                 lines [i] = s.Substring (idx + 1);
3719                         }
3720                         xml_comment_buffer.Remove (current_comment_start, length);
3721                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3722                 }
3723
3724                 //
3725                 // Checks if there was incorrect doc comments and raise
3726                 // warnings.
3727                 //
3728                 public void check_incorrect_doc_comment ()
3729                 {
3730                         if (xml_comment_buffer.Length > 0)
3731                                 WarningMisplacedComment (Location);
3732                 }
3733
3734                 //
3735                 // Consumes the saved xml comment lines (if any)
3736                 // as for current target member or type.
3737                 //
3738                 public string consume_doc_comment ()
3739                 {
3740                         if (xml_comment_buffer.Length > 0) {
3741                                 string ret = xml_comment_buffer.ToString ();
3742                                 reset_doc_comment ();
3743                                 return ret;
3744                         }
3745                         return null;
3746                 }
3747
3748                 void reset_doc_comment ()
3749                 {
3750                         xml_comment_buffer.Length = 0;
3751                 }
3752
3753                 public void cleanup ()
3754                 {
3755                         if (ifstack != null && ifstack.Count >= 1) {
3756                                 int state = ifstack.Pop ();
3757                                 if ((state & REGION) != 0)
3758                                         Report.Error (1038, Location, "#endregion directive expected");
3759                                 else
3760                                         Report.Error (1027, Location, "Expected `#endif' directive");
3761                         }
3762                 }
3763         }
3764
3765         //
3766         // Indicates whether it accepts XML documentation or not.
3767         //
3768         public enum XmlCommentState {
3769                 // comment is allowed in this state.
3770                 Allowed,
3771                 // comment is not allowed in this state.
3772                 NotAllowed,
3773                 // once comments appeared when it is NotAllowed, then the
3774                 // state is changed to it, until the state is changed to
3775                 // .Allowed.
3776                 Error
3777         }
3778 }
3779