mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20 using System.Collections;
  21
  22 namespace Mono.CSharp
  23 {
  24         //
  25         // This class has to be used by parser only, it reuses token
  26         // details once a file is parsed
  27         //
  28         public class LocatedToken
  29         {
  30                 public int row, column;
  31                 public string value;
  32                 public SourceFile file;
  33
  34                 public LocatedToken ()
  35                 {
  36                 }
  37
  38                 public LocatedToken (string value, Location loc)
  39                 {
  40                         this.value = value;
  41                         file = loc.SourceFile;
  42                         row = loc.Row;
  43                         column = loc.Column;
  44                 }
  45
  46                 public override string ToString ()
  47                 {
  48                         return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  49                 }
  50
  51                 public Location Location
  52                 {
  53                         get { return new Location (file, row, column); }
  54                 }
  55
  56                 public string Value
  57                 {
  58                         get { return value; }
  59                 }
  60         }
  61
  62         /// <summary>
  63         ///    Tokenizer for C# source code.
  64         /// </summary>
  65         public class Tokenizer : yyParser.yyInput
  66         {
  67                 class KeywordEntry<T>
  68                 {
  69                         public readonly T Token;
  70                         public KeywordEntry<T> Next;
  71                         public readonly char[] Value;
  72
  73                         public KeywordEntry (string value, T token)
  74                         {
  75                                 this.Value = value.ToCharArray ();
  76                                 this.Token = token;
  77                         }
  78                 }
  79
  80                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  81                 {
  82                         readonly int length;
  83
  84                         public IdentifiersComparer (int length)
  85                         {
  86                                 this.length = length;
  87                         }
  88
  89                         public bool Equals (char[] x, char[] y)
  90                         {
  91                                 for (int i = 0; i < length; ++i)
  92                                         if (x [i] != y [i])
  93                                                 return false;
  94
  95                                 return true;
  96                         }
  97
  98                         public int GetHashCode (char[] obj)
  99                         {
 100                                 int h = 0;
 101                                 for (int i = 0; i < length; ++i)
 102                                         h = (h << 5) - h + obj [i];
 103
 104                                 return h;
 105                         }
 106                 }
 107
 108                 public class LocatedTokenBuffer
 109                 {
 110                         readonly LocatedToken[] buffer;
 111                         public int pos;
 112
 113                         public LocatedTokenBuffer ()
 114                         {
 115                                 buffer = new LocatedToken[0];
 116                         }
 117
 118                         public LocatedTokenBuffer (LocatedToken[] buffer)
 119                         {
 120                                 this.buffer = buffer ?? new LocatedToken[0];
 121                         }
 122
 123                         public LocatedToken Create (SourceFile file, int row, int column)
 124                         {
 125                                 return Create (null, file, row, column);
 126                         }
 127
 128                         public LocatedToken Create (string value, SourceFile file, int row, int column)
 129                         {
 130                                 //
 131                                 // TODO: I am not very happy about the logic but it's the best
 132                                 // what I could come up with for now.
 133                                 // Ideally we should be using just tiny buffer (256 elements) which
 134                                 // is enough to hold all details for currect stack and recycle elements
 135                                 // poped from the stack but there is a trick needed to recycle
 136                                 // them properly.
 137                                 //
 138                                 LocatedToken entry;
 139                                 if (pos >= buffer.Length) {
 140                                         entry = new LocatedToken ();
 141                                 } else {
 142                                         entry = buffer[pos];
 143                                         if (entry == null) {
 144                                                 entry = new LocatedToken ();
 145                                                 buffer[pos] = entry;
 146                                         }
 147
 148                                         ++pos;
 149                                 }
 150                                 entry.value = value;
 151                                 entry.file = file;
 152                                 entry.row = row;
 153                                 entry.column = column;
 154                                 return entry;
 155                         }
 156
 157                         //
 158                         // Used for token not required by expression evaluator
 159                         //
 160                         [Conditional ("FULL_AST")]
 161                         public void CreateOptional (SourceFile file, int row, int col, ref object token)
 162                         {
 163                                 token = Create (file, row, col);
 164                         }
 165                 }
 166
 167                 public enum PreprocessorDirective
 168                 {
 169                         Invalid = 0,
 170
 171                         Region = 1,
 172                         Endregion = 2,
 173                         If = 3 | RequiresArgument,
 174                         Endif = 4,
 175                         Elif = 5 | RequiresArgument,
 176                         Else = 6,
 177                         Define = 7 | RequiresArgument,
 178                         Undef = 8 | RequiresArgument,
 179                         Error = 9,
 180                         Warning = 10,
 181                         Pragma = 11 | CustomArgumentsParsing,
 182                         Line = 12 | CustomArgumentsParsing,
 183
 184                         CustomArgumentsParsing = 1 << 10,
 185                         RequiresArgument = 1 << 11
 186                 }
 187
 188                 readonly SeekableStreamReader reader;
 189                 readonly CompilationSourceFile source_file;
 190                 readonly CompilerContext context;
 191                 readonly Report Report;
 192
 193
 194                 SourceFile current_source;
 195                 Location hidden_block_start;
 196                 int ref_line = 1;
 197                 int line = 1;
 198                 int col = 0;
 199                 int previous_col;
 200                 int current_token;
 201                 readonly int tab_size;
 202                 bool handle_get_set = false;
 203                 bool handle_remove_add = false;
 204                 bool handle_where;
 205                 bool lambda_arguments_parsing;
 206                 List<Location> escaped_identifiers;
 207                 int parsing_generic_less_than;
 208                 readonly bool doc_processing;
 209                 readonly LocatedTokenBuffer ltb;
 210
 211                 //
 212                 // Used mainly for parser optimizations. Some expressions for instance
 213                 // can appear only in block (including initializer, base initializer)
 214                 // scope only
 215                 //
 216                 public int parsing_block;
 217                 internal bool query_parsing;
 218
 219                 //
 220                 // When parsing type only, useful for ambiguous nullable types
 221                 //
 222                 public int parsing_type;
 223
 224                 //
 225                 // Set when parsing generic declaration (type or method header)
 226                 //
 227                 public bool parsing_generic_declaration;
 228                 public bool parsing_generic_declaration_doc;
 229
 230                 //
 231                 // The value indicates that we have not reach any declaration or
 232                 // namespace yet
 233                 //
 234                 public int parsing_declaration;
 235
 236                 public bool parsing_attribute_section;
 237
 238                 public bool parsing_modifiers;
 239
 240                 public bool parsing_catch_when;
 241
 242                 int parsing_string_interpolation;
 243
 244                 public bool parsing_interpolation_format;
 245
 246                 //
 247                 // The special characters to inject on streams to run the unit parser
 248                 // in the special expression mode. Using private characters from
 249                 // Plane Sixteen (U+100000 to U+10FFFD)
 250                 //
 251                 // This character is only tested just before the tokenizer is about to report
 252                 // an error;   So on the regular operation mode, this addition will have no
 253                 // impact on the tokenizer's performance.
 254                 //
 255
 256                 public const int EvalStatementParserCharacter = 0x100000;
 257                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 258                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 259                 public const int DocumentationXref = 0x100003;
 260
 261                 const int UnicodeLS = 0x2028;
 262                 const int UnicodePS = 0x2029;
 263
 264                 //
 265                 // XML documentation buffer. The save point is used to divide
 266                 // comments on types and comments on members.
 267                 //
 268                 StringBuilder xml_comment_buffer;
 269
 270                 //
 271                 // See comment on XmlCommentState enumeration.
 272                 //
 273                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 274
 275                 //
 276                 // Whether tokens have been seen on this line
 277                 //
 278                 bool tokens_seen = false;
 279
 280                 //
 281                 // Set to true once the GENERATE_COMPLETION token has bee
 282                 // returned.   This helps produce one GENERATE_COMPLETION,
 283                 // as many COMPLETE_COMPLETION as necessary to complete the
 284                 // AST tree and one final EOF.
 285                 //
 286                 bool generated;
 287
 288                 //
 289                 // Whether a token has been seen on the file
 290                 // This is needed because `define' is not allowed to be used
 291                 // after a token has been seen.
 292                 //
 293                 bool any_token_seen;
 294
 295                 //
 296                 // Class variables
 297                 //
 298                 static readonly KeywordEntry<int>[][] keywords;
 299                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 300                 static readonly HashSet<string> keyword_strings;
 301                 static readonly NumberStyles styles;
 302                 static readonly NumberFormatInfo csharp_format_info;
 303
 304                 // Pragma arguments
 305                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 306                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 307                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 308                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 309                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 310                 static readonly char[] line_default = "default".ToCharArray ();
 311
 312                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 313
 314                 public bool PropertyParsing {
 315                         get { return handle_get_set; }
 316                         set { handle_get_set = value; }
 317                 }
 318
 319                 public bool EventParsing {
 320                         get { return handle_remove_add; }
 321                         set { handle_remove_add = value; }
 322                 }
 323
 324                 public bool ConstraintsParsing {
 325                         get { return handle_where; }
 326                         set { handle_where = value; }
 327                 }
 328
 329                 public XmlCommentState doc_state {
 330                         get { return xml_doc_state; }
 331                         set {
 332                                 if (value == XmlCommentState.Allowed) {
 333                                         check_incorrect_doc_comment ();
 334                                         reset_doc_comment ();
 335                                 }
 336                                 xml_doc_state = value;
 337                         }
 338                 }
 339
 340                 //
 341                 // This is used to trigger completion generation on the parser
 342                 public bool CompleteOnEOF;
 343
 344                 void AddEscapedIdentifier (Location loc)
 345                 {
 346                         if (escaped_identifiers == null)
 347                                 escaped_identifiers = new List<Location> ();
 348
 349                         escaped_identifiers.Add (loc);
 350                 }
 351
 352                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 353                 {
 354                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 355                 }
 356
 357                 //
 358                 // Values for the associated token returned
 359                 //
 360                 internal int putback_char;      // Used by repl only
 361                 object val;
 362
 363                 //
 364                 // Pre-processor
 365                 //
 366                 const int TAKING        = 1;
 367                 const int ELSE_SEEN     = 4;
 368                 const int PARENT_TAKING = 8;
 369                 const int REGION        = 16;
 370
 371                 //
 372                 // pre-processor if stack state:
 373                 //
 374                 Stack<int> ifstack;
 375
 376                 public const int MaxIdentifierLength = 512;
 377                 public const int MaxNumberLength = 512;
 378
 379                 readonly char[] id_builder;
 380                 readonly Dictionary<char[], string>[] identifiers;
 381                 readonly char[] number_builder;
 382                 int number_pos;
 383
 384                 char[] value_builder = new char[64];
 385
 386                 public int Line {
 387                         get {
 388                                 return ref_line;
 389                         }
 390                 }
 391
 392                 //
 393                 // This is used when the tokenizer needs to save
 394                 // the current position as it needs to do some parsing
 395                 // on its own to deamiguate a token in behalf of the
 396                 // parser.
 397                 //
 398                 Stack<Position> position_stack = new Stack<Position> (2);
 399
 400                 class Position {
 401                         public int position;
 402                         public int line;
 403                         public int ref_line;
 404                         public int col;
 405                         public Location hidden;
 406                         public int putback_char;
 407                         public int previous_col;
 408                         public Stack<int> ifstack;
 409                         public int parsing_generic_less_than;
 410                         public int current_token;
 411                         public object val;
 412                         public int parsing_string_interpolation;
 413
 414                         public Position (Tokenizer t)
 415                         {
 416                                 position = t.reader.Position;
 417                                 line = t.line;
 418                                 ref_line = t.ref_line;
 419                                 col = t.col;
 420                                 hidden = t.hidden_block_start;
 421                                 putback_char = t.putback_char;
 422                                 previous_col = t.previous_col;
 423                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 424                                         // There is no simple way to clone Stack<T> all
 425                                         // methods reverse the order
 426                                         var clone = t.ifstack.ToArray ();
 427                                         Array.Reverse (clone);
 428                                         ifstack = new Stack<int> (clone);
 429                                 }
 430                                 parsing_generic_less_than = t.parsing_generic_less_than;
 431                                 parsing_string_interpolation = t.parsing_string_interpolation;
 432                                 current_token = t.current_token;
 433                                 val = t.val;
 434                         }
 435                 }
 436
 437                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 438                 {
 439                         this.source_file = file;
 440                         this.context = file.Compiler;
 441                         this.current_source = file.SourceFile;
 442                         this.identifiers = session.Identifiers;
 443                         this.id_builder = session.IDBuilder;
 444                         this.number_builder = session.NumberBuilder;
 445                         this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 446                         this.Report = report;
 447
 448                         reader = input;
 449
 450                         putback_char = -1;
 451
 452                         xml_comment_buffer = new StringBuilder ();
 453                         doc_processing = context.Settings.DocumentationFile != null;
 454
 455                         tab_size = context.Settings.TabSize;
 456                 }
 457
 458                 public void PushPosition ()
 459                 {
 460                         position_stack.Push (new Position (this));
 461                 }
 462
 463                 public void PopPosition ()
 464                 {
 465                         Position p = position_stack.Pop ();
 466
 467                         reader.Position = p.position;
 468                         ref_line = p.ref_line;
 469                         line = p.line;
 470                         col = p.col;
 471                         hidden_block_start = p.hidden;
 472                         putback_char = p.putback_char;
 473                         previous_col = p.previous_col;
 474                         ifstack = p.ifstack;
 475                         parsing_generic_less_than = p.parsing_generic_less_than;
 476                         parsing_string_interpolation = p.parsing_string_interpolation;
 477                         current_token = p.current_token;
 478                         val = p.val;
 479                 }
 480
 481                 // Do not reset the position, ignore it.
 482                 public void DiscardPosition ()
 483                 {
 484                         position_stack.Pop ();
 485                 }
 486
 487                 static void AddKeyword (string kw, int token)
 488                 {
 489                         keyword_strings.Add (kw);
 490
 491                         AddKeyword (keywords, kw, token);
 492                 }
 493
 494                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 495                 {
 496                         AddKeyword (keywords_preprocessor, kw, directive);
 497                 }
 498
 499                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 500                 {
 501                         int length = kw.Length;
 502                         if (keywords[length] == null) {
 503                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 504                         }
 505
 506                         int char_index = kw[0] - '_';
 507                         var kwe = keywords[length][char_index];
 508                         if (kwe == null) {
 509                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 510                                 return;
 511                         }
 512
 513                         while (kwe.Next != null) {
 514                                 kwe = kwe.Next;
 515                         }
 516
 517                         kwe.Next = new KeywordEntry<T> (kw, token);
 518                 }
 519
 520                 //
 521                 // Class initializer
 522                 //
 523                 static Tokenizer ()
 524                 {
 525                         keyword_strings = new HashSet<string> ();
 526
 527                         // 11 is the length of the longest keyword for now
 528                         keywords = new KeywordEntry<int>[11][];
 529
 530                         AddKeyword ("__arglist", Token.ARGLIST);
 531                         AddKeyword ("__makeref", Token.MAKEREF);
 532                         AddKeyword ("__reftype", Token.REFTYPE);
 533                         AddKeyword ("__refvalue", Token.REFVALUE);
 534                         AddKeyword ("abstract", Token.ABSTRACT);
 535                         AddKeyword ("as", Token.AS);
 536                         AddKeyword ("add", Token.ADD);
 537                         AddKeyword ("base", Token.BASE);
 538                         AddKeyword ("bool", Token.BOOL);
 539                         AddKeyword ("break", Token.BREAK);
 540                         AddKeyword ("byte", Token.BYTE);
 541                         AddKeyword ("case", Token.CASE);
 542                         AddKeyword ("catch", Token.CATCH);
 543                         AddKeyword ("char", Token.CHAR);
 544                         AddKeyword ("checked", Token.CHECKED);
 545                         AddKeyword ("class", Token.CLASS);
 546                         AddKeyword ("const", Token.CONST);
 547                         AddKeyword ("continue", Token.CONTINUE);
 548                         AddKeyword ("decimal", Token.DECIMAL);
 549                         AddKeyword ("default", Token.DEFAULT);
 550                         AddKeyword ("delegate", Token.DELEGATE);
 551                         AddKeyword ("do", Token.DO);
 552                         AddKeyword ("double", Token.DOUBLE);
 553                         AddKeyword ("else", Token.ELSE);
 554                         AddKeyword ("enum", Token.ENUM);
 555                         AddKeyword ("event", Token.EVENT);
 556                         AddKeyword ("explicit", Token.EXPLICIT);
 557                         AddKeyword ("extern", Token.EXTERN);
 558                         AddKeyword ("false", Token.FALSE);
 559                         AddKeyword ("finally", Token.FINALLY);
 560                         AddKeyword ("fixed", Token.FIXED);
 561                         AddKeyword ("float", Token.FLOAT);
 562                         AddKeyword ("for", Token.FOR);
 563                         AddKeyword ("foreach", Token.FOREACH);
 564                         AddKeyword ("goto", Token.GOTO);
 565                         AddKeyword ("get", Token.GET);
 566                         AddKeyword ("if", Token.IF);
 567                         AddKeyword ("implicit", Token.IMPLICIT);
 568                         AddKeyword ("in", Token.IN);
 569                         AddKeyword ("int", Token.INT);
 570                         AddKeyword ("interface", Token.INTERFACE);
 571                         AddKeyword ("internal", Token.INTERNAL);
 572                         AddKeyword ("is", Token.IS);
 573                         AddKeyword ("lock", Token.LOCK);
 574                         AddKeyword ("long", Token.LONG);
 575                         AddKeyword ("namespace", Token.NAMESPACE);
 576                         AddKeyword ("new", Token.NEW);
 577                         AddKeyword ("null", Token.NULL);
 578                         AddKeyword ("object", Token.OBJECT);
 579                         AddKeyword ("operator", Token.OPERATOR);
 580                         AddKeyword ("out", Token.OUT);
 581                         AddKeyword ("override", Token.OVERRIDE);
 582                         AddKeyword ("params", Token.PARAMS);
 583                         AddKeyword ("private", Token.PRIVATE);
 584                         AddKeyword ("protected", Token.PROTECTED);
 585                         AddKeyword ("public", Token.PUBLIC);
 586                         AddKeyword ("readonly", Token.READONLY);
 587                         AddKeyword ("ref", Token.REF);
 588                         AddKeyword ("remove", Token.REMOVE);
 589                         AddKeyword ("return", Token.RETURN);
 590                         AddKeyword ("sbyte", Token.SBYTE);
 591                         AddKeyword ("sealed", Token.SEALED);
 592                         AddKeyword ("set", Token.SET);
 593                         AddKeyword ("short", Token.SHORT);
 594                         AddKeyword ("sizeof", Token.SIZEOF);
 595                         AddKeyword ("stackalloc", Token.STACKALLOC);
 596                         AddKeyword ("static", Token.STATIC);
 597                         AddKeyword ("string", Token.STRING);
 598                         AddKeyword ("struct", Token.STRUCT);
 599                         AddKeyword ("switch", Token.SWITCH);
 600                         AddKeyword ("this", Token.THIS);
 601                         AddKeyword ("throw", Token.THROW);
 602                         AddKeyword ("true", Token.TRUE);
 603                         AddKeyword ("try", Token.TRY);
 604                         AddKeyword ("typeof", Token.TYPEOF);
 605                         AddKeyword ("uint", Token.UINT);
 606                         AddKeyword ("ulong", Token.ULONG);
 607                         AddKeyword ("unchecked", Token.UNCHECKED);
 608                         AddKeyword ("unsafe", Token.UNSAFE);
 609                         AddKeyword ("ushort", Token.USHORT);
 610                         AddKeyword ("using", Token.USING);
 611                         AddKeyword ("virtual", Token.VIRTUAL);
 612                         AddKeyword ("void", Token.VOID);
 613                         AddKeyword ("volatile", Token.VOLATILE);
 614                         AddKeyword ("while", Token.WHILE);
 615                         AddKeyword ("partial", Token.PARTIAL);
 616                         AddKeyword ("where", Token.WHERE);
 617
 618                         // LINQ keywords
 619                         AddKeyword ("from", Token.FROM);
 620                         AddKeyword ("join", Token.JOIN);
 621                         AddKeyword ("on", Token.ON);
 622                         AddKeyword ("equals", Token.EQUALS);
 623                         AddKeyword ("select", Token.SELECT);
 624                         AddKeyword ("group", Token.GROUP);
 625                         AddKeyword ("by", Token.BY);
 626                         AddKeyword ("let", Token.LET);
 627                         AddKeyword ("orderby", Token.ORDERBY);
 628                         AddKeyword ("ascending", Token.ASCENDING);
 629                         AddKeyword ("descending", Token.DESCENDING);
 630                         AddKeyword ("into", Token.INTO);
 631
 632                         // Contextual async keywords
 633                         AddKeyword ("async", Token.ASYNC);
 634                         AddKeyword ("await", Token.AWAIT);
 635
 636                         // Contextual filter catch keyword
 637                         AddKeyword ("when", Token.WHEN);
 638
 639                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 640
 641                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 642                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 643                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 644                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 645                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 646                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 647                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 648                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 649                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 650                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 651                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 652                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 653
 654                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 655                         styles = NumberStyles.Float;
 656                 }
 657
 658                 int GetKeyword (char[] id, int id_len)
 659                 {
 660                         //
 661                         // Keywords are stored in an array of arrays grouped by their
 662                         // length and then by the first character
 663                         //
 664                         if (id_len >= keywords.Length || keywords [id_len] == null)
 665                                 return -1;
 666
 667                         int first_index = id [0] - '_';
 668                         if (first_index > 'z' - '_')
 669                                 return -1;
 670
 671                         var kwe = keywords [id_len] [first_index];
 672                         if (kwe == null)
 673                                 return -1;
 674
 675                         int res;
 676                         do {
 677                                 res = kwe.Token;
 678                                 for (int i = 1; i < id_len; ++i) {
 679                                         if (id [i] != kwe.Value [i]) {
 680                                                 res = 0;
 681                                                 kwe = kwe.Next;
 682                                                 break;
 683                                         }
 684                                 }
 685                         } while (res == 0 && kwe != null);
 686
 687                         if (res == 0)
 688                                 return -1;
 689
 690                         int next_token;
 691                         switch (res) {
 692                         case Token.GET:
 693                         case Token.SET:
 694                                 if (!handle_get_set)
 695                                         res = -1;
 696                                 break;
 697                         case Token.REMOVE:
 698                         case Token.ADD:
 699                                 if (!handle_remove_add)
 700                                         res = -1;
 701                                 break;
 702                         case Token.EXTERN:
 703                                 if (parsing_declaration == 0)
 704                                         res = Token.EXTERN_ALIAS;
 705                                 break;
 706                         case Token.DEFAULT:
 707                                 if (peek_token () == Token.COLON) {
 708                                         token ();
 709                                         res = Token.DEFAULT_COLON;
 710                                 }
 711                                 break;
 712                         case Token.WHEN:
 713                                 if (current_token != Token.CATCH && !parsing_catch_when)
 714                                         res = -1;
 715                                 break;
 716                         case Token.WHERE:
 717                                 if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 718                                         res = -1;
 719                                 break;
 720                         case Token.FROM:
 721                                 //
 722                                 // A query expression is any expression that starts with `from identifier'
 723                                 // followed by any token except ; , =
 724                                 //
 725                                 if (!query_parsing) {
 726                                         if (lambda_arguments_parsing || parsing_block == 0) {
 727                                                 res = -1;
 728                                                 break;
 729                                         }
 730
 731                                         PushPosition ();
 732                                         // HACK: to disable generics micro-parser, because PushPosition does not
 733                                         // store identifiers array
 734                                         parsing_generic_less_than = 1;
 735                                         switch (xtoken ()) {
 736                                         case Token.IDENTIFIER:
 737                                         case Token.INT:
 738                                         case Token.BOOL:
 739                                         case Token.BYTE:
 740                                         case Token.CHAR:
 741                                         case Token.DECIMAL:
 742                                         case Token.DOUBLE:
 743                                         case Token.FLOAT:
 744                                         case Token.LONG:
 745                                         case Token.OBJECT:
 746                                         case Token.STRING:
 747                                         case Token.UINT:
 748                                         case Token.ULONG:
 749                                                 next_token = xtoken ();
 750                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 751                                                         goto default;
 752
 753                                                 res = Token.FROM_FIRST;
 754                                                 query_parsing = true;
 755                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 756                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 757                                                 break;
 758                                         case Token.VOID:
 759                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 760                                                 break;
 761                                         default:
 762                                                 PopPosition ();
 763                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 764                                                 // which has been overwritten before we grabbed the identifier
 765                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 766                                                 return -1;
 767                                         }
 768                                         PopPosition ();
 769                                 }
 770                                 break;
 771                         case Token.JOIN:
 772                         case Token.ON:
 773                         case Token.EQUALS:
 774                         case Token.SELECT:
 775                         case Token.GROUP:
 776                         case Token.BY:
 777                         case Token.LET:
 778                         case Token.ORDERBY:
 779                         case Token.ASCENDING:
 780                         case Token.DESCENDING:
 781                         case Token.INTO:
 782                                 if (!query_parsing)
 783                                         res = -1;
 784                                 break;
 785
 786                         case Token.USING:
 787                         case Token.NAMESPACE:
 788                                 // TODO: some explanation needed
 789                                 check_incorrect_doc_comment ();
 790                                 parsing_modifiers = false;
 791                                 break;
 792
 793                         case Token.PARTIAL:
 794                                 if (parsing_block > 0) {
 795                                         res = -1;
 796                                         break;
 797                                 }
 798
 799                                 // Save current position and parse next token.
 800                                 PushPosition ();
 801
 802                                 next_token = token ();
 803                                 bool ok = (next_token == Token.CLASS) ||
 804                                         (next_token == Token.STRUCT) ||
 805                                         (next_token == Token.INTERFACE) ||
 806                                         (next_token == Token.VOID);
 807
 808                                 PopPosition ();
 809
 810                                 if (ok) {
 811                                         if (next_token == Token.VOID) {
 812                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 813                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 814                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 815                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 816
 817                                         return res;
 818                                 }
 819
 820                                 if (next_token < Token.LAST_KEYWORD) {
 821                                         Report.Error (267, Location,
 822                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 823                                         return token ();
 824                                 }
 825
 826                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 827                                 // which has been overwritten before we grabbed the identifier
 828                                 id_builder[0] = 'p';
 829                                 id_builder[1] = 'a';
 830                                 id_builder[2] = 'r';
 831                                 id_builder[3] = 't';
 832                                 id_builder[4] = 'i';
 833                                 id_builder[5] = 'a';
 834                                 id_builder[6] = 'l';
 835                                 res = -1;
 836                                 break;
 837
 838                         case Token.ASYNC:
 839                                 if (parsing_modifiers) {
 840                                         //
 841                                         // Skip attributes section or constructor called async
 842                                         //
 843                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 844                                                 res = -1;
 845                                         } else {
 846                                                 // async is keyword
 847                                         }
 848                                 } else if (parsing_block > 0) {
 849                                         switch (peek_token ()) {
 850                                         case Token.DELEGATE:
 851                                         case Token.OPEN_PARENS_LAMBDA:
 852                                                 // async is keyword
 853                                                 break;
 854                                         case Token.IDENTIFIER:
 855                                                 PushPosition ();
 856                                                 xtoken ();
 857                                                 if (xtoken () != Token.ARROW) {
 858                                                         PopPosition ();
 859                                                         goto default;
 860                                                 }
 861
 862                                                 PopPosition ();
 863                                                 break;
 864                                         default:
 865                                                 // peek_token could overwrite id_buffer
 866                                                 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 867                                                 res = -1;
 868                                                 break;
 869                                         }
 870                                 } else {
 871                                         res = -1;
 872                                 }
 873
 874                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 875                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 876                                 }
 877
 878                                 break;
 879
 880                         case Token.AWAIT:
 881                                 if (parsing_block == 0)
 882                                         res = -1;
 883
 884                                 break;
 885                         }
 886
 887
 888                         return res;
 889                 }
 890
 891                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 892                 {
 893                         //
 894                         // Keywords are stored in an array of arrays grouped by their
 895                         // length and then by the first character
 896                         //
 897                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 898                                 return PreprocessorDirective.Invalid;
 899
 900                         int first_index = id[0] - '_';
 901                         if (first_index > 'z' - '_')
 902                                 return PreprocessorDirective.Invalid;
 903
 904                         var kwe = keywords_preprocessor[id_len][first_index];
 905                         if (kwe == null)
 906                                 return PreprocessorDirective.Invalid;
 907
 908                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 909                         do {
 910                                 res = kwe.Token;
 911                                 for (int i = 1; i < id_len; ++i) {
 912                                         if (id[i] != kwe.Value[i]) {
 913                                                 res = 0;
 914                                                 kwe = kwe.Next;
 915                                                 break;
 916                                         }
 917                                 }
 918                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 919
 920                         return res;
 921                 }
 922
 923                 public Location Location {
 924                         get {
 925                                 return new Location (current_source, ref_line, col);
 926                         }
 927                 }
 928
 929                 static bool is_identifier_start_character (int c)
 930                 {
 931                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')
 932                                 return true;
 933
 934                         if (c < 0x80)
 935                                 return false;
 936
 937                         return is_identifier_start_character_slow_part ((char) c);
 938                 }
 939
 940                 static bool is_identifier_part_character (char c)
 941                 {
 942                         if (c >= 'a' && c <= 'z')
 943                                 return true;
 944
 945                         if (c >= 'A' && c <= 'Z')
 946                                 return true;
 947
 948                         if (c == '_' || (c >= '0' && c <= '9'))
 949                                 return true;
 950
 951                         if (c < 0x80)
 952                                 return false;
 953
 954                         return is_identifier_part_character_slow_part (c);
 955                 }
 956
 957                 static bool is_identifier_start_character_slow_part (char c)
 958                 {
 959                         switch (Char.GetUnicodeCategory (c)) {
 960                         case UnicodeCategory.LetterNumber:
 961                         case UnicodeCategory.UppercaseLetter:
 962                         case UnicodeCategory.LowercaseLetter:
 963                         case UnicodeCategory.TitlecaseLetter:
 964                         case UnicodeCategory.ModifierLetter:
 965                         case UnicodeCategory.OtherLetter:
 966                                 return true;
 967                         }
 968                         return false;
 969                 }
 970
 971                 static bool is_identifier_part_character_slow_part (char c)
 972                 {
 973                         switch (Char.GetUnicodeCategory (c)) {
 974                         // connecting-character:  A Unicode character of the class Pc
 975                         case UnicodeCategory.ConnectorPunctuation:
 976
 977                         // combining-character: A Unicode character of classes Mn or Mc
 978                         case UnicodeCategory.NonSpacingMark:
 979                         case UnicodeCategory.SpacingCombiningMark:
 980
 981                         // decimal-digit-character: A Unicode character of the class Nd
 982                         case UnicodeCategory.DecimalDigitNumber:
 983
 984                         // plus is_identifier_start_character_slow_part
 985                         case UnicodeCategory.LetterNumber:
 986                         case UnicodeCategory.UppercaseLetter:
 987                         case UnicodeCategory.LowercaseLetter:
 988                         case UnicodeCategory.TitlecaseLetter:
 989                         case UnicodeCategory.ModifierLetter:
 990                         case UnicodeCategory.OtherLetter:
 991                                 return true;
 992
 993                         // formatting-character: A Unicode character of the class Cf
 994                         case UnicodeCategory.Format:
 995                                 // csc bug compatibility which recognizes it as a whitespace
 996                                 return c != 0xFEFF;
 997                         }
 998
 999                         return false;
1000                 }
1001
1002                 public static bool IsKeyword (string s)
1003                 {
1004                         return keyword_strings.Contains (s);
1005                 }
1006
1007                 //
1008                 // Open parens micro parser. Detects both lambda and cast ambiguity.
1009                 //
1010                 int TokenizeOpenParens ()
1011                 {
1012                         int ptoken;
1013                         current_token = -1;
1014
1015                         int bracket_level = 0;
1016                         bool is_type = false;
1017                         bool can_be_type = false;
1018
1019                         while (true) {
1020                                 ptoken = current_token;
1021                                 token ();
1022
1023                                 switch (current_token) {
1024                                 case Token.CLOSE_PARENS:
1025                                         token ();
1026
1027                                         //
1028                                         // Expression inside parens is lambda, (int i) =>
1029                                         //
1030                                         if (current_token == Token.ARROW)
1031                                                 return Token.OPEN_PARENS_LAMBDA;
1032
1033                                         //
1034                                         // Expression inside parens is single type, (int[])
1035                                         //
1036                                         if (is_type) {
1037                                                 if (current_token == Token.SEMICOLON)
1038                                                         return Token.OPEN_PARENS;
1039
1040                                                 return Token.OPEN_PARENS_CAST;
1041                                         }
1042
1043                                         //
1044                                         // Expression is possible cast, look at next token, (T)null
1045                                         //
1046                                         if (can_be_type) {
1047                                                 switch (current_token) {
1048                                                 case Token.OPEN_PARENS:
1049                                                 case Token.BANG:
1050                                                 case Token.TILDE:
1051                                                 case Token.IDENTIFIER:
1052                                                 case Token.LITERAL:
1053                                                 case Token.BASE:
1054                                                 case Token.CHECKED:
1055                                                 case Token.DELEGATE:
1056                                                 case Token.FALSE:
1057                                                 case Token.FIXED:
1058                                                 case Token.NEW:
1059                                                 case Token.NULL:
1060                                                 case Token.SIZEOF:
1061                                                 case Token.THIS:
1062                                                 case Token.THROW:
1063                                                 case Token.TRUE:
1064                                                 case Token.TYPEOF:
1065                                                 case Token.UNCHECKED:
1066                                                 case Token.UNSAFE:
1067                                                 case Token.DEFAULT:
1068                                                 case Token.AWAIT:
1069
1070                                                 //
1071                                                 // These can be part of a member access
1072                                                 //
1073                                                 case Token.INT:
1074                                                 case Token.UINT:
1075                                                 case Token.SHORT:
1076                                                 case Token.USHORT:
1077                                                 case Token.LONG:
1078                                                 case Token.ULONG:
1079                                                 case Token.DOUBLE:
1080                                                 case Token.FLOAT:
1081                                                 case Token.CHAR:
1082                                                 case Token.BYTE:
1083                                                 case Token.DECIMAL:
1084                                                 case Token.BOOL:
1085                                                 case Token.STRING:
1086                                                         return Token.OPEN_PARENS_CAST;
1087                                                 }
1088                                         }
1089                                         return Token.OPEN_PARENS;
1090
1091                                 case Token.DOT:
1092                                 case Token.DOUBLE_COLON:
1093                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1094                                                 goto default;
1095
1096                                         continue;
1097
1098                                 case Token.IDENTIFIER:
1099                                 case Token.AWAIT:
1100                                         switch (ptoken) {
1101                                         case Token.DOT:
1102                                                 if (bracket_level == 0) {
1103                                                         is_type = false;
1104                                                         can_be_type = true;
1105                                                 }
1106
1107                                                 continue;
1108                                         case Token.OP_GENERICS_LT:
1109                                         case Token.COMMA:
1110                                         case Token.DOUBLE_COLON:
1111                                         case -1:
1112                                                 if (bracket_level == 0)
1113                                                         can_be_type = true;
1114                                                 continue;
1115                                         default:
1116                                                 can_be_type = is_type = false;
1117                                                 continue;
1118                                         }
1119
1120                                 case Token.OBJECT:
1121                                 case Token.STRING:
1122                                 case Token.BOOL:
1123                                 case Token.DECIMAL:
1124                                 case Token.FLOAT:
1125                                 case Token.DOUBLE:
1126                                 case Token.SBYTE:
1127                                 case Token.BYTE:
1128                                 case Token.SHORT:
1129                                 case Token.USHORT:
1130                                 case Token.INT:
1131                                 case Token.UINT:
1132                                 case Token.LONG:
1133                                 case Token.ULONG:
1134                                 case Token.CHAR:
1135                                 case Token.VOID:
1136                                         if (bracket_level == 0)
1137                                                 is_type = true;
1138                                         continue;
1139
1140                                 case Token.COMMA:
1141                                         if (bracket_level == 0) {
1142                                                 bracket_level = 100;
1143                                                 can_be_type = is_type = false;
1144                                         }
1145                                         continue;
1146
1147                                 case Token.OP_GENERICS_LT:
1148                                 case Token.OPEN_BRACKET:
1149                                         if (bracket_level++ == 0)
1150                                                 is_type = true;
1151                                         continue;
1152
1153                                 case Token.OP_GENERICS_GT:
1154                                 case Token.CLOSE_BRACKET:
1155                                         --bracket_level;
1156                                         continue;
1157
1158                                 case Token.INTERR_NULLABLE:
1159                                 case Token.STAR:
1160                                         if (bracket_level == 0)
1161                                                 is_type = true;
1162                                         continue;
1163
1164                                 case Token.REF:
1165                                 case Token.OUT:
1166                                         can_be_type = is_type = false;
1167                                         continue;
1168
1169                                 default:
1170                                         return Token.OPEN_PARENS;
1171                                 }
1172                         }
1173                 }
1174
1175                 public static bool IsValidIdentifier (string s)
1176                 {
1177                         if (s == null || s.Length == 0)
1178                                 return false;
1179
1180                         if (!is_identifier_start_character (s [0]))
1181                                 return false;
1182
1183                         for (int i = 1; i < s.Length; i ++)
1184                                 if (! is_identifier_part_character (s [i]))
1185                                         return false;
1186
1187                         return true;
1188                 }
1189
1190                 bool parse_less_than (ref int genericDimension)
1191                 {
1192                 start:
1193                         int the_token = token ();
1194                         if (the_token == Token.OPEN_BRACKET) {
1195                                 while (true) {
1196                                         the_token = token ();
1197                                         if (the_token == Token.EOF)
1198                                                 return true;
1199
1200                                         if (the_token == Token.CLOSE_BRACKET)
1201                                                 break;
1202                                 }
1203                                 the_token = token ();
1204                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1205                                 the_token = token ();
1206                         }
1207                         switch (the_token) {
1208                         case Token.IDENTIFIER:
1209                         case Token.OBJECT:
1210                         case Token.STRING:
1211                         case Token.BOOL:
1212                         case Token.DECIMAL:
1213                         case Token.FLOAT:
1214                         case Token.DOUBLE:
1215                         case Token.SBYTE:
1216                         case Token.BYTE:
1217                         case Token.SHORT:
1218                         case Token.USHORT:
1219                         case Token.INT:
1220                         case Token.UINT:
1221                         case Token.LONG:
1222                         case Token.ULONG:
1223                         case Token.CHAR:
1224                         case Token.VOID:
1225                                 break;
1226                         case Token.OP_GENERICS_GT:
1227                                 genericDimension = 1;
1228                                 return true;
1229                         case Token.IN:
1230                         case Token.OUT:
1231                                 return true;
1232                         case Token.COMMA:
1233                                 do {
1234                                         ++genericDimension;
1235                                         the_token = token ();
1236                                 } while (the_token == Token.COMMA);
1237
1238                                 if (the_token == Token.OP_GENERICS_GT) {
1239                                         ++genericDimension;
1240                                         return true;
1241                                 }
1242
1243                                 return false;
1244                         default:
1245                                 return false;
1246                         }
1247                 again:
1248                         the_token = token ();
1249
1250                         if (the_token == Token.OP_GENERICS_GT)
1251                                 return true;
1252                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1253                                 goto start;
1254                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1255                                 goto again;
1256                         else if (the_token == Token.OP_GENERICS_LT) {
1257                                 if (!parse_less_than (ref genericDimension))
1258                                         return false;
1259                                 goto again;
1260                         } else if (the_token == Token.OPEN_BRACKET) {
1261                         rank_specifiers:
1262                                 the_token = token ();
1263                                 if (the_token == Token.CLOSE_BRACKET)
1264                                         goto again;
1265                                 else if (the_token == Token.COMMA)
1266                                         goto rank_specifiers;
1267                                 return false;
1268                         }
1269
1270                         return false;
1271                 }
1272
1273                 public int peek_token ()
1274                 {
1275                         int the_token;
1276
1277                         PushPosition ();
1278                         the_token = token ();
1279                         PopPosition ();
1280
1281                         return the_token;
1282                 }
1283
1284                 //
1285                 // Tonizes `?' using custom disambiguous rules to return one
1286                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1287                 //
1288                 // Tricky expression looks like:
1289                 //
1290                 // Foo ? a = x ? b : c;
1291                 //
1292                 int TokenizePossibleNullableType ()
1293                 {
1294                         if (parsing_block == 0 || parsing_type > 0)
1295                                 return Token.INTERR_NULLABLE;
1296
1297                         int d = peek_char ();
1298                         if (d == '?') {
1299                                 get_char ();
1300                                 return Token.OP_COALESCING;
1301                         }
1302
1303                         if (d == '.') {
1304                                 return Token.INTERR_OPERATOR;
1305                         }
1306
1307                         if (d != ' ') {
1308                                 if (d == ',' || d == ';' || d == '>')
1309                                         return Token.INTERR_NULLABLE;
1310                                 if (d == '*' || (d >= '0' && d <= '9'))
1311                                         return Token.INTERR;
1312                         }
1313
1314                         PushPosition ();
1315                         current_token = Token.NONE;
1316                         int next_token;
1317                         int parens = 0;
1318                         int generics = 0;
1319                         int brackets = 0;
1320
1321                         var nt = xtoken ();
1322                         switch (nt) {
1323                         case Token.DOT:
1324                         case Token.OPEN_BRACKET_EXPR:
1325                                 next_token = Token.INTERR_OPERATOR;
1326                                 break;
1327                         case Token.LITERAL:
1328                         case Token.TRUE:
1329                         case Token.FALSE:
1330                         case Token.NULL:
1331                         case Token.THIS:
1332                         case Token.NEW:
1333                         case Token.INTERPOLATED_STRING:
1334                                 next_token = Token.INTERR;
1335                                 break;
1336
1337                         case Token.SEMICOLON:
1338                         case Token.COMMA:
1339                         case Token.CLOSE_PARENS:
1340                         case Token.OPEN_BRACKET:
1341                         case Token.OP_GENERICS_GT:
1342                         case Token.INTERR:
1343                         case Token.OP_COALESCING:
1344                         case Token.COLON:
1345                                 next_token = Token.INTERR_NULLABLE;
1346                                 break;
1347
1348                         case Token.OPEN_PARENS:
1349                         case Token.OPEN_PARENS_CAST:
1350                         case Token.OPEN_PARENS_LAMBDA:
1351                                 next_token = -1;
1352                                 ++parens;
1353                                 break;
1354
1355                         case Token.OP_GENERICS_LT:
1356                         case Token.OP_GENERICS_LT_DECL:
1357                         case Token.GENERIC_DIMENSION:
1358                                 next_token = -1;
1359                                 ++generics;
1360                                 break;
1361
1362                         default:
1363                                 next_token = -1;
1364                                 break;
1365                         }
1366
1367                         if (next_token == -1) {
1368                                 switch (xtoken ()) {
1369                                 case Token.COMMA:
1370                                 case Token.SEMICOLON:
1371                                 case Token.OPEN_BRACE:
1372                                 case Token.IN:
1373                                         next_token = Token.INTERR_NULLABLE;
1374                                         break;
1375
1376                                 case Token.COLON:
1377                                         next_token = Token.INTERR;
1378                                         break;
1379
1380                                 case Token.OPEN_PARENS:
1381                                 case Token.OPEN_PARENS_CAST:
1382                                 case Token.OPEN_PARENS_LAMBDA:
1383                                         ++parens;
1384                                         goto default;
1385
1386                                 case Token.OPEN_BRACKET:
1387                                 case Token.OPEN_BRACKET_EXPR:
1388                                         ++brackets;
1389                                         goto default;
1390
1391                                 case Token.CLOSE_PARENS:
1392                                         --parens;
1393                                         goto default;
1394
1395                                 case Token.OP_GENERICS_LT:
1396                                 case Token.OP_GENERICS_LT_DECL:
1397                                 case Token.GENERIC_DIMENSION:
1398                                         ++generics;
1399                                         goto default;
1400
1401                                 default:
1402                                         int ntoken;
1403                                         int interrs = 1;
1404                                         int colons = 0;
1405                                         int braces = 0;
1406                                         //
1407                                         // All shorcuts failed, do it hard way
1408                                         //
1409                                         while ((ntoken = xtoken ()) != Token.EOF) {
1410                                                 switch (ntoken) {
1411                                                 case Token.OPEN_BRACE:
1412                                                         ++braces;
1413                                                         continue;
1414                                                 case Token.OPEN_PARENS:
1415                                                 case Token.OPEN_PARENS_CAST:
1416                                                 case Token.OPEN_PARENS_LAMBDA:
1417                                                         ++parens;
1418                                                         continue;
1419                                                 case Token.CLOSE_BRACE:
1420                                                         --braces;
1421                                                         continue;
1422                                                 case Token.OP_GENERICS_LT:
1423                                                 case Token.OP_GENERICS_LT_DECL:
1424                                                 case Token.GENERIC_DIMENSION:
1425                                                         ++generics;
1426                                                         continue;
1427                                                 case Token.OPEN_BRACKET:
1428                                                 case Token.OPEN_BRACKET_EXPR:
1429                                                         ++brackets;
1430                                                         continue;
1431                                                 case Token.CLOSE_BRACKET:
1432                                                         --brackets;
1433                                                         continue;
1434                                                 case Token.CLOSE_PARENS:
1435                                                         if (parens > 0) {
1436                                                                 --parens;
1437                                                                 continue;
1438                                                         }
1439
1440                                                         PopPosition ();
1441                                                         return Token.INTERR_NULLABLE;
1442
1443                                                 case Token.OP_GENERICS_GT:
1444                                                         if (generics > 0) {
1445                                                                 --generics;
1446                                                                 continue;
1447                                                         }
1448
1449                                                         PopPosition ();
1450                                                         return Token.INTERR_NULLABLE;
1451                                                 }
1452
1453                                                 if (braces != 0)
1454                                                         continue;
1455
1456                                                 if (ntoken == Token.SEMICOLON)
1457                                                         break;
1458
1459                                                 if (parens != 0)
1460                                                         continue;
1461
1462                                                 if (ntoken == Token.COMMA) {
1463                                                         if (generics != 0 || brackets != 0)
1464                                                                 continue;
1465
1466                                                         PopPosition ();
1467                                                         return Token.INTERR_NULLABLE;
1468                                                 }
1469
1470                                                 if (ntoken == Token.COLON) {
1471                                                         if (++colons == interrs)
1472                                                                 break;
1473                                                         continue;
1474                                                 }
1475
1476                                                 if (ntoken == Token.INTERR) {
1477                                                         ++interrs;
1478                                                         continue;
1479                                                 }
1480                                         }
1481
1482                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1483                                         break;
1484                                 }
1485                         }
1486
1487                         PopPosition ();
1488                         return next_token;
1489                 }
1490
1491                 bool decimal_digits (int c)
1492                 {
1493                         int d;
1494                         bool seen_digits = false;
1495
1496                         if (c != -1){
1497                                 if (number_pos == MaxNumberLength)
1498                                         Error_NumericConstantTooLong ();
1499                                 number_builder [number_pos++] = (char) c;
1500                         }
1501
1502                         //
1503                         // We use peek_char2, because decimal_digits needs to do a
1504                         // 2-character look-ahead (5.ToString for example).
1505                         //
1506                         while ((d = peek_char2 ()) != -1){
1507                                 if (d >= '0' && d <= '9'){
1508                                         if (number_pos == MaxNumberLength)
1509                                                 Error_NumericConstantTooLong ();
1510                                         number_builder [number_pos++] = (char) d;
1511                                         get_char ();
1512                                         seen_digits = true;
1513                                 } else
1514                                         break;
1515                         }
1516
1517                         return seen_digits;
1518                 }
1519
1520                 static bool is_hex (int e)
1521                 {
1522                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1523                 }
1524
1525                 static TypeCode real_type_suffix (int c)
1526                 {
1527                         switch (c){
1528                         case 'F': case 'f':
1529                                 return TypeCode.Single;
1530                         case 'D': case 'd':
1531                                 return TypeCode.Double;
1532                         case 'M': case 'm':
1533                                 return TypeCode.Decimal;
1534                         default:
1535                                 return TypeCode.Empty;
1536                         }
1537                 }
1538
1539                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1540                 {
1541                         bool is_unsigned = false;
1542                         bool is_long = false;
1543
1544                         if (c != -1){
1545                                 bool scanning = true;
1546                                 do {
1547                                         switch (c){
1548                                         case 'U': case 'u':
1549                                                 if (is_unsigned)
1550                                                         scanning = false;
1551                                                 is_unsigned = true;
1552                                                 get_char ();
1553                                                 break;
1554
1555                                         case 'l':
1556                                                 if (!is_unsigned){
1557                                                         //
1558                                                         // if we have not seen anything in between
1559                                                         // report this error
1560                                                         //
1561                                                         Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1562                                                 }
1563
1564                                                 goto case 'L';
1565
1566                                         case 'L':
1567                                                 if (is_long)
1568                                                         scanning = false;
1569                                                 is_long = true;
1570                                                 get_char ();
1571                                                 break;
1572
1573                                         default:
1574                                                 scanning = false;
1575                                                 break;
1576                                         }
1577                                         c = peek_char ();
1578                                 } while (scanning);
1579                         }
1580
1581                         if (is_long && is_unsigned){
1582                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1583                         }
1584
1585                         if (is_unsigned){
1586                                 // uint if possible, or ulong else.
1587
1588                                 if ((ul & 0xffffffff00000000) == 0)
1589                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1590                                 else
1591                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1592                         } else if (is_long){
1593                                 // long if possible, ulong otherwise
1594                                 if ((ul & 0x8000000000000000) != 0)
1595                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1596                                 else
1597                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1598                         } else {
1599                                 // int, uint, long or ulong in that order
1600                                 if ((ul & 0xffffffff00000000) == 0){
1601                                         uint ui = (uint) ul;
1602
1603                                         if ((ui & 0x80000000) != 0)
1604                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1605                                         else
1606                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1607                                 } else {
1608                                         if ((ul & 0x8000000000000000) != 0)
1609                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1610                                         else
1611                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1612                                 }
1613                         }
1614                 }
1615
1616                 //
1617                 // given `c' as the next char in the input decide whether
1618                 // we need to convert to a special type, and then choose
1619                 // the best representation for the integer
1620                 //
1621                 ILiteralConstant adjust_int (int c, Location loc)
1622                 {
1623                         try {
1624                                 if (number_pos > 9){
1625                                         ulong ul = (uint) (number_builder [0] - '0');
1626
1627                                         for (int i = 1; i < number_pos; i++){
1628                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1629                                         }
1630
1631                                         return integer_type_suffix (ul, c, loc);
1632                                 } else {
1633                                         uint ui = (uint) (number_builder [0] - '0');
1634
1635                                         for (int i = 1; i < number_pos; i++){
1636                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1637                                         }
1638
1639                                         return integer_type_suffix (ui, c, loc);
1640                                 }
1641                         } catch (OverflowException) {
1642                                 Error_NumericConstantTooLong ();
1643                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1644                         }
1645                         catch (FormatException) {
1646                                 Report.Error (1013, Location, "Invalid number");
1647                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1648                         }
1649                 }
1650
1651                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1652                 {
1653                         string s = new string (number_builder, 0, number_pos);
1654                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1655
1656                         switch (t){
1657                         case TypeCode.Decimal:
1658                                 try {
1659                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1660                                 } catch (OverflowException) {
1661                                         Report.Error (594, Location, error_details, "decimal");
1662                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1663                                 }
1664                         case TypeCode.Single:
1665                                 try {
1666                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1667                                 } catch (OverflowException) {
1668                                         Report.Error (594, Location, error_details, "float");
1669                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1670                                 }
1671                         default:
1672                                 try {
1673                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1674                                 } catch (OverflowException) {
1675                                         Report.Error (594, loc, error_details, "double");
1676                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1677                                 }
1678                         }
1679                 }
1680
1681                 ILiteralConstant handle_hex (Location loc)
1682                 {
1683                         int d;
1684                         ulong ul;
1685
1686                         get_char ();
1687                         while ((d = peek_char ()) != -1){
1688                                 if (is_hex (d)){
1689                                         number_builder [number_pos++] = (char) d;
1690                                         get_char ();
1691                                 } else
1692                                         break;
1693                         }
1694
1695                         string s = new String (number_builder, 0, number_pos);
1696
1697                         try {
1698                                 if (number_pos <= 8)
1699                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1700                                 else
1701                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1702
1703                                 return integer_type_suffix (ul, peek_char (), loc);
1704                         } catch (OverflowException){
1705                                 Error_NumericConstantTooLong ();
1706                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1707                         }
1708                         catch (FormatException) {
1709                                 Report.Error (1013, Location, "Invalid number");
1710                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1711                         }
1712                 }
1713
1714                 //
1715                 // Invoked if we know we have .digits or digits
1716                 //
1717                 int is_number (int c, bool dotLead)
1718                 {
1719                         ILiteralConstant res;
1720
1721 #if FULL_AST
1722                         int read_start = reader.Position - 1;
1723                         if (dotLead) {
1724                                 //
1725                                 // Caller did peek_char
1726                                 //
1727                                 --read_start;
1728                         }
1729 #endif
1730                         number_pos = 0;
1731                         var loc = Location;
1732
1733                         if (!dotLead){
1734                                 if (c == '0'){
1735                                         int peek = peek_char ();
1736
1737                                         if (peek == 'x' || peek == 'X') {
1738                                                 val = res = handle_hex (loc);
1739 #if FULL_AST
1740                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1741 #endif
1742
1743                                                 return Token.LITERAL;
1744                                         }
1745                                 }
1746                                 decimal_digits (c);
1747                                 c = peek_char ();
1748                         }
1749
1750                         //
1751                         // We need to handle the case of
1752                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1753                         //
1754                         bool is_real = false;
1755                         if (c == '.'){
1756                                 if (!dotLead)
1757                                         get_char ();
1758
1759                                 if (decimal_digits ('.')){
1760                                         is_real = true;
1761                                         c = peek_char ();
1762                                 } else {
1763                                         putback ('.');
1764                                         number_pos--;
1765                                         val = res = adjust_int (-1, loc);
1766
1767 #if FULL_AST
1768                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1769 #endif
1770                                         return Token.LITERAL;
1771                                 }
1772                         }
1773
1774                         if (c == 'e' || c == 'E'){
1775                                 is_real = true;
1776                                 get_char ();
1777                                 if (number_pos == MaxNumberLength)
1778                                         Error_NumericConstantTooLong ();
1779                                 number_builder [number_pos++] = (char) c;
1780                                 c = get_char ();
1781
1782                                 if (c == '+'){
1783                                         if (number_pos == MaxNumberLength)
1784                                                 Error_NumericConstantTooLong ();
1785                                         number_builder [number_pos++] = '+';
1786                                         c = -1;
1787                                 } else if (c == '-') {
1788                                         if (number_pos == MaxNumberLength)
1789                                                 Error_NumericConstantTooLong ();
1790                                         number_builder [number_pos++] = '-';
1791                                         c = -1;
1792                                 } else {
1793                                         if (number_pos == MaxNumberLength)
1794                                                 Error_NumericConstantTooLong ();
1795                                         number_builder [number_pos++] = '+';
1796                                 }
1797
1798                                 decimal_digits (c);
1799                                 c = peek_char ();
1800                         }
1801
1802                         var type = real_type_suffix (c);
1803                         if (type == TypeCode.Empty && !is_real) {
1804                                 res = adjust_int (c, loc);
1805                         } else {
1806                                 is_real = true;
1807
1808                                 if (type != TypeCode.Empty) {
1809                                         get_char ();
1810                                 }
1811
1812                                 res = adjust_real (type, loc);
1813                         }
1814
1815                         val = res;
1816
1817 #if FULL_AST
1818                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1819                         if (chars[chars.Length - 1] == '\r')
1820                                 Array.Resize (ref chars, chars.Length - 1);
1821                         res.ParsedValue = chars;
1822 #endif
1823
1824                         return Token.LITERAL;
1825                 }
1826
1827                 //
1828                 // Accepts exactly count (4 or 8) hex, no more no less
1829                 //
1830                 int getHex (int count, out int surrogate, out bool error)
1831                 {
1832                         int i;
1833                         int total = 0;
1834                         int c;
1835                         int top = count != -1 ? count : 4;
1836
1837                         get_char ();
1838                         error = false;
1839                         surrogate = 0;
1840                         for (i = 0; i < top; i++){
1841                                 c = get_char ();
1842
1843                                 if (c >= '0' && c <= '9')
1844                                         c = (int) c - (int) '0';
1845                                 else if (c >= 'A' && c <= 'F')
1846                                         c = (int) c - (int) 'A' + 10;
1847                                 else if (c >= 'a' && c <= 'f')
1848                                         c = (int) c - (int) 'a' + 10;
1849                                 else {
1850                                         error = true;
1851                                         return 0;
1852                                 }
1853
1854                                 total = (total * 16) + c;
1855                                 if (count == -1){
1856                                         int p = peek_char ();
1857                                         if (p == -1)
1858                                                 break;
1859                                         if (!is_hex ((char)p))
1860                                                 break;
1861                                 }
1862                         }
1863
1864                         if (top == 8) {
1865                                 if (total > 0x0010FFFF) {
1866                                         error = true;
1867                                         return 0;
1868                                 }
1869
1870                                 if (total >= 0x00010000) {
1871                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1872                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1873                                 }
1874                         }
1875
1876                         return total;
1877                 }
1878
1879                 int escape (int c, out int surrogate)
1880                 {
1881                         bool error;
1882                         int d;
1883                         int v;
1884
1885                         d = peek_char ();
1886                         if (c != '\\') {
1887                                 surrogate = 0;
1888                                 return c;
1889                         }
1890
1891                         switch (d){
1892                         case 'a':
1893                                 v = '\a'; break;
1894                         case 'b':
1895                                 v = '\b'; break;
1896                         case 'n':
1897                                 v = '\n'; break;
1898                         case 't':
1899                                 v = '\t'; break;
1900                         case 'v':
1901                                 v = '\v'; break;
1902                         case 'r':
1903                                 v = '\r'; break;
1904                         case '\\':
1905                                 v = '\\'; break;
1906                         case 'f':
1907                                 v = '\f'; break;
1908                         case '0':
1909                                 v = 0; break;
1910                         case '"':
1911                                 v = '"'; break;
1912                         case '\'':
1913                                 v = '\''; break;
1914                         case 'x':
1915                                 v = getHex (-1, out surrogate, out error);
1916                                 if (error)
1917                                         goto default;
1918                                 return v;
1919                         case 'u':
1920                         case 'U':
1921                                 return EscapeUnicode (d, out surrogate);
1922                         default:
1923                                 surrogate = 0;
1924                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1925                                 return d;
1926                         }
1927
1928                         get_char ();
1929                         surrogate = 0;
1930                         return v;
1931                 }
1932
1933                 int EscapeUnicode (int ch, out int surrogate)
1934                 {
1935                         bool error;
1936                         if (ch == 'U') {
1937                                 ch = getHex (8, out surrogate, out error);
1938                         } else {
1939                                 ch = getHex (4, out surrogate, out error);
1940                         }
1941
1942                         if (error)
1943                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1944
1945                         return ch;
1946                 }
1947
1948                 int get_char ()
1949                 {
1950                         int x;
1951                         if (putback_char != -1) {
1952                                 x = putback_char;
1953                                 putback_char = -1;
1954                         } else {
1955                                 x = reader.Read ();
1956                         }
1957
1958                         if (x <= 13) {
1959                                 if (x == '\r') {
1960                                         if (peek_char () == '\n') {
1961                                                 putback_char = -1;
1962                                         }
1963
1964                                         x = '\n';
1965                                         advance_line ();
1966                                 } else if (x == '\n') {
1967                                         advance_line ();
1968                                 } else {
1969                                         col++;
1970                                 }
1971                         } else if (x >= UnicodeLS && x <= UnicodePS) {
1972                                 advance_line ();
1973                         } else {
1974                                 col++;
1975                         }
1976
1977                         return x;
1978                 }
1979
1980                 void advance_line ()
1981                 {
1982                         line++;
1983                         ref_line++;
1984                         previous_col = col;
1985                         col = 0;
1986                 }
1987
1988                 int peek_char ()
1989                 {
1990                         if (putback_char == -1)
1991                                 putback_char = reader.Read ();
1992                         return putback_char;
1993                 }
1994
1995                 int peek_char2 ()
1996                 {
1997                         if (putback_char != -1)
1998                                 return putback_char;
1999                         return reader.Peek ();
2000                 }
2001
2002                 public void putback (int c)
2003                 {
2004                         if (putback_char != -1) {
2005                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
2006                         }
2007
2008                         if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
2009                                 // It won't happen though.
2010                                 line--;
2011                                 ref_line--;
2012                                 col = previous_col;
2013                         }
2014                         else
2015                                 col--;
2016                         putback_char = c;
2017                 }
2018
2019                 public bool advance ()
2020                 {
2021                         return peek_char () != -1 || CompleteOnEOF;
2022                 }
2023
2024                 public Object Value {
2025                         get {
2026                                 return val;
2027                         }
2028                 }
2029
2030                 public Object value ()
2031                 {
2032                         return val;
2033                 }
2034
2035                 public int token ()
2036                 {
2037                         current_token = xtoken ();
2038                         return current_token;
2039                 }
2040
2041                 int TokenizePreprocessorKeyword (out int c)
2042                 {
2043                         // skip over white space
2044                         do {
2045                                 c = get_char ();
2046                         } while (c == ' ' || c == '\t');
2047
2048
2049                         int pos = 0;
2050                         while (c != -1 && c >= 'a' && c <= 'z') {
2051                                 id_builder[pos++] = (char) c;
2052                                 c = get_char ();
2053                                 if (c == '\\') {
2054                                         int peek = peek_char ();
2055                                         if (peek == 'U' || peek == 'u') {
2056                                                 int surrogate;
2057                                                 c = EscapeUnicode (c, out surrogate);
2058                                                 if (surrogate != 0) {
2059                                                         if (is_identifier_part_character ((char) c)) {
2060                                                                 id_builder[pos++] = (char) c;
2061                                                         }
2062                                                         c = surrogate;
2063                                                 }
2064                                         }
2065                                 }
2066                         }
2067
2068                         return pos;
2069                 }
2070
2071                 PreprocessorDirective get_cmd_arg (out string arg)
2072                 {
2073                         int c;
2074
2075                         tokens_seen = false;
2076                         arg = "";
2077
2078                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorKeyword (out c));
2079
2080                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
2081                                 return cmd;
2082
2083                         // skip over white space
2084                         while (c == ' ' || c == '\t')
2085                                 c = get_char ();
2086
2087                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
2088                         int pos = 0;
2089
2090                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2091                                 if (c == '\\' && has_identifier_argument >= 0) {
2092                                         if (has_identifier_argument != 0) {
2093                                                 has_identifier_argument = 1;
2094
2095                                                 int peek = peek_char ();
2096                                                 if (peek == 'U' || peek == 'u') {
2097                                                         int surrogate;
2098                                                         c = EscapeUnicode (c, out surrogate);
2099                                                         if (surrogate != 0) {
2100                                                                 if (is_identifier_part_character ((char) c)) {
2101                                                                         if (pos == value_builder.Length)
2102                                                                                 Array.Resize (ref value_builder, pos * 2);
2103
2104                                                                         value_builder[pos++] = (char) c;
2105                                                                 }
2106                                                                 c = surrogate;
2107                                                         }
2108                                                 }
2109                                         } else {
2110                                                 has_identifier_argument = -1;
2111                                         }
2112                                 } else if (c == '/' && peek_char () == '/') {
2113                                         //
2114                                         // Eat single-line comments
2115                                         //
2116                                         get_char ();
2117                                         ReadToEndOfLine ();
2118                                         break;
2119                                 }
2120
2121                                 if (pos == value_builder.Length)
2122                                         Array.Resize (ref value_builder, pos * 2);
2123
2124                                 value_builder[pos++] = (char) c;
2125                                 c = get_char ();
2126                         }
2127
2128                         if (pos != 0) {
2129                                 if (pos > MaxIdentifierLength)
2130                                         arg = new string (value_builder, 0, pos);
2131                                 else
2132                                         arg = InternIdentifier (value_builder, pos);
2133
2134                                 // Eat any trailing whitespaces
2135                                 arg = arg.Trim (simple_whitespaces);
2136                         }
2137
2138                         return cmd;
2139                 }
2140
2141                 //
2142                 // Handles the #line directive
2143                 //
2144                 bool PreProcessLine ()
2145                 {
2146                         Location loc = Location;
2147
2148                         int c;
2149
2150                         int length = TokenizePreprocessorKeyword (out c);
2151                         if (length == line_default.Length) {
2152                                 if (!IsTokenIdentifierEqual (line_default))
2153                                         return false;
2154
2155                                 current_source = source_file.SourceFile;
2156                                 if (!hidden_block_start.IsNull) {
2157                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
2158                                         hidden_block_start = Location.Null;
2159                                 }
2160
2161                                 ref_line = line;
2162                                 return true;
2163                         }
2164
2165                         if (length == line_hidden.Length) {
2166                                 if (!IsTokenIdentifierEqual (line_hidden))
2167                                         return false;
2168
2169                                 if (hidden_block_start.IsNull)
2170                                         hidden_block_start = loc;
2171
2172                                 return true;
2173                         }
2174
2175                         if (length != 0 || c < '0' || c > '9') {
2176                                 //
2177                                 // Eat any remaining characters to continue parsing on next line
2178                                 //
2179                                 ReadToEndOfLine ();
2180                                 return false;
2181                         }
2182
2183                         int new_line = TokenizeNumber (c);
2184                         if (new_line < 1) {
2185                                 //
2186                                 // Eat any remaining characters to continue parsing on next line
2187                                 //
2188                                 ReadToEndOfLine ();
2189                                 return new_line != 0;
2190                         }
2191
2192                         c = get_char ();
2193                         if (c == ' ') {
2194                                 // skip over white space
2195                                 do {
2196                                         c = get_char ();
2197                                 } while (c == ' ' || c == '\t');
2198                         } else if (c == '"') {
2199                                 c = 0;
2200                         }
2201
2202                         if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
2203                                 //
2204                                 // Eat any remaining characters to continue parsing on next line
2205                                 //
2206                                 ReadToEndOfLine ();
2207
2208                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2209                                 return true;
2210                         }
2211
2212                         string new_file_name = null;
2213                         if (c == '"') {
2214                                 new_file_name = TokenizeFileName (ref c);
2215
2216                                 // skip over white space
2217                                 while (c == ' ' || c == '\t') {
2218                                         c = get_char ();
2219                                 }
2220                         }
2221
2222                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2223
2224                         } else if (c == '/') {
2225                                 ReadSingleLineComment ();
2226                         } else {
2227                                 //
2228                                 // Eat any remaining characters to continue parsing on next line
2229                                 //
2230                                 ReadToEndOfLine ();
2231
2232                                 Error_EndLineExpected ();
2233                                 return true;
2234                         }
2235
2236                         if (new_file_name != null) {
2237                                 current_source = context.LookupFile (source_file, new_file_name);
2238                                 source_file.AddIncludeFile (current_source);
2239                         }
2240
2241                         if (!hidden_block_start.IsNull) {
2242                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2243                                 hidden_block_start = Location.Null;
2244                         }
2245
2246                         ref_line = new_line;
2247                         return true;
2248                 }
2249
2250                 //
2251                 // Handles #define and #undef
2252                 //
2253                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2254                 {
2255                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2256                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2257                                 return;
2258                         }
2259
2260                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2261                                 Error_EndLineExpected ();
2262                                 return;
2263                         }
2264
2265                         if (!is_identifier_start_character (ident [0]))
2266                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2267
2268                         foreach (char c in ident.Substring (1)){
2269                                 if (!is_identifier_part_character (c)){
2270                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2271                                         return;
2272                                 }
2273                         }
2274
2275                         if (!caller_is_taking)
2276                                 return;
2277
2278                         if (is_define) {
2279                                 //
2280                                 // #define ident
2281                                 //
2282                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2283                                         return;
2284
2285                                 source_file.AddDefine (ident);
2286                         } else {
2287                                 //
2288                                 // #undef ident
2289                                 //
2290                                 source_file.AddUndefine (ident);
2291                         }
2292                 }
2293
2294                 byte read_hex (out bool error)
2295                 {
2296                         int total;
2297                         int c = get_char ();
2298
2299                         if ((c >= '0') && (c <= '9'))
2300                                 total = (int) c - (int) '0';
2301                         else if ((c >= 'A') && (c <= 'F'))
2302                                 total = (int) c - (int) 'A' + 10;
2303                         else if ((c >= 'a') && (c <= 'f'))
2304                                 total = (int) c - (int) 'a' + 10;
2305                         else {
2306                                 error = true;
2307                                 return 0;
2308                         }
2309
2310                         total *= 16;
2311                         c = get_char ();
2312
2313                         if ((c >= '0') && (c <= '9'))
2314                                 total += (int) c - (int) '0';
2315                         else if ((c >= 'A') && (c <= 'F'))
2316                                 total += (int) c - (int) 'A' + 10;
2317                         else if ((c >= 'a') && (c <= 'f'))
2318                                 total += (int) c - (int) 'a' + 10;
2319                         else {
2320                                 error = true;
2321                                 return 0;
2322                         }
2323
2324                         error = false;
2325                         return (byte) total;
2326                 }
2327
2328                 //
2329                 // Parses #pragma checksum
2330                 //
2331                 bool ParsePragmaChecksum ()
2332                 {
2333                         //
2334                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2335                         //
2336                         // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
2337                         //
2338                         int c = get_char ();
2339
2340                         if (c != '"')
2341                                 return false;
2342
2343                         string file_name = TokenizeFileName (ref c);
2344
2345                         // TODO: Any white-spaces count
2346                         if (c != ' ')
2347                                 return false;
2348
2349                         SourceFile file = context.LookupFile (source_file, file_name);
2350
2351                         if (get_char () != '"' || get_char () != '{')
2352                                 return false;
2353
2354                         bool error;
2355                         byte[] guid_bytes = new byte [16];
2356                         int i = 0;
2357
2358                         for (; i < 4; i++) {
2359                                 guid_bytes [i] = read_hex (out error);
2360                                 if (error)
2361                                         return false;
2362                         }
2363
2364                         if (get_char () != '-')
2365                                 return false;
2366
2367                         for (; i < 10; i++) {
2368                                 guid_bytes [i] = read_hex (out error);
2369                                 if (error)
2370                                         return false;
2371
2372                                 guid_bytes [i++] = read_hex (out error);
2373                                 if (error)
2374                                         return false;
2375
2376                                 if (get_char () != '-')
2377                                         return false;
2378                         }
2379
2380                         for (; i < 16; i++) {
2381                                 guid_bytes [i] = read_hex (out error);
2382                                 if (error)
2383                                         return false;
2384                         }
2385
2386                         if (get_char () != '}' || get_char () != '"')
2387                                 return false;
2388
2389                         // TODO: Any white-spaces count
2390                         c = get_char ();
2391                         if (c != ' ')
2392                                 return false;
2393
2394                         if (get_char () != '"')
2395                                 return false;
2396
2397                         // Any length of checksum
2398                         List<byte> checksum_bytes = new List<byte> (16);
2399
2400                         var checksum_location = Location;
2401                         c = peek_char ();
2402                         while (c != '"' && c != -1) {
2403                                 checksum_bytes.Add (read_hex (out error));
2404                                 if (error)
2405                                         return false;
2406
2407                                 c = peek_char ();
2408                         }
2409
2410                         if (c == '/') {
2411                                 ReadSingleLineComment ();
2412                         } else if (get_char () != '"') {
2413                                 return false;
2414                         }
2415
2416                         if (context.Settings.GenerateDebugInfo) {
2417                                 var chsum = checksum_bytes.ToArray ();
2418
2419                                 if (file.HasChecksum) {
2420                                         if (!ArrayComparer.IsEqual (file.Checksum, chsum)) {
2421                                                 // TODO: Report.SymbolRelatedToPreviousError
2422                                                 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name);
2423                                         }
2424                                 }
2425
2426                                 file.SetChecksum (guid_bytes, chsum);
2427                                 current_source.AutoGenerated = true;
2428                         }
2429
2430                         return true;
2431                 }
2432
2433                 bool IsTokenIdentifierEqual (char[] identifier)
2434                 {
2435                         for (int i = 0; i < identifier.Length; ++i) {
2436                                 if (identifier[i] != id_builder[i])
2437                                         return false;
2438                         }
2439
2440                         return true;
2441                 }
2442
2443                 bool ScanClosingInterpolationBrace ()
2444                 {
2445                         PushPosition ();
2446
2447                         bool? res = null;
2448                         int str_quote = 0;
2449                         do {
2450                                 var c = reader.Read ();
2451                                 switch (c) {
2452                                 case '\"':
2453                                         ++str_quote;
2454                                         break;
2455                                 case -1:
2456                                         res = false;
2457                                         break;
2458                                 case '}':
2459                                         if (str_quote % 2 == 1) {
2460                                                 res = true;
2461                                         }
2462
2463                                         break;
2464                                 }
2465                         } while (res == null);
2466
2467                         PopPosition ();
2468                         return res.Value;
2469                 }
2470
2471                 int TokenizeNumber (int value)
2472                 {
2473                         number_pos = 0;
2474
2475                         decimal_digits (value);
2476                         uint ui = (uint) (number_builder[0] - '0');
2477
2478                         try {
2479                                 for (int i = 1; i < number_pos; i++) {
2480                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2481                                 }
2482
2483                                 return (int) ui;
2484                         } catch (OverflowException) {
2485                                 Error_NumericConstantTooLong ();
2486                                 return -1;
2487                         }
2488                 }
2489
2490                 string TokenizeFileName (ref int c)
2491                 {
2492                         var string_builder = new StringBuilder ();
2493                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2494                                 c = get_char ();
2495                                 if (c == '"') {
2496                                         c = get_char ();
2497                                         break;
2498                                 }
2499
2500                                 string_builder.Append ((char) c);
2501                         }
2502
2503                         if (string_builder.Length == 0) {
2504                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2505                         }
2506
2507
2508                         return string_builder.ToString ();
2509                 }
2510
2511                 int TokenizePragmaWarningIdentifier (ref int c, ref bool identifier)
2512                 {
2513
2514                         if ((c >= '0' && c <= '9') || is_identifier_start_character (c)) {
2515                                 int number;
2516
2517                                 if (c >= '0' && c <= '9') {
2518                                         number_pos = 0;
2519                                         number = TokenizeNumber (c);
2520
2521                                         c = get_char ();
2522
2523                                         if (c != ' ' && c != '\t' && c != ',' && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS) {
2524                                                 return ReadPragmaWarningComment (c);
2525                                         }
2526                                 } else {
2527                                         //
2528                                         // LAMESPEC v6: No spec what identifier really is in this context, it seems keywords are allowed too
2529                                         //
2530                                         int pos = 0;
2531                                         number = -1;
2532                                         id_builder [pos++] = (char)c;
2533                                         while (c < MaxIdentifierLength) {
2534                                                 c = reader.Read ();
2535                                                 id_builder [pos] = (char)c;
2536
2537                                                 if (c >= '0' && c <= '9') {
2538                                                         if (pos == 6 && id_builder [0] == 'C' && id_builder [1] == 'S') {
2539                                                                 // Recognize CSXXXX as C# XXXX warning
2540                                                                 number = 0;
2541                                                                 int pow = 1000;
2542                                                                 for (int i = 0; i < 4; ++i) {
2543                                                                         var ch = id_builder [i + 2];
2544                                                                         if (ch < '0' || ch > '9') {
2545                                                                                 number = -1;
2546                                                                                 break;
2547                                                                         }
2548
2549                                                                         number += (ch - '0') * pow;
2550                                                                         pow /= 10;
2551                                                                 }
2552                                                         }
2553                                                 } else if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '_') {
2554                                                         break;
2555                                                 }
2556
2557                                                 ++pos;
2558                                         }
2559
2560                                         if (number < 0) {
2561                                                 identifier = true;
2562                                                 number = pos;
2563                                         }
2564                                 }
2565
2566                                 // skip over white space
2567                                 while (c == ' ' || c == '\t')
2568                                         c = get_char ();
2569
2570                                 if (c == ',') {
2571                                         c = get_char ();
2572                                 }
2573
2574                                 // skip over white space
2575                                 while (c == ' ' || c == '\t')
2576                                         c = get_char ();
2577
2578                                 return number;
2579                         }
2580
2581                         return ReadPragmaWarningComment (c);
2582                 }
2583
2584                 int ReadPragmaWarningComment (int c)
2585                 {
2586                         if (c == '/') {
2587                                 ReadSingleLineComment ();
2588                         } else {
2589                                 Report.Warning (1692, 1, Location, "Invalid number");
2590
2591                                 // Read everything till the end of the line or file
2592                                 ReadToEndOfLine ();
2593                         }
2594
2595                         return -1;
2596                 }
2597
2598                 void ReadToEndOfLine ()
2599                 {
2600                         int c;
2601                         do {
2602                                 c = get_char ();
2603                         } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
2604                 }
2605
2606                 void ReadSingleLineComment ()
2607                 {
2608                         if (peek_char () != '/')
2609                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2610
2611                         // Read everything till the end of the line or file
2612                         ReadToEndOfLine ();
2613                 }
2614
2615                 /// <summary>
2616                 /// Handles #pragma directive
2617                 /// </summary>
2618                 void ParsePragmaDirective ()
2619                 {
2620                         int c;
2621                         int length = TokenizePreprocessorKeyword (out c);
2622                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2623                                 length = TokenizePreprocessorKeyword (out c);
2624
2625                                 //
2626                                 // #pragma warning disable
2627                                 // #pragma warning restore
2628                                 //
2629                                 if (length == pragma_warning_disable.Length) {
2630                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2631                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2632                                                 // skip over white space
2633                                                 while (c == ' ' || c == '\t')
2634                                                         c = get_char ();
2635
2636                                                 var loc = Location;
2637
2638                                                 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
2639                                                         if (c == '/')
2640                                                                 ReadSingleLineComment ();
2641
2642                                                         //
2643                                                         // Disable/Restore all warnings
2644                                                         //
2645                                                         if (disable) {
2646                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2647                                                         } else {
2648                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2649                                                         }
2650                                                 } else {
2651                                                         //
2652                                                         // Disable/Restore a warning or group of warnings
2653                                                         //
2654                                                         int code;
2655                                                         do {
2656                                                                 bool identifier = false;
2657                                                                 code = TokenizePragmaWarningIdentifier (ref c, ref identifier);
2658                                                                 if (code > 0) {
2659                                                                         if (identifier) {
2660                                                                                 // no-op, custom warnings cannot occur in mcs
2661                                                                         } else if (disable) {
2662                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2663                                                                         } else {
2664                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2665                                                                         }
2666                                                                 }
2667                                                         } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
2668                                                 }
2669
2670                                                 return;
2671                                         }
2672                                 }
2673
2674                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2675
2676                                 // Eat any remaining characters on the line
2677                                 ReadToEndOfLine ();
2678
2679                                 return;
2680                         }
2681
2682                         //
2683                         // #pragma checksum
2684                         //
2685                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2686                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2687                                         Report.Warning (1695, 1, Location,
2688                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2689                                 }
2690
2691                                 return;
2692                         }
2693
2694                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2695
2696                         // Eat any remaining characters on the line
2697                         ReadToEndOfLine ();
2698                 }
2699
2700                 bool eval_val (string s)
2701                 {
2702                         if (s == "true")
2703                                 return true;
2704                         if (s == "false")
2705                                 return false;
2706
2707                         return source_file.IsConditionalDefined (s);
2708                 }
2709
2710                 bool pp_primary (ref string s)
2711                 {
2712                         s = s.Trim ();
2713                         int len = s.Length;
2714
2715                         if (len > 0){
2716                                 char c = s [0];
2717
2718                                 if (c == '('){
2719                                         s = s.Substring (1);
2720                                         bool val = pp_expr (ref s, false);
2721                                         if (s.Length > 0 && s [0] == ')'){
2722                                                 s = s.Substring (1);
2723                                                 return val;
2724                                         }
2725                                         Error_InvalidDirective ();
2726                                         return false;
2727                                 }
2728
2729                                 if (is_identifier_start_character (c)){
2730                                         int j = 1;
2731
2732                                         while (j < len){
2733                                                 c = s [j];
2734
2735                                                 if (is_identifier_part_character (c)){
2736                                                         j++;
2737                                                         continue;
2738                                                 }
2739                                                 bool v = eval_val (s.Substring (0, j));
2740                                                 s = s.Substring (j);
2741                                                 return v;
2742                                         }
2743                                         bool vv = eval_val (s);
2744                                         s = "";
2745                                         return vv;
2746                                 }
2747                         }
2748                         Error_InvalidDirective ();
2749                         return false;
2750                 }
2751
2752                 bool pp_unary (ref string s)
2753                 {
2754                         s = s.Trim ();
2755                         int len = s.Length;
2756
2757                         if (len > 0){
2758                                 if (s [0] == '!'){
2759                                         if (len > 1 && s [1] == '='){
2760                                                 Error_InvalidDirective ();
2761                                                 return false;
2762                                         }
2763                                         s = s.Substring (1);
2764                                         return ! pp_primary (ref s);
2765                                 } else
2766                                         return pp_primary (ref s);
2767                         } else {
2768                                 Error_InvalidDirective ();
2769                                 return false;
2770                         }
2771                 }
2772
2773                 bool pp_eq (ref string s)
2774                 {
2775                         bool va = pp_unary (ref s);
2776
2777                         s = s.Trim ();
2778                         int len = s.Length;
2779                         if (len > 0){
2780                                 if (s [0] == '='){
2781                                         if (len > 2 && s [1] == '='){
2782                                                 s = s.Substring (2);
2783                                                 return va == pp_unary (ref s);
2784                                         } else {
2785                                                 Error_InvalidDirective ();
2786                                                 return false;
2787                                         }
2788                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2789                                         s = s.Substring (2);
2790
2791                                         return va != pp_unary (ref s);
2792
2793                                 }
2794                         }
2795
2796                         return va;
2797
2798                 }
2799
2800                 bool pp_and (ref string s)
2801                 {
2802                         bool va = pp_eq (ref s);
2803
2804                         s = s.Trim ();
2805                         int len = s.Length;
2806                         if (len > 0){
2807                                 if (s [0] == '&'){
2808                                         if (len > 2 && s [1] == '&'){
2809                                                 s = s.Substring (2);
2810                                                 return (va & pp_and (ref s));
2811                                         } else {
2812                                                 Error_InvalidDirective ();
2813                                                 return false;
2814                                         }
2815                                 }
2816                         }
2817                         return va;
2818                 }
2819
2820                 //
2821                 // Evaluates an expression for `#if' or `#elif'
2822                 //
2823                 bool pp_expr (ref string s, bool isTerm)
2824                 {
2825                         bool va = pp_and (ref s);
2826                         s = s.Trim ();
2827                         int len = s.Length;
2828                         if (len > 0){
2829                                 char c = s [0];
2830
2831                                 if (c == '|'){
2832                                         if (len > 2 && s [1] == '|'){
2833                                                 s = s.Substring (2);
2834                                                 return va | pp_expr (ref s, isTerm);
2835                                         } else {
2836                                                 Error_InvalidDirective ();
2837                                                 return false;
2838                                         }
2839                                 }
2840                                 if (isTerm) {
2841                                         Error_EndLineExpected ();
2842                                         return false;
2843                                 }
2844                         }
2845
2846                         return va;
2847                 }
2848
2849                 bool eval (string s)
2850                 {
2851                         bool v = pp_expr (ref s, true);
2852                         s = s.Trim ();
2853                         if (s.Length != 0){
2854                                 return false;
2855                         }
2856
2857                         return v;
2858                 }
2859
2860                 void Error_NumericConstantTooLong ()
2861                 {
2862                         Report.Error (1021, Location, "Integral constant is too large");
2863                 }
2864
2865                 void Error_InvalidDirective ()
2866                 {
2867                         Report.Error (1517, Location, "Invalid preprocessor directive");
2868                 }
2869
2870                 void Error_UnexpectedDirective (string extra)
2871                 {
2872                         Report.Error (
2873                                 1028, Location,
2874                                 "Unexpected processor directive ({0})", extra);
2875                 }
2876
2877                 void Error_TokensSeen ()
2878                 {
2879                         Report.Error (1032, Location,
2880                                 "Cannot define or undefine preprocessor symbols after first token in file");
2881                 }
2882
2883                 void Eror_WrongPreprocessorLocation ()
2884                 {
2885                         Report.Error (1040, Location,
2886                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2887                 }
2888
2889                 void Error_EndLineExpected ()
2890                 {
2891                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2892                 }
2893
2894                 //
2895                 // Raises a warning when tokenizer found documentation comment
2896                 // on unexpected place
2897                 //
2898                 void WarningMisplacedComment (Location loc)
2899                 {
2900                         if (doc_state != XmlCommentState.Error) {
2901                                 doc_state = XmlCommentState.Error;
2902                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2903                         }
2904                 }
2905
2906                 //
2907                 // if true, then the code continues processing the code
2908                 // if false, the code stays in a loop until another directive is
2909                 // reached.
2910                 // When caller_is_taking is false we ignore all directives except the ones
2911                 // which can help us to identify where the #if block ends
2912                 bool ParsePreprocessingDirective (bool caller_is_taking)
2913                 {
2914                         string arg;
2915                         bool region_directive = false;
2916
2917                         var directive = get_cmd_arg (out arg);
2918
2919                         //
2920                         // The first group of pre-processing instructions is always processed
2921                         //
2922                         switch (directive) {
2923                         case PreprocessorDirective.Region:
2924                                 region_directive = true;
2925                                 arg = "true";
2926                                 goto case PreprocessorDirective.If;
2927
2928                         case PreprocessorDirective.Endregion:
2929                                 if (ifstack == null || ifstack.Count == 0){
2930                                         Error_UnexpectedDirective ("no #region for this #endregion");
2931                                         return true;
2932                                 }
2933                                 int pop = ifstack.Pop ();
2934
2935                                 if ((pop & REGION) == 0)
2936                                         Report.Error (1027, Location, "Expected `#endif' directive");
2937
2938                                 return caller_is_taking;
2939
2940                         case PreprocessorDirective.If:
2941                                 if (ifstack == null)
2942                                         ifstack = new Stack<int> (2);
2943
2944                                 int flags = region_directive ? REGION : 0;
2945                                 if (ifstack.Count == 0){
2946                                         flags |= PARENT_TAKING;
2947                                 } else {
2948                                         int state = ifstack.Peek ();
2949                                         if ((state & TAKING) != 0) {
2950                                                 flags |= PARENT_TAKING;
2951                                         }
2952                                 }
2953
2954                                 if (eval (arg) && caller_is_taking) {
2955                                         ifstack.Push (flags | TAKING);
2956                                         return true;
2957                                 }
2958                                 ifstack.Push (flags);
2959                                 return false;
2960
2961                         case PreprocessorDirective.Endif:
2962                                 if (ifstack == null || ifstack.Count == 0){
2963                                         Error_UnexpectedDirective ("no #if for this #endif");
2964                                         return true;
2965                                 } else {
2966                                         pop = ifstack.Pop ();
2967
2968                                         if ((pop & REGION) != 0)
2969                                                 Report.Error (1038, Location, "#endregion directive expected");
2970
2971                                         if (arg.Length != 0) {
2972                                                 Error_EndLineExpected ();
2973                                         }
2974
2975                                         if (ifstack.Count == 0)
2976                                                 return true;
2977
2978                                         int state = ifstack.Peek ();
2979                                         return (state & TAKING) != 0;
2980                                 }
2981
2982                         case PreprocessorDirective.Elif:
2983                                 if (ifstack == null || ifstack.Count == 0){
2984                                         Error_UnexpectedDirective ("no #if for this #elif");
2985                                         return true;
2986                                 } else {
2987                                         int state = ifstack.Pop ();
2988
2989                                         if ((state & REGION) != 0) {
2990                                                 Report.Error (1038, Location, "#endregion directive expected");
2991                                                 return true;
2992                                         }
2993
2994                                         if ((state & ELSE_SEEN) != 0){
2995                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2996                                                 return true;
2997                                         }
2998
2999                                         if ((state & TAKING) != 0) {
3000                                                 ifstack.Push (0);
3001                                                 return false;
3002                                         }
3003
3004                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
3005                                                 ifstack.Push (state | TAKING);
3006                                                 return true;
3007                                         }
3008
3009                                         ifstack.Push (state);
3010                                         return false;
3011                                 }
3012
3013                         case PreprocessorDirective.Else:
3014                                 if (ifstack == null || ifstack.Count == 0){
3015                                         Error_UnexpectedDirective ("no #if for this #else");
3016                                         return true;
3017                                 } else {
3018                                         int state = ifstack.Peek ();
3019
3020                                         if ((state & REGION) != 0) {
3021                                                 Report.Error (1038, Location, "#endregion directive expected");
3022                                                 return true;
3023                                         }
3024
3025                                         if ((state & ELSE_SEEN) != 0){
3026                                                 Error_UnexpectedDirective ("#else within #else");
3027                                                 return true;
3028                                         }
3029
3030                                         ifstack.Pop ();
3031
3032                                         if (arg.Length != 0) {
3033                                                 Error_EndLineExpected ();
3034                                                 return true;
3035                                         }
3036
3037                                         bool ret = false;
3038                                         if ((state & PARENT_TAKING) != 0) {
3039                                                 ret = (state & TAKING) == 0;
3040
3041                                                 if (ret)
3042                                                         state |= TAKING;
3043                                                 else
3044                                                         state &= ~TAKING;
3045                                         }
3046
3047                                         ifstack.Push (state | ELSE_SEEN);
3048
3049                                         return ret;
3050                                 }
3051                         case PreprocessorDirective.Define:
3052                                 if (any_token_seen){
3053                                         if (caller_is_taking)
3054                                                 Error_TokensSeen ();
3055                                         return caller_is_taking;
3056                                 }
3057                                 PreProcessDefinition (true, arg, caller_is_taking);
3058                                 return caller_is_taking;
3059
3060                         case PreprocessorDirective.Undef:
3061                                 if (any_token_seen){
3062                                         if (caller_is_taking)
3063                                                 Error_TokensSeen ();
3064                                         return caller_is_taking;
3065                                 }
3066                                 PreProcessDefinition (false, arg, caller_is_taking);
3067                                 return caller_is_taking;
3068
3069                         case PreprocessorDirective.Invalid:
3070                                 Report.Error (1024, Location, "Wrong preprocessor directive");
3071                                 return true;
3072                         }
3073
3074                         //
3075                         // These are only processed if we are in a `taking' block
3076                         //
3077                         if (!caller_is_taking)
3078                                 return false;
3079
3080                         switch (directive){
3081                         case PreprocessorDirective.Error:
3082                                 Report.Error (1029, Location, "#error: '{0}'", arg);
3083                                 return true;
3084
3085                         case PreprocessorDirective.Warning:
3086                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
3087                                 return true;
3088
3089                         case PreprocessorDirective.Pragma:
3090                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
3091                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
3092                                 }
3093
3094                                 ParsePragmaDirective ();
3095                                 return true;
3096
3097                         case PreprocessorDirective.Line:
3098                                 Location loc = Location;
3099                                 if (!PreProcessLine ())
3100                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
3101
3102                                 return caller_is_taking;
3103                         }
3104
3105                         throw new NotImplementedException (directive.ToString ());
3106                 }
3107
3108                 int consume_string (bool quoted)
3109                 {
3110                         int c;
3111                         int pos = 0;
3112                         Location start_location = Location;
3113                         if (quoted)
3114                                 start_location = start_location - 1;
3115
3116 #if FULL_AST
3117                         int reader_pos = reader.Position;
3118 #endif
3119
3120                         while (true){
3121                                 // Cannot use get_char because of \r in quoted strings
3122                                 if (putback_char != -1) {
3123                                         c = putback_char;
3124                                         putback_char = -1;
3125                                 } else {
3126                                         c = reader.Read ();
3127                                 }
3128
3129                                 if (c == '"') {
3130                                         ++col;
3131
3132                                         if (quoted && peek_char () == '"') {
3133                                                 if (pos == value_builder.Length)
3134                                                         Array.Resize (ref value_builder, pos * 2);
3135
3136                                                 value_builder[pos++] = (char) c;
3137                                                 get_char ();
3138                                                 continue;
3139                                         }
3140
3141                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3142                                         val = res;
3143 #if FULL_AST
3144                                         res.ParsedValue = quoted ?
3145                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
3146                                                 reader.ReadChars (reader_pos - 1, reader.Position);
3147 #endif
3148
3149                                         return Token.LITERAL;
3150                                 }
3151
3152                                 if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3153                                         if (!quoted) {
3154                                                 Report.Error (1010, Location, "Newline in constant");
3155
3156                                                 advance_line ();
3157
3158                                                 // Don't add \r to string literal
3159                                                 if (pos > 1 && value_builder [pos - 1] == '\r')
3160                                                         --pos;
3161
3162                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
3163                                                 return Token.LITERAL;
3164                                         }
3165
3166                                         advance_line ();
3167                                 } else if (c == '\\' && !quoted) {
3168                                         ++col;
3169                                         int surrogate;
3170                                         c = escape (c, out surrogate);
3171                                         if (c == -1)
3172                                                 return Token.ERROR;
3173                                         if (surrogate != 0) {
3174                                                 if (pos == value_builder.Length)
3175                                                         Array.Resize (ref value_builder, pos * 2);
3176
3177                                                 value_builder[pos++] = (char) c;
3178                                                 c = surrogate;
3179                                         }
3180                                 } else if (c == -1) {
3181                                         Report.Error (1039, Location, "Unterminated string literal");
3182                                         return Token.EOF;
3183                                 } else {
3184                                         ++col;
3185                                 }
3186
3187                                 if (pos == value_builder.Length)
3188                                         Array.Resize (ref value_builder, pos * 2);
3189
3190                                 value_builder[pos++] = (char) c;
3191                         }
3192                 }
3193
3194                 private int consume_identifier (int s)
3195                 {
3196                         int res = consume_identifier (s, false);
3197
3198                         if (doc_state == XmlCommentState.Allowed)
3199                                 doc_state = XmlCommentState.NotAllowed;
3200
3201                         return res;
3202                 }
3203
3204                 int consume_identifier (int c, bool quoted)
3205                 {
3206                         //
3207                         // This method is very performance sensitive. It accounts
3208                         // for approximately 25% of all parser time
3209                         //
3210
3211                         int pos = 0;
3212                         int column = col;
3213                         if (quoted)
3214                                 --column;
3215
3216                         if (c == '\\') {
3217                                 int surrogate;
3218                                 c = escape (c, out surrogate);
3219                                 if (quoted || is_identifier_start_character (c)) {
3220                                         // it's added bellow
3221                                 } else if (surrogate != 0) {
3222                                         id_builder [pos++] = (char)c;
3223                                         c = surrogate;
3224                                 } else {
3225                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3226                                         return Token.ERROR;
3227                                 }
3228                         }
3229
3230                         id_builder [pos++] = (char) c;
3231
3232                         try {
3233                                 while (true) {
3234                                         c = reader.Read ();
3235
3236                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
3237                                                 id_builder [pos++] = (char) c;
3238                                                 continue;
3239                                         }
3240
3241                                         if (c < 0x80) {
3242                                                 if (c == '\\') {
3243                                                         int surrogate;
3244                                                         c = escape (c, out surrogate);
3245                                                         if (is_identifier_part_character ((char) c))
3246                                                                 id_builder[pos++] = (char) c;
3247                                                         else if (surrogate != 0) {
3248                                                                 c = surrogate;
3249                                                         } else {
3250                                                                 switch (c) {
3251                                                                 // TODO: Probably need more whitespace characters
3252                                                                 case 0xFEFF:
3253                                                                         putback_char = c;
3254                                                                         break;
3255                                                                 default:
3256                                                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3257                                                                         return Token.ERROR;
3258                                                                 }
3259                                                         }
3260
3261                                                         continue;
3262                                                 }
3263                                         } else if (is_identifier_part_character_slow_part ((char) c)) {
3264                                                 id_builder [pos++] = (char) c;
3265                                                 continue;
3266                                         }
3267
3268                                         putback_char = c;
3269                                         break;
3270                                 }
3271                         } catch (IndexOutOfRangeException) {
3272                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
3273                                 --pos;
3274                                 col += pos;
3275                         }
3276
3277                         col += pos - 1;
3278
3279                         //
3280                         // Optimization: avoids doing the keyword lookup
3281                         // on uppercase letters
3282                         //
3283                         if (id_builder [0] >= '_' && !quoted) {
3284                                 int keyword = GetKeyword (id_builder, pos);
3285                                 if (keyword != -1) {
3286                                         val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column);
3287                                         return keyword;
3288                                 }
3289                         }
3290
3291                         string s = InternIdentifier (id_builder, pos);
3292                         val = ltb.Create (s, current_source, ref_line, column);
3293                         if (quoted && parsing_attribute_section)
3294                                 AddEscapedIdentifier (((LocatedToken) val).Location);
3295
3296                         return Token.IDENTIFIER;
3297                 }
3298
3299                 string InternIdentifier (char[] charBuffer, int length)
3300                 {
3301                         //
3302                         // Keep identifiers in an array of hashtables to avoid needless
3303                         // allocations
3304                         //
3305                         var identifiers_group = identifiers[length];
3306                         string s;
3307                         if (identifiers_group != null) {
3308                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3309                                         return s;
3310                                 }
3311                         } else {
3312                                 // TODO: this should be number of files dependant
3313                                 // corlib compilation peaks at 1000 and System.Core at 150
3314                                 int capacity = length > 20 ? 10 : 100;
3315                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3316                                 identifiers[length] = identifiers_group;
3317                         }
3318
3319                         char[] chars = new char[length];
3320                         Array.Copy (charBuffer, chars, length);
3321
3322                         s = new string (charBuffer, 0, length);
3323                         identifiers_group.Add (chars, s);
3324                         return s;
3325                 }
3326
3327                 public int xtoken ()
3328                 {
3329                         if (parsing_interpolation_format) {
3330                                 return TokenizeInterpolationFormat ();
3331                         }
3332
3333                         int d, c;
3334
3335                         // Whether we have seen comments on the current line
3336                         bool comments_seen = false;
3337                         while ((c = get_char ()) != -1) {
3338                                 switch (c) {
3339                                 case '\t':
3340                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3341                                         continue;
3342
3343                                 case ' ':
3344                                 case '\f':
3345                                 case '\v':
3346                                 case 0xa0:
3347                                 case 0:
3348                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3349                                         continue;
3350
3351 /*                              This is required for compatibility with .NET
3352                                 case 0xEF:
3353                                         if (peek_char () == 0xBB) {
3354                                                 PushPosition ();
3355                                                 get_char ();
3356                                                 if (get_char () == 0xBF)
3357                                                         continue;
3358                                                 PopPosition ();
3359                                         }
3360                                         break;
3361 */
3362                                 case '\\':
3363                                         tokens_seen = true;
3364                                         return consume_identifier (c);
3365
3366                                 case '{':
3367                                         val = ltb.Create (current_source, ref_line, col);
3368                                         return Token.OPEN_BRACE;
3369                                 case '}':
3370                                         if (parsing_string_interpolation > 0) {
3371                                                 --parsing_string_interpolation;
3372                                                 return TokenizeInterpolatedString ();
3373                                         }
3374
3375                                         val = ltb.Create (current_source, ref_line, col);
3376                                         return Token.CLOSE_BRACE;
3377                                 case '[':
3378                                         // To block doccomment inside attribute declaration.
3379                                         if (doc_state == XmlCommentState.Allowed)
3380                                                 doc_state = XmlCommentState.NotAllowed;
3381
3382                                         val = ltb.Create (current_source, ref_line, col);
3383
3384                                         if (parsing_block == 0 || lambda_arguments_parsing)
3385                                                 return Token.OPEN_BRACKET;
3386
3387                                         int next = peek_char ();
3388                                         switch (next) {
3389                                         case ']':
3390                                         case ',':
3391                                                 return Token.OPEN_BRACKET;
3392
3393                                         case ' ':
3394                                         case '\f':
3395                                         case '\v':
3396                                         case '\r':
3397                                         case '\n':
3398                                         case UnicodeLS:
3399                                         case UnicodePS:
3400                                         case '/':
3401                                                 next = peek_token ();
3402                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3403                                                         return Token.OPEN_BRACKET;
3404
3405                                                 return Token.OPEN_BRACKET_EXPR;
3406                                         default:
3407                                                 return Token.OPEN_BRACKET_EXPR;
3408                                         }
3409                                 case ']':
3410                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3411                                         return Token.CLOSE_BRACKET;
3412                                 case '(':
3413                                         val = ltb.Create (current_source, ref_line, col);
3414                                         //
3415                                         // An expression versions of parens can appear in block context only
3416                                         //
3417                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3418
3419                                                 //
3420                                                 // Optmize most common case where we know that parens
3421                                                 // is not special
3422                                                 //
3423                                                 switch (current_token) {
3424                                                 case Token.IDENTIFIER:
3425                                                 case Token.IF:
3426                                                 case Token.FOR:
3427                                                 case Token.FOREACH:
3428                                                 case Token.TYPEOF:
3429                                                 case Token.WHILE:
3430                                                 case Token.SWITCH:
3431                                                 case Token.USING:
3432                                                 case Token.DEFAULT:
3433                                                 case Token.DELEGATE:
3434                                                 case Token.OP_GENERICS_GT:
3435                                                         return Token.OPEN_PARENS;
3436                                                 }
3437
3438                                                 // Optimize using peek
3439                                                 int xx = peek_char ();
3440                                                 switch (xx) {
3441                                                 case '(':
3442                                                 case '\'':
3443                                                 case '"':
3444                                                 case '0':
3445                                                 case '1':
3446                                                         return Token.OPEN_PARENS;
3447                                                 }
3448
3449                                                 lambda_arguments_parsing = true;
3450                                                 PushPosition ();
3451                                                 d = TokenizeOpenParens ();
3452                                                 PopPosition ();
3453                                                 lambda_arguments_parsing = false;
3454                                                 return d;
3455                                         }
3456
3457                                         return Token.OPEN_PARENS;
3458                                 case ')':
3459                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3460                                         return Token.CLOSE_PARENS;
3461                                 case ',':
3462                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3463                                         return Token.COMMA;
3464                                 case ';':
3465                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3466                                         return Token.SEMICOLON;
3467                                 case '~':
3468                                         val = ltb.Create (current_source, ref_line, col);
3469                                         return Token.TILDE;
3470                                 case '?':
3471                                         val = ltb.Create (current_source, ref_line, col);
3472                                         return TokenizePossibleNullableType ();
3473                                 case '<':
3474                                         val = ltb.Create (current_source, ref_line, col);
3475                                         if (parsing_generic_less_than++ > 0)
3476                                                 return Token.OP_GENERICS_LT;
3477
3478                                         return TokenizeLessThan ();
3479
3480                                 case '>':
3481                                         val = ltb.Create (current_source, ref_line, col);
3482                                         d = peek_char ();
3483
3484                                         if (d == '='){
3485                                                 get_char ();
3486                                                 return Token.OP_GE;
3487                                         }
3488
3489                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3490                                                 parsing_generic_less_than--;
3491                                                 return Token.OP_GENERICS_GT;
3492                                         }
3493
3494                                         if (d == '>') {
3495                                                 get_char ();
3496                                                 d = peek_char ();
3497
3498                                                 if (d == '=') {
3499                                                         get_char ();
3500                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3501                                                 }
3502                                                 return Token.OP_SHIFT_RIGHT;
3503                                         }
3504
3505                                         return Token.OP_GT;
3506
3507                                 case '+':
3508                                         val = ltb.Create (current_source, ref_line, col);
3509                                         d = peek_char ();
3510                                         if (d == '+') {
3511                                                 d = Token.OP_INC;
3512                                         } else if (d == '=') {
3513                                                 d = Token.OP_ADD_ASSIGN;
3514                                         } else {
3515                                                 return Token.PLUS;
3516                                         }
3517                                         get_char ();
3518                                         return d;
3519
3520                                 case '-':
3521                                         val = ltb.Create (current_source, ref_line, col);
3522                                         d = peek_char ();
3523                                         if (d == '-') {
3524                                                 d = Token.OP_DEC;
3525                                         } else if (d == '=')
3526                                                 d = Token.OP_SUB_ASSIGN;
3527                                         else if (d == '>')
3528                                                 d = Token.OP_PTR;
3529                                         else {
3530                                                 return Token.MINUS;
3531                                         }
3532                                         get_char ();
3533                                         return d;
3534
3535                                 case '!':
3536                                         val = ltb.Create (current_source, ref_line, col);
3537                                         if (peek_char () == '='){
3538                                                 get_char ();
3539                                                 return Token.OP_NE;
3540                                         }
3541                                         return Token.BANG;
3542
3543                                 case '=':
3544                                         val = ltb.Create (current_source, ref_line, col);
3545                                         d = peek_char ();
3546                                         if (d == '='){
3547                                                 get_char ();
3548                                                 return Token.OP_EQ;
3549                                         }
3550                                         if (d == '>'){
3551                                                 get_char ();
3552                                                 return Token.ARROW;
3553                                         }
3554
3555                                         return Token.ASSIGN;
3556
3557                                 case '&':
3558                                         val = ltb.Create (current_source, ref_line, col);
3559                                         d = peek_char ();
3560                                         if (d == '&'){
3561                                                 get_char ();
3562                                                 return Token.OP_AND;
3563                                         }
3564                                         if (d == '='){
3565                                                 get_char ();
3566                                                 return Token.OP_AND_ASSIGN;
3567                                         }
3568                                         return Token.BITWISE_AND;
3569
3570                                 case '|':
3571                                         val = ltb.Create (current_source, ref_line, col);
3572                                         d = peek_char ();
3573                                         if (d == '|'){
3574                                                 get_char ();
3575                                                 return Token.OP_OR;
3576                                         }
3577                                         if (d == '='){
3578                                                 get_char ();
3579                                                 return Token.OP_OR_ASSIGN;
3580                                         }
3581                                         return Token.BITWISE_OR;
3582
3583                                 case '*':
3584                                         val = ltb.Create (current_source, ref_line, col);
3585                                         if (peek_char () == '='){
3586                                                 get_char ();
3587                                                 return Token.OP_MULT_ASSIGN;
3588                                         }
3589                                         return Token.STAR;
3590
3591                                 case '/':
3592                                         d = peek_char ();
3593                                         if (d == '='){
3594                                                 val = ltb.Create (current_source, ref_line, col);
3595                                                 get_char ();
3596                                                 return Token.OP_DIV_ASSIGN;
3597                                         }
3598
3599                                         // Handle double-slash comments.
3600                                         if (d == '/'){
3601                                                 if (parsing_string_interpolation > 0) {
3602                                                         Report.Error (8077, Location, "A single-line comment may not be used in an interpolated string");
3603                                                         goto case '}';
3604                                                 }
3605
3606                                                 get_char ();
3607                                                 if (doc_processing) {
3608                                                         if (peek_char () == '/') {
3609                                                                 get_char ();
3610                                                                 // Don't allow ////.
3611                                                                 if ((d = peek_char ()) != '/') {
3612                                                                         if (doc_state == XmlCommentState.Allowed)
3613                                                                                 handle_one_line_xml_comment ();
3614                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3615                                                                                 WarningMisplacedComment (Location - 3);
3616                                                                 }
3617                                                         } else {
3618                                                                 if (xml_comment_buffer.Length > 0)
3619                                                                         doc_state = XmlCommentState.NotAllowed;
3620                                                         }
3621                                                 }
3622
3623                                                 ReadToEndOfLine ();
3624
3625                                                 any_token_seen |= tokens_seen;
3626                                                 tokens_seen = false;
3627                                                 comments_seen = false;
3628                                                 continue;
3629                                         } else if (d == '*'){
3630                                                 get_char ();
3631                                                 bool docAppend = false;
3632                                                 if (doc_processing && peek_char () == '*') {
3633                                                         get_char ();
3634                                                         // But when it is /**/, just do nothing.
3635                                                         if (peek_char () == '/') {
3636                                                                 get_char ();
3637                                                                 continue;
3638                                                         }
3639                                                         if (doc_state == XmlCommentState.Allowed)
3640                                                                 docAppend = true;
3641                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3642                                                                 WarningMisplacedComment (Location - 2);
3643                                                         }
3644                                                 }
3645
3646                                                 int current_comment_start = 0;
3647                                                 if (docAppend) {
3648                                                         current_comment_start = xml_comment_buffer.Length;
3649                                                         xml_comment_buffer.Append (Environment.NewLine);
3650                                                 }
3651
3652                                                 while ((d = get_char ()) != -1){
3653                                                         if (d == '*' && peek_char () == '/'){
3654                                                                 get_char ();
3655                                                                 comments_seen = true;
3656                                                                 break;
3657                                                         }
3658                                                         if (docAppend)
3659                                                                 xml_comment_buffer.Append ((char) d);
3660
3661                                                         if (d == '\n' || d == UnicodeLS || d == UnicodePS){
3662                                                                 any_token_seen |= tokens_seen;
3663                                                                 tokens_seen = false;
3664                                                                 //
3665                                                                 // Reset 'comments_seen' just to be consistent.
3666                                                                 // It doesn't matter either way, here.
3667                                                                 //
3668                                                                 comments_seen = false;
3669                                                         }
3670                                                 }
3671                                                 if (!comments_seen)
3672                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3673
3674                                                 if (docAppend)
3675                                                         update_formatted_doc_comment (current_comment_start);
3676                                                 continue;
3677                                         }
3678                                         val = ltb.Create (current_source, ref_line, col);
3679                                         return Token.DIV;
3680
3681                                 case '%':
3682                                         val = ltb.Create (current_source, ref_line, col);
3683                                         if (peek_char () == '='){
3684                                                 get_char ();
3685                                                 return Token.OP_MOD_ASSIGN;
3686                                         }
3687                                         return Token.PERCENT;
3688
3689                                 case '^':
3690                                         val = ltb.Create (current_source, ref_line, col);
3691                                         if (peek_char () == '='){
3692                                                 get_char ();
3693                                                 return Token.OP_XOR_ASSIGN;
3694                                         }
3695                                         return Token.CARRET;
3696
3697                                 case ':':
3698                                         val = ltb.Create (current_source, ref_line, col);
3699                                         if (peek_char () == ':') {
3700                                                 get_char ();
3701                                                 return Token.DOUBLE_COLON;
3702                                         }
3703                                         return Token.COLON;
3704
3705                                 case '0': case '1': case '2': case '3': case '4':
3706                                 case '5': case '6': case '7': case '8': case '9':
3707                                         tokens_seen = true;
3708                                         return is_number (c, false);
3709
3710                                 case '\n': // white space
3711                                 case UnicodeLS:
3712                                 case UnicodePS:
3713                                         any_token_seen |= tokens_seen;
3714                                         tokens_seen = false;
3715                                         comments_seen = false;
3716                                         continue;
3717
3718                                 case '.':
3719                                         tokens_seen = true;
3720                                         d = peek_char ();
3721                                         if (d >= '0' && d <= '9')
3722                                                 return is_number (c, true);
3723
3724                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3725                                         return Token.DOT;
3726
3727                                 case '#':
3728                                         if (tokens_seen || comments_seen) {
3729                                                 Eror_WrongPreprocessorLocation ();
3730                                                 return Token.ERROR;
3731                                         }
3732
3733                                         if (ParsePreprocessingDirective (true))
3734                                                 continue;
3735
3736                                         bool directive_expected = false;
3737                                         while ((c = get_char ()) != -1) {
3738                                                 if (col == 1) {
3739                                                         directive_expected = true;
3740                                                 } else if (!directive_expected) {
3741                                                         // TODO: Implement comment support for disabled code and uncomment this code
3742 //                                                      if (c == '#') {
3743 //                                                              Eror_WrongPreprocessorLocation ();
3744 //                                                              return Token.ERROR;
3745 //                                                      }
3746                                                         continue;
3747                                                 }
3748
3749                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
3750                                                         continue;
3751
3752                                                 if (c == '#') {
3753                                                         if (ParsePreprocessingDirective (false))
3754                                                                 break;
3755                                                 }
3756                                                 directive_expected = false;
3757                                         }
3758
3759                                         if (c != -1) {
3760                                                 tokens_seen = false;
3761                                                 continue;
3762                                         }
3763
3764                                         return Token.EOF;
3765
3766                                 case '"':
3767                                         if (parsing_string_interpolation > 0 && !ScanClosingInterpolationBrace ()) {
3768                                                 parsing_string_interpolation = 0;
3769                                                 Report.Error (8076, Location, "Missing close delimiter `}' for interpolated expression");
3770                                                 val = new StringLiteral (context.BuiltinTypes, "", Location);
3771                                                 return Token.INTERPOLATED_STRING_END;
3772                                         }
3773
3774                                         return consume_string (false);
3775
3776                                 case '\'':
3777                                         return TokenizeBackslash ();
3778
3779                                 case '@':
3780                                         c = get_char ();
3781                                         if (c == '"') {
3782                                                 tokens_seen = true;
3783                                                 return consume_string (true);
3784                                         }
3785
3786                                         if (is_identifier_start_character (c)){
3787                                                 return consume_identifier (c, true);
3788                                         }
3789
3790                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3791                                         return Token.ERROR;
3792
3793                                 case '$':
3794                                         if (peek_char () == '"') {
3795                                                 get_char ();
3796                                                 return TokenizeInterpolatedString ();
3797                                         }
3798
3799                                         break;
3800                                 case EvalStatementParserCharacter:
3801                                         return Token.EVAL_STATEMENT_PARSER;
3802                                 case EvalCompilationUnitParserCharacter:
3803                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3804                                 case EvalUsingDeclarationsParserCharacter:
3805                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3806                                 case DocumentationXref:
3807                                         return Token.DOC_SEE;
3808                                 }
3809
3810                                 if (is_identifier_start_character (c)) {
3811                                         tokens_seen = true;
3812                                         return consume_identifier (c);
3813                                 }
3814
3815                                 if (char.IsWhiteSpace ((char) c))
3816                                         continue;
3817
3818                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3819                         }
3820
3821                         if (CompleteOnEOF){
3822                                 if (generated)
3823                                         return Token.COMPLETE_COMPLETION;
3824
3825                                 generated = true;
3826                                 return Token.GENERATE_COMPLETION;
3827                         }
3828
3829
3830                         return Token.EOF;
3831                 }
3832
3833                 int TokenizeBackslash ()
3834                 {
3835 #if FULL_AST
3836                         int read_start = reader.Position;
3837 #endif
3838                         Location start_location = Location;
3839                         int c = get_char ();
3840                         tokens_seen = true;
3841                         if (c == '\'') {
3842                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3843                                 Report.Error (1011, start_location, "Empty character literal");
3844                                 return Token.LITERAL;
3845                         }
3846
3847                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3848                                 Report.Error (1010, start_location, "Newline in constant");
3849                                 return Token.ERROR;
3850                         }
3851
3852                         int d;
3853                         c = escape (c, out d);
3854                         if (c == -1)
3855                                 return Token.ERROR;
3856                         if (d != 0)
3857                                 throw new NotImplementedException ();
3858
3859                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3860                         val = res;
3861                         c = get_char ();
3862
3863                         if (c != '\'') {
3864                                 Report.Error (1012, start_location, "Too many characters in character literal");
3865
3866                                 // Try to recover, read until newline or next "'"
3867                                 while ((c = get_char ()) != -1) {
3868                                         if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
3869                                                 break;
3870                                 }
3871                         }
3872
3873 #if FULL_AST
3874                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3875 #endif
3876
3877                         return Token.LITERAL;
3878                 }
3879
3880                 int TokenizeLessThan ()
3881                 {
3882                         int d;
3883
3884                         // Save current position and parse next token.
3885                         PushPosition ();
3886                         int generic_dimension = 0;
3887                         if (parse_less_than (ref generic_dimension)) {
3888                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3889                                         d = Token.OP_GENERICS_LT_DECL;
3890                                 } else {
3891                                         if (generic_dimension > 0) {
3892                                                 val = generic_dimension;
3893                                                 DiscardPosition ();
3894                                                 return Token.GENERIC_DIMENSION;
3895                                         }
3896
3897                                         d = Token.OP_GENERICS_LT;
3898                                 }
3899                                 PopPosition ();
3900                                 return d;
3901                         }
3902
3903                         PopPosition ();
3904                         parsing_generic_less_than = 0;
3905
3906                         d = peek_char ();
3907                         if (d == '<') {
3908                                 get_char ();
3909                                 d = peek_char ();
3910
3911                                 if (d == '=') {
3912                                         get_char ();
3913                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3914                                 }
3915                                 return Token.OP_SHIFT_LEFT;
3916                         }
3917
3918                         if (d == '=') {
3919                                 get_char ();
3920                                 return Token.OP_LE;
3921                         }
3922                         return Token.OP_LT;
3923                 }
3924
3925                 int TokenizeInterpolatedString ()
3926                 {
3927                         int pos = 0;
3928                         var start_location = Location;
3929
3930                         while (true) {
3931                                 var ch = get_char ();
3932                                 switch (ch) {
3933                                 case '"':
3934                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3935                                         return Token.INTERPOLATED_STRING_END;
3936                                 case '{':
3937                                         if (peek_char () == '{') {
3938                                                 value_builder [pos++] = (char)ch;
3939                                                 get_char ();
3940                                                 break;
3941                                         }
3942
3943                                         ++parsing_string_interpolation;
3944                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3945                                         return Token.INTERPOLATED_STRING;
3946                                 case '\\':
3947                                         ++col;
3948                                         int surrogate;
3949                                         ch = escape (ch, out surrogate);
3950                                         if (ch == -1)
3951                                                 return Token.ERROR;
3952
3953                                         if (ch == '{' || ch == '}') {
3954                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
3955                                         }
3956
3957                                         if (surrogate != 0) {
3958                                                 if (pos == value_builder.Length)
3959                                                         Array.Resize (ref value_builder, pos * 2);
3960
3961                                                 if (pos == value_builder.Length)
3962                                                         Array.Resize (ref value_builder, pos * 2);
3963
3964                                                 value_builder [pos++] = (char)ch;
3965                                                 ch = surrogate;
3966                                         }
3967
3968                                         break;
3969                                 case -1:
3970                                         return Token.EOF;
3971                                 }
3972
3973                                 ++col;
3974                                 if (pos == value_builder.Length)
3975                                         Array.Resize (ref value_builder, pos * 2);
3976
3977                                 value_builder[pos++] = (char) ch;
3978                         }
3979                 }
3980
3981                 int TokenizeInterpolationFormat ()
3982                 {
3983                         int pos = 0;
3984                         int braces = 0;
3985                         while (true) {
3986                                 var ch = get_char ();
3987                                 switch (ch) {
3988                                 case '{':
3989                                         ++braces;
3990                                         break;
3991                                 case '}':
3992                                         if (braces == 0) {
3993                                                 putback_char = ch;
3994                                                 if (pos == 0) {
3995                                                         Report.Error (8089, Location, "Empty interpolated expression format specifier");
3996                                                 } else if (Array.IndexOf (simple_whitespaces, value_builder [pos - 1]) >= 0) {
3997                                                         Report.Error (8088, Location, "A interpolated expression format specifier may not contain trailing whitespace");
3998                                                 }
3999
4000                                                 val = CreateStringFromBuilder (pos);
4001                                                 return Token.LITERAL;
4002                                         }
4003
4004                                         --braces;
4005                                         break;
4006                                 case '\\':
4007                                         ++col;
4008                                         int surrogate;
4009                                         ch = escape (ch, out surrogate);
4010                                         if (ch == -1)
4011                                                 return Token.ERROR;
4012
4013                                         if (ch == '{' || ch == '}') {
4014                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
4015                                         }
4016
4017                                         if (surrogate != 0) {
4018                                                 if (pos == value_builder.Length)
4019                                                         Array.Resize (ref value_builder, pos * 2);
4020
4021                                                 value_builder [pos++] = (char)ch;
4022                                                 ch = surrogate;
4023                                         }
4024
4025                                         break;
4026                                 case -1:
4027                                         return Token.EOF;
4028                                 }
4029
4030                                 ++col;
4031                                 value_builder[pos++] = (char) ch;
4032                         }
4033                 }
4034
4035                 string CreateStringFromBuilder (int pos)
4036                 {
4037                         if (pos == 0)
4038                                 return string.Empty;
4039                         if (pos <= 4)
4040                                 return InternIdentifier (value_builder, pos);
4041
4042                         return new string (value_builder, 0, pos);
4043                 }
4044
4045                 //
4046                 // Handles one line xml comment
4047                 //
4048                 private void handle_one_line_xml_comment ()
4049                 {
4050                         int c;
4051                         while ((c = peek_char ()) == ' ')
4052                                 get_char (); // skip heading whitespaces.
4053                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
4054                                 xml_comment_buffer.Append ((char) get_char ());
4055                         }
4056                         if (c == '\r' || c == '\n')
4057                                 xml_comment_buffer.Append (Environment.NewLine);
4058                 }
4059
4060                 //
4061                 // Remove heading "*" in Javadoc-like xml documentation.
4062                 //
4063                 private void update_formatted_doc_comment (int current_comment_start)
4064                 {
4065                         int length = xml_comment_buffer.Length - current_comment_start;
4066                         string [] lines = xml_comment_buffer.ToString (
4067                                 current_comment_start,
4068                                 length).Replace ("\r", "").Split ('\n');
4069
4070                         // The first line starts with /**, thus it is not target
4071                         // for the format check.
4072                         for (int i = 1; i < lines.Length; i++) {
4073                                 string s = lines [i];
4074                                 int idx = s.IndexOf ('*');
4075                                 string head = null;
4076                                 if (idx < 0) {
4077                                         if (i < lines.Length - 1)
4078                                                 return;
4079                                         head = s;
4080                                 } else
4081                                         head = s.Substring (0, idx);
4082                                 foreach (char c in head)
4083                                         if (c != ' ')
4084                                                 return;
4085                                 lines [i] = s.Substring (idx + 1);
4086                         }
4087                         xml_comment_buffer.Remove (current_comment_start, length);
4088                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
4089                 }
4090
4091                 //
4092                 // Checks if there was incorrect doc comments and raise
4093                 // warnings.
4094                 //
4095                 public void check_incorrect_doc_comment ()
4096                 {
4097                         if (xml_comment_buffer.Length > 0)
4098                                 WarningMisplacedComment (Location);
4099                 }
4100
4101                 //
4102                 // Consumes the saved xml comment lines (if any)
4103                 // as for current target member or type.
4104                 //
4105                 public string consume_doc_comment ()
4106                 {
4107                         if (xml_comment_buffer.Length > 0) {
4108                                 string ret = xml_comment_buffer.ToString ();
4109                                 reset_doc_comment ();
4110                                 return ret;
4111                         }
4112                         return null;
4113                 }
4114
4115                 void reset_doc_comment ()
4116                 {
4117                         xml_comment_buffer.Length = 0;
4118                 }
4119
4120                 public void cleanup ()
4121                 {
4122                         if (ifstack != null && ifstack.Count >= 1) {
4123                                 int state = ifstack.Pop ();
4124                                 if ((state & REGION) != 0)
4125                                         Report.Error (1038, Location, "#endregion directive expected");
4126                                 else
4127                                         Report.Error (1027, Location, "Expected `#endif' directive");
4128                         }
4129                 }
4130         }
4131
4132         //
4133         // Indicates whether it accepts XML documentation or not.
4134         //
4135         public enum XmlCommentState {
4136                 // comment is allowed in this state.
4137                 Allowed,
4138                 // comment is not allowed in this state.
4139                 NotAllowed,
4140                 // once comments appeared when it is NotAllowed, then the
4141                 // state is changed to it, until the state is changed to
4142                 // .Allowed.
4143                 Error
4144         }
4145 }
4146