mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20 using System.Collections;
  21
  22 namespace Mono.CSharp
  23 {
  24         //
  25         // This class has to be used by parser only, it reuses token
  26         // details once a file is parsed
  27         //
  28         public class LocatedToken
  29         {
  30                 public int row, column;
  31                 public string value;
  32                 public SourceFile file;
  33
  34                 public LocatedToken ()
  35                 {
  36                 }
  37
  38                 public LocatedToken (string value, Location loc)
  39                 {
  40                         this.value = value;
  41                         file = loc.SourceFile;
  42                         row = loc.Row;
  43                         column = loc.Column;
  44                 }
  45
  46                 public override string ToString ()
  47                 {
  48                         return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  49                 }
  50
  51                 public Location Location
  52                 {
  53                         get { return new Location (file, row, column); }
  54                 }
  55
  56                 public string Value
  57                 {
  58                         get { return value; }
  59                 }
  60         }
  61
  62         /// <summary>
  63         ///    Tokenizer for C# source code.
  64         /// </summary>
  65         public class Tokenizer : yyParser.yyInput
  66         {
  67                 class KeywordEntry<T>
  68                 {
  69                         public readonly T Token;
  70                         public KeywordEntry<T> Next;
  71                         public readonly char[] Value;
  72
  73                         public KeywordEntry (string value, T token)
  74                         {
  75                                 this.Value = value.ToCharArray ();
  76                                 this.Token = token;
  77                         }
  78                 }
  79
  80                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  81                 {
  82                         readonly int length;
  83
  84                         public IdentifiersComparer (int length)
  85                         {
  86                                 this.length = length;
  87                         }
  88
  89                         public bool Equals (char[] x, char[] y)
  90                         {
  91                                 for (int i = 0; i < length; ++i)
  92                                         if (x [i] != y [i])
  93                                                 return false;
  94
  95                                 return true;
  96                         }
  97
  98                         public int GetHashCode (char[] obj)
  99                         {
 100                                 int h = 0;
 101                                 for (int i = 0; i < length; ++i)
 102                                         h = (h << 5) - h + obj [i];
 103
 104                                 return h;
 105                         }
 106                 }
 107
 108                 public class LocatedTokenBuffer
 109                 {
 110                         readonly LocatedToken[] buffer;
 111                         public int pos;
 112
 113                         public LocatedTokenBuffer ()
 114                         {
 115                                 buffer = new LocatedToken[0];
 116                         }
 117
 118                         public LocatedTokenBuffer (LocatedToken[] buffer)
 119                         {
 120                                 this.buffer = buffer ?? new LocatedToken[0];
 121                         }
 122
 123                         public LocatedToken Create (SourceFile file, int row, int column)
 124                         {
 125                                 return Create (null, file, row, column);
 126                         }
 127
 128                         public LocatedToken Create (string value, SourceFile file, int row, int column)
 129                         {
 130                                 //
 131                                 // TODO: I am not very happy about the logic but it's the best
 132                                 // what I could come up with for now.
 133                                 // Ideally we should be using just tiny buffer (256 elements) which
 134                                 // is enough to hold all details for currect stack and recycle elements
 135                                 // poped from the stack but there is a trick needed to recycle
 136                                 // them properly.
 137                                 //
 138                                 LocatedToken entry;
 139                                 if (pos >= buffer.Length) {
 140                                         entry = new LocatedToken ();
 141                                 } else {
 142                                         entry = buffer[pos];
 143                                         if (entry == null) {
 144                                                 entry = new LocatedToken ();
 145                                                 buffer[pos] = entry;
 146                                         }
 147
 148                                         ++pos;
 149                                 }
 150                                 entry.value = value;
 151                                 entry.file = file;
 152                                 entry.row = row;
 153                                 entry.column = column;
 154                                 return entry;
 155                         }
 156
 157                         //
 158                         // Used for token not required by expression evaluator
 159                         //
 160                         [Conditional ("FULL_AST")]
 161                         public void CreateOptional (SourceFile file, int row, int col, ref object token)
 162                         {
 163                                 token = Create (file, row, col);
 164                         }
 165                 }
 166
 167                 public enum PreprocessorDirective
 168                 {
 169                         Invalid = 0,
 170
 171                         Region = 1,
 172                         Endregion = 2,
 173                         If = 3 | RequiresArgument,
 174                         Endif = 4,
 175                         Elif = 5 | RequiresArgument,
 176                         Else = 6,
 177                         Define = 7 | RequiresArgument,
 178                         Undef = 8 | RequiresArgument,
 179                         Error = 9,
 180                         Warning = 10,
 181                         Pragma = 11 | CustomArgumentsParsing,
 182                         Line = 12 | CustomArgumentsParsing,
 183
 184                         CustomArgumentsParsing = 1 << 10,
 185                         RequiresArgument = 1 << 11
 186                 }
 187
 188                 readonly SeekableStreamReader reader;
 189                 readonly CompilationSourceFile source_file;
 190                 readonly CompilerContext context;
 191                 readonly Report Report;
 192
 193
 194                 SourceFile current_source;
 195                 Location hidden_block_start;
 196                 int ref_line = 1;
 197                 int line = 1;
 198                 int col = 0;
 199                 int previous_col;
 200                 int current_token;
 201                 readonly int tab_size;
 202                 bool handle_get_set = false;
 203                 bool handle_remove_add = false;
 204                 bool handle_where;
 205                 bool lambda_arguments_parsing;
 206                 List<Location> escaped_identifiers;
 207                 int parsing_generic_less_than;
 208                 readonly bool doc_processing;
 209                 readonly LocatedTokenBuffer ltb;
 210
 211                 //
 212                 // Used mainly for parser optimizations. Some expressions for instance
 213                 // can appear only in block (including initializer, base initializer)
 214                 // scope only
 215                 //
 216                 public int parsing_block;
 217                 internal bool query_parsing;
 218
 219                 //
 220                 // When parsing type only, useful for ambiguous nullable types
 221                 //
 222                 public int parsing_type;
 223
 224                 //
 225                 // Set when parsing generic declaration (type or method header)
 226                 //
 227                 public bool parsing_generic_declaration;
 228                 public bool parsing_generic_declaration_doc;
 229
 230                 //
 231                 // The value indicates that we have not reach any declaration or
 232                 // namespace yet
 233                 //
 234                 public int parsing_declaration;
 235
 236                 public bool parsing_attribute_section;
 237
 238                 public bool parsing_modifiers;
 239
 240                 public bool parsing_catch_when;
 241
 242                 int parsing_string_interpolation;
 243                 Stack<bool> parsing_string_interpolation_quoted;
 244
 245                 public bool parsing_interpolation_format;
 246
 247                 //
 248                 // The special characters to inject on streams to run the unit parser
 249                 // in the special expression mode. Using private characters from
 250                 // Plane Sixteen (U+100000 to U+10FFFD)
 251                 //
 252                 // This character is only tested just before the tokenizer is about to report
 253                 // an error;   So on the regular operation mode, this addition will have no
 254                 // impact on the tokenizer's performance.
 255                 //
 256
 257                 public const int EvalStatementParserCharacter = 0x100000;
 258                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 259                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 260                 public const int DocumentationXref = 0x100003;
 261
 262                 const int UnicodeLS = 0x2028;
 263                 const int UnicodePS = 0x2029;
 264
 265                 //
 266                 // XML documentation buffer. The save point is used to divide
 267                 // comments on types and comments on members.
 268                 //
 269                 StringBuilder xml_comment_buffer;
 270
 271                 //
 272                 // See comment on XmlCommentState enumeration.
 273                 //
 274                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 275
 276                 //
 277                 // Whether tokens have been seen on this line
 278                 //
 279                 bool tokens_seen = false;
 280
 281                 //
 282                 // Set to true once the GENERATE_COMPLETION token has bee
 283                 // returned.   This helps produce one GENERATE_COMPLETION,
 284                 // as many COMPLETE_COMPLETION as necessary to complete the
 285                 // AST tree and one final EOF.
 286                 //
 287                 bool generated;
 288
 289                 //
 290                 // Whether a token has been seen on the file
 291                 // This is needed because `define' is not allowed to be used
 292                 // after a token has been seen.
 293                 //
 294                 bool any_token_seen;
 295
 296                 //
 297                 // Class variables
 298                 //
 299                 static readonly KeywordEntry<int>[][] keywords;
 300                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 301                 static readonly HashSet<string> keyword_strings;
 302                 static readonly NumberStyles styles;
 303                 static readonly NumberFormatInfo csharp_format_info;
 304
 305                 // Pragma arguments
 306                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 307                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 308                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 309                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 310                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 311                 static readonly char[] line_default = "default".ToCharArray ();
 312
 313                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 314
 315                 public bool PropertyParsing {
 316                         get { return handle_get_set; }
 317                         set { handle_get_set = value; }
 318                 }
 319
 320                 public bool EventParsing {
 321                         get { return handle_remove_add; }
 322                         set { handle_remove_add = value; }
 323                 }
 324
 325                 public bool ConstraintsParsing {
 326                         get { return handle_where; }
 327                         set { handle_where = value; }
 328                 }
 329
 330                 public XmlCommentState doc_state {
 331                         get { return xml_doc_state; }
 332                         set {
 333                                 if (value == XmlCommentState.Allowed) {
 334                                         check_incorrect_doc_comment ();
 335                                         reset_doc_comment ();
 336                                 }
 337                                 xml_doc_state = value;
 338                         }
 339                 }
 340
 341                 //
 342                 // This is used to trigger completion generation on the parser
 343                 public bool CompleteOnEOF;
 344
 345                 void AddEscapedIdentifier (Location loc)
 346                 {
 347                         if (escaped_identifiers == null)
 348                                 escaped_identifiers = new List<Location> ();
 349
 350                         escaped_identifiers.Add (loc);
 351                 }
 352
 353                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 354                 {
 355                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 356                 }
 357
 358                 //
 359                 // Values for the associated token returned
 360                 //
 361                 internal int putback_char;      // Used by repl only
 362                 object val;
 363
 364                 //
 365                 // Pre-processor
 366                 //
 367                 const int TAKING        = 1;
 368                 const int ELSE_SEEN     = 4;
 369                 const int PARENT_TAKING = 8;
 370                 const int REGION        = 16;
 371
 372                 //
 373                 // pre-processor if stack state:
 374                 //
 375                 Stack<int> ifstack;
 376
 377                 public const int MaxIdentifierLength = 512;
 378                 public const int MaxNumberLength = 512;
 379
 380                 readonly char[] id_builder;
 381                 readonly Dictionary<char[], string>[] identifiers;
 382                 readonly char[] number_builder;
 383                 int number_pos;
 384
 385                 char[] value_builder = new char[64];
 386
 387                 public int Line {
 388                         get {
 389                                 return ref_line;
 390                         }
 391                 }
 392
 393                 //
 394                 // This is used when the tokenizer needs to save
 395                 // the current position as it needs to do some parsing
 396                 // on its own to deamiguate a token in behalf of the
 397                 // parser.
 398                 //
 399                 Stack<Position> position_stack = new Stack<Position> (2);
 400
 401                 class Position {
 402                         public int position;
 403                         public int line;
 404                         public int ref_line;
 405                         public int col;
 406                         public Location hidden;
 407                         public int putback_char;
 408                         public int previous_col;
 409                         public Stack<int> ifstack;
 410                         public int parsing_generic_less_than;
 411                         public int current_token;
 412                         public object val;
 413                         public int parsing_string_interpolation;
 414                         public Stack<bool> parsing_string_interpolation_quoted;
 415
 416                         public Position (Tokenizer t)
 417                         {
 418                                 position = t.reader.Position;
 419                                 line = t.line;
 420                                 ref_line = t.ref_line;
 421                                 col = t.col;
 422                                 hidden = t.hidden_block_start;
 423                                 putback_char = t.putback_char;
 424                                 previous_col = t.previous_col;
 425                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 426                                         // There is no simple way to clone Stack<T> all
 427                                         // methods reverse the order
 428                                         var clone = t.ifstack.ToArray ();
 429                                         Array.Reverse (clone);
 430                                         ifstack = new Stack<int> (clone);
 431                                 }
 432                                 parsing_generic_less_than = t.parsing_generic_less_than;
 433                                 current_token = t.current_token;
 434                                 val = t.val;
 435                                 parsing_string_interpolation = t.parsing_string_interpolation;
 436                                 if (t.parsing_string_interpolation_quoted != null && t.parsing_string_interpolation_quoted.Count != 0) {
 437                                         var clone = t.parsing_string_interpolation_quoted.ToArray ();
 438                                         Array.Reverse (clone);
 439                                         parsing_string_interpolation_quoted = new Stack<bool> (clone);
 440                                 }
 441                         }
 442                 }
 443
 444                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 445                 {
 446                         this.source_file = file;
 447                         this.context = file.Compiler;
 448                         this.current_source = file.SourceFile;
 449                         this.identifiers = session.Identifiers;
 450                         this.id_builder = session.IDBuilder;
 451                         this.number_builder = session.NumberBuilder;
 452                         this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 453                         this.Report = report;
 454
 455                         reader = input;
 456
 457                         putback_char = -1;
 458
 459                         xml_comment_buffer = new StringBuilder ();
 460                         doc_processing = context.Settings.DocumentationFile != null;
 461
 462                         tab_size = context.Settings.TabSize;
 463                 }
 464
 465                 public void PushPosition ()
 466                 {
 467                         position_stack.Push (new Position (this));
 468                 }
 469
 470                 public void PopPosition ()
 471                 {
 472                         Position p = position_stack.Pop ();
 473
 474                         reader.Position = p.position;
 475                         ref_line = p.ref_line;
 476                         line = p.line;
 477                         col = p.col;
 478                         hidden_block_start = p.hidden;
 479                         putback_char = p.putback_char;
 480                         previous_col = p.previous_col;
 481                         ifstack = p.ifstack;
 482                         parsing_generic_less_than = p.parsing_generic_less_than;
 483                         parsing_string_interpolation = p.parsing_string_interpolation;
 484                         parsing_string_interpolation_quoted = p.parsing_string_interpolation_quoted;
 485                         current_token = p.current_token;
 486                         val = p.val;
 487                 }
 488
 489                 // Do not reset the position, ignore it.
 490                 public void DiscardPosition ()
 491                 {
 492                         position_stack.Pop ();
 493                 }
 494
 495                 static void AddKeyword (string kw, int token)
 496                 {
 497                         keyword_strings.Add (kw);
 498
 499                         AddKeyword (keywords, kw, token);
 500                 }
 501
 502                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 503                 {
 504                         AddKeyword (keywords_preprocessor, kw, directive);
 505                 }
 506
 507                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 508                 {
 509                         int length = kw.Length;
 510                         if (keywords[length] == null) {
 511                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 512                         }
 513
 514                         int char_index = kw[0] - '_';
 515                         var kwe = keywords[length][char_index];
 516                         if (kwe == null) {
 517                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 518                                 return;
 519                         }
 520
 521                         while (kwe.Next != null) {
 522                                 kwe = kwe.Next;
 523                         }
 524
 525                         kwe.Next = new KeywordEntry<T> (kw, token);
 526                 }
 527
 528                 //
 529                 // Class initializer
 530                 //
 531                 static Tokenizer ()
 532                 {
 533                         keyword_strings = new HashSet<string> ();
 534
 535                         // 11 is the length of the longest keyword for now
 536                         keywords = new KeywordEntry<int>[11][];
 537
 538                         AddKeyword ("__arglist", Token.ARGLIST);
 539                         AddKeyword ("__makeref", Token.MAKEREF);
 540                         AddKeyword ("__reftype", Token.REFTYPE);
 541                         AddKeyword ("__refvalue", Token.REFVALUE);
 542                         AddKeyword ("abstract", Token.ABSTRACT);
 543                         AddKeyword ("as", Token.AS);
 544                         AddKeyword ("add", Token.ADD);
 545                         AddKeyword ("base", Token.BASE);
 546                         AddKeyword ("bool", Token.BOOL);
 547                         AddKeyword ("break", Token.BREAK);
 548                         AddKeyword ("byte", Token.BYTE);
 549                         AddKeyword ("case", Token.CASE);
 550                         AddKeyword ("catch", Token.CATCH);
 551                         AddKeyword ("char", Token.CHAR);
 552                         AddKeyword ("checked", Token.CHECKED);
 553                         AddKeyword ("class", Token.CLASS);
 554                         AddKeyword ("const", Token.CONST);
 555                         AddKeyword ("continue", Token.CONTINUE);
 556                         AddKeyword ("decimal", Token.DECIMAL);
 557                         AddKeyword ("default", Token.DEFAULT);
 558                         AddKeyword ("delegate", Token.DELEGATE);
 559                         AddKeyword ("do", Token.DO);
 560                         AddKeyword ("double", Token.DOUBLE);
 561                         AddKeyword ("else", Token.ELSE);
 562                         AddKeyword ("enum", Token.ENUM);
 563                         AddKeyword ("event", Token.EVENT);
 564                         AddKeyword ("explicit", Token.EXPLICIT);
 565                         AddKeyword ("extern", Token.EXTERN);
 566                         AddKeyword ("false", Token.FALSE);
 567                         AddKeyword ("finally", Token.FINALLY);
 568                         AddKeyword ("fixed", Token.FIXED);
 569                         AddKeyword ("float", Token.FLOAT);
 570                         AddKeyword ("for", Token.FOR);
 571                         AddKeyword ("foreach", Token.FOREACH);
 572                         AddKeyword ("goto", Token.GOTO);
 573                         AddKeyword ("get", Token.GET);
 574                         AddKeyword ("if", Token.IF);
 575                         AddKeyword ("implicit", Token.IMPLICIT);
 576                         AddKeyword ("in", Token.IN);
 577                         AddKeyword ("int", Token.INT);
 578                         AddKeyword ("interface", Token.INTERFACE);
 579                         AddKeyword ("internal", Token.INTERNAL);
 580                         AddKeyword ("is", Token.IS);
 581                         AddKeyword ("lock", Token.LOCK);
 582                         AddKeyword ("long", Token.LONG);
 583                         AddKeyword ("namespace", Token.NAMESPACE);
 584                         AddKeyword ("new", Token.NEW);
 585                         AddKeyword ("null", Token.NULL);
 586                         AddKeyword ("object", Token.OBJECT);
 587                         AddKeyword ("operator", Token.OPERATOR);
 588                         AddKeyword ("out", Token.OUT);
 589                         AddKeyword ("override", Token.OVERRIDE);
 590                         AddKeyword ("params", Token.PARAMS);
 591                         AddKeyword ("private", Token.PRIVATE);
 592                         AddKeyword ("protected", Token.PROTECTED);
 593                         AddKeyword ("public", Token.PUBLIC);
 594                         AddKeyword ("readonly", Token.READONLY);
 595                         AddKeyword ("ref", Token.REF);
 596                         AddKeyword ("remove", Token.REMOVE);
 597                         AddKeyword ("return", Token.RETURN);
 598                         AddKeyword ("sbyte", Token.SBYTE);
 599                         AddKeyword ("sealed", Token.SEALED);
 600                         AddKeyword ("set", Token.SET);
 601                         AddKeyword ("short", Token.SHORT);
 602                         AddKeyword ("sizeof", Token.SIZEOF);
 603                         AddKeyword ("stackalloc", Token.STACKALLOC);
 604                         AddKeyword ("static", Token.STATIC);
 605                         AddKeyword ("string", Token.STRING);
 606                         AddKeyword ("struct", Token.STRUCT);
 607                         AddKeyword ("switch", Token.SWITCH);
 608                         AddKeyword ("this", Token.THIS);
 609                         AddKeyword ("throw", Token.THROW);
 610                         AddKeyword ("true", Token.TRUE);
 611                         AddKeyword ("try", Token.TRY);
 612                         AddKeyword ("typeof", Token.TYPEOF);
 613                         AddKeyword ("uint", Token.UINT);
 614                         AddKeyword ("ulong", Token.ULONG);
 615                         AddKeyword ("unchecked", Token.UNCHECKED);
 616                         AddKeyword ("unsafe", Token.UNSAFE);
 617                         AddKeyword ("ushort", Token.USHORT);
 618                         AddKeyword ("using", Token.USING);
 619                         AddKeyword ("virtual", Token.VIRTUAL);
 620                         AddKeyword ("void", Token.VOID);
 621                         AddKeyword ("volatile", Token.VOLATILE);
 622                         AddKeyword ("while", Token.WHILE);
 623                         AddKeyword ("partial", Token.PARTIAL);
 624                         AddKeyword ("where", Token.WHERE);
 625
 626                         // LINQ keywords
 627                         AddKeyword ("from", Token.FROM);
 628                         AddKeyword ("join", Token.JOIN);
 629                         AddKeyword ("on", Token.ON);
 630                         AddKeyword ("equals", Token.EQUALS);
 631                         AddKeyword ("select", Token.SELECT);
 632                         AddKeyword ("group", Token.GROUP);
 633                         AddKeyword ("by", Token.BY);
 634                         AddKeyword ("let", Token.LET);
 635                         AddKeyword ("orderby", Token.ORDERBY);
 636                         AddKeyword ("ascending", Token.ASCENDING);
 637                         AddKeyword ("descending", Token.DESCENDING);
 638                         AddKeyword ("into", Token.INTO);
 639
 640                         // Contextual async keywords
 641                         AddKeyword ("async", Token.ASYNC);
 642                         AddKeyword ("await", Token.AWAIT);
 643
 644                         // Contextual filter catch keyword
 645                         AddKeyword ("when", Token.WHEN);
 646
 647                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 648
 649                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 650                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 651                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 652                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 653                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 654                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 655                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 656                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 657                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 658                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 659                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 660                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 661
 662                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 663                         styles = NumberStyles.Float;
 664                 }
 665
 666                 int GetKeyword (char[] id, int id_len)
 667                 {
 668                         //
 669                         // Keywords are stored in an array of arrays grouped by their
 670                         // length and then by the first character
 671                         //
 672                         if (id_len >= keywords.Length || keywords [id_len] == null)
 673                                 return -1;
 674
 675                         int first_index = id [0] - '_';
 676                         if (first_index > 'z' - '_')
 677                                 return -1;
 678
 679                         var kwe = keywords [id_len] [first_index];
 680                         if (kwe == null)
 681                                 return -1;
 682
 683                         int res;
 684                         do {
 685                                 res = kwe.Token;
 686                                 for (int i = 1; i < id_len; ++i) {
 687                                         if (id [i] != kwe.Value [i]) {
 688                                                 res = 0;
 689                                                 kwe = kwe.Next;
 690                                                 break;
 691                                         }
 692                                 }
 693                         } while (res == 0 && kwe != null);
 694
 695                         if (res == 0)
 696                                 return -1;
 697
 698                         int next_token;
 699                         switch (res) {
 700                         case Token.GET:
 701                         case Token.SET:
 702                                 if (!handle_get_set)
 703                                         res = -1;
 704                                 break;
 705                         case Token.REMOVE:
 706                         case Token.ADD:
 707                                 if (!handle_remove_add)
 708                                         res = -1;
 709                                 break;
 710                         case Token.EXTERN:
 711                                 if (parsing_declaration == 0)
 712                                         res = Token.EXTERN_ALIAS;
 713                                 break;
 714                         case Token.DEFAULT:
 715                                 if (peek_token () == Token.COLON) {
 716                                         token ();
 717                                         res = Token.DEFAULT_COLON;
 718                                 }
 719                                 break;
 720                         case Token.WHEN:
 721                                 if (current_token != Token.CATCH && !parsing_catch_when)
 722                                         res = -1;
 723                                 break;
 724                         case Token.WHERE:
 725                                 if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 726                                         res = -1;
 727                                 break;
 728                         case Token.FROM:
 729                                 //
 730                                 // A query expression is any expression that starts with `from identifier'
 731                                 // followed by any token except ; , =
 732                                 //
 733                                 if (!query_parsing) {
 734                                         if (lambda_arguments_parsing || parsing_block == 0) {
 735                                                 res = -1;
 736                                                 break;
 737                                         }
 738
 739                                         PushPosition ();
 740                                         // HACK: to disable generics micro-parser, because PushPosition does not
 741                                         // store identifiers array
 742                                         parsing_generic_less_than = 1;
 743                                         switch (xtoken ()) {
 744                                         case Token.IDENTIFIER:
 745                                         case Token.INT:
 746                                         case Token.BOOL:
 747                                         case Token.BYTE:
 748                                         case Token.CHAR:
 749                                         case Token.DECIMAL:
 750                                         case Token.DOUBLE:
 751                                         case Token.FLOAT:
 752                                         case Token.LONG:
 753                                         case Token.OBJECT:
 754                                         case Token.STRING:
 755                                         case Token.UINT:
 756                                         case Token.ULONG:
 757                                                 next_token = xtoken ();
 758                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 759                                                         goto default;
 760
 761                                                 res = Token.FROM_FIRST;
 762                                                 query_parsing = true;
 763                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 764                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 765                                                 break;
 766                                         case Token.VOID:
 767                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 768                                                 break;
 769                                         default:
 770                                                 PopPosition ();
 771                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 772                                                 // which has been overwritten before we grabbed the identifier
 773                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 774                                                 return -1;
 775                                         }
 776                                         PopPosition ();
 777                                 }
 778                                 break;
 779                         case Token.JOIN:
 780                         case Token.ON:
 781                         case Token.EQUALS:
 782                         case Token.SELECT:
 783                         case Token.GROUP:
 784                         case Token.BY:
 785                         case Token.LET:
 786                         case Token.ORDERBY:
 787                         case Token.ASCENDING:
 788                         case Token.DESCENDING:
 789                         case Token.INTO:
 790                                 if (!query_parsing)
 791                                         res = -1;
 792                                 break;
 793
 794                         case Token.USING:
 795                         case Token.NAMESPACE:
 796                                 // TODO: some explanation needed
 797                                 check_incorrect_doc_comment ();
 798                                 parsing_modifiers = false;
 799                                 break;
 800
 801                         case Token.PARTIAL:
 802                                 if (parsing_block > 0) {
 803                                         res = -1;
 804                                         break;
 805                                 }
 806
 807                                 // Save current position and parse next token.
 808                                 PushPosition ();
 809
 810                                 next_token = token ();
 811                                 bool ok = (next_token == Token.CLASS) ||
 812                                         (next_token == Token.STRUCT) ||
 813                                         (next_token == Token.INTERFACE) ||
 814                                         (next_token == Token.VOID);
 815
 816                                 PopPosition ();
 817
 818                                 if (ok) {
 819                                         if (next_token == Token.VOID) {
 820                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 821                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 822                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 823                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 824
 825                                         return res;
 826                                 }
 827
 828                                 if (next_token < Token.LAST_KEYWORD) {
 829                                         Report.Error (267, Location,
 830                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 831                                         return token ();
 832                                 }
 833
 834                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 835                                 // which has been overwritten before we grabbed the identifier
 836                                 id_builder[0] = 'p';
 837                                 id_builder[1] = 'a';
 838                                 id_builder[2] = 'r';
 839                                 id_builder[3] = 't';
 840                                 id_builder[4] = 'i';
 841                                 id_builder[5] = 'a';
 842                                 id_builder[6] = 'l';
 843                                 res = -1;
 844                                 break;
 845
 846                         case Token.ASYNC:
 847                                 if (parsing_modifiers) {
 848                                         //
 849                                         // Skip attributes section or constructor called async
 850                                         //
 851                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 852                                                 res = -1;
 853                                         } else {
 854                                                 // async is keyword
 855                                         }
 856                                 } else if (parsing_block > 0) {
 857                                         switch (peek_token ()) {
 858                                         case Token.DELEGATE:
 859                                         case Token.OPEN_PARENS_LAMBDA:
 860                                                 // async is keyword
 861                                                 break;
 862                                         case Token.IDENTIFIER:
 863                                                 PushPosition ();
 864                                                 xtoken ();
 865                                                 if (xtoken () != Token.ARROW) {
 866                                                         PopPosition ();
 867                                                         goto default;
 868                                                 }
 869
 870                                                 PopPosition ();
 871                                                 break;
 872                                         default:
 873                                                 // peek_token could overwrite id_buffer
 874                                                 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 875                                                 res = -1;
 876                                                 break;
 877                                         }
 878                                 } else {
 879                                         res = -1;
 880                                 }
 881
 882                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 883                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 884                                 }
 885
 886                                 break;
 887
 888                         case Token.AWAIT:
 889                                 if (parsing_block == 0)
 890                                         res = -1;
 891
 892                                 break;
 893                         }
 894
 895
 896                         return res;
 897                 }
 898
 899                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 900                 {
 901                         //
 902                         // Keywords are stored in an array of arrays grouped by their
 903                         // length and then by the first character
 904                         //
 905                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 906                                 return PreprocessorDirective.Invalid;
 907
 908                         int first_index = id[0] - '_';
 909                         if (first_index > 'z' - '_')
 910                                 return PreprocessorDirective.Invalid;
 911
 912                         var kwe = keywords_preprocessor[id_len][first_index];
 913                         if (kwe == null)
 914                                 return PreprocessorDirective.Invalid;
 915
 916                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 917                         do {
 918                                 res = kwe.Token;
 919                                 for (int i = 1; i < id_len; ++i) {
 920                                         if (id[i] != kwe.Value[i]) {
 921                                                 res = 0;
 922                                                 kwe = kwe.Next;
 923                                                 break;
 924                                         }
 925                                 }
 926                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 927
 928                         return res;
 929                 }
 930
 931                 public Location Location {
 932                         get {
 933                                 return new Location (current_source, ref_line, col);
 934                         }
 935                 }
 936
 937                 static bool is_identifier_start_character (int c)
 938                 {
 939                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')
 940                                 return true;
 941
 942                         if (c < 0x80)
 943                                 return false;
 944
 945                         return is_identifier_start_character_slow_part ((char) c);
 946                 }
 947
 948                 static bool is_identifier_part_character (char c)
 949                 {
 950                         if (c >= 'a' && c <= 'z')
 951                                 return true;
 952
 953                         if (c >= 'A' && c <= 'Z')
 954                                 return true;
 955
 956                         if (c == '_' || (c >= '0' && c <= '9'))
 957                                 return true;
 958
 959                         if (c < 0x80)
 960                                 return false;
 961
 962                         return is_identifier_part_character_slow_part (c);
 963                 }
 964
 965                 static bool is_identifier_start_character_slow_part (char c)
 966                 {
 967                         switch (Char.GetUnicodeCategory (c)) {
 968                         case UnicodeCategory.LetterNumber:
 969                         case UnicodeCategory.UppercaseLetter:
 970                         case UnicodeCategory.LowercaseLetter:
 971                         case UnicodeCategory.TitlecaseLetter:
 972                         case UnicodeCategory.ModifierLetter:
 973                         case UnicodeCategory.OtherLetter:
 974                                 return true;
 975                         }
 976                         return false;
 977                 }
 978
 979                 static bool is_identifier_part_character_slow_part (char c)
 980                 {
 981                         switch (Char.GetUnicodeCategory (c)) {
 982                         // connecting-character:  A Unicode character of the class Pc
 983                         case UnicodeCategory.ConnectorPunctuation:
 984
 985                         // combining-character: A Unicode character of classes Mn or Mc
 986                         case UnicodeCategory.NonSpacingMark:
 987                         case UnicodeCategory.SpacingCombiningMark:
 988
 989                         // decimal-digit-character: A Unicode character of the class Nd
 990                         case UnicodeCategory.DecimalDigitNumber:
 991
 992                         // plus is_identifier_start_character_slow_part
 993                         case UnicodeCategory.LetterNumber:
 994                         case UnicodeCategory.UppercaseLetter:
 995                         case UnicodeCategory.LowercaseLetter:
 996                         case UnicodeCategory.TitlecaseLetter:
 997                         case UnicodeCategory.ModifierLetter:
 998                         case UnicodeCategory.OtherLetter:
 999                                 return true;
1000
1001                         // formatting-character: A Unicode character of the class Cf
1002                         case UnicodeCategory.Format:
1003                                 // csc bug compatibility which recognizes it as a whitespace
1004                                 return c != 0xFEFF;
1005                         }
1006
1007                         return false;
1008                 }
1009
1010                 public static bool IsKeyword (string s)
1011                 {
1012                         return keyword_strings.Contains (s);
1013                 }
1014
1015                 //
1016                 // Open parens micro parser. Detects both lambda and cast ambiguity.
1017                 //
1018                 int TokenizeOpenParens ()
1019                 {
1020                         int ptoken;
1021                         current_token = -1;
1022
1023                         int bracket_level = 0;
1024                         bool is_type = false;
1025                         bool can_be_type = false;
1026
1027                         while (true) {
1028                                 ptoken = current_token;
1029                                 token ();
1030
1031                                 switch (current_token) {
1032                                 case Token.CLOSE_PARENS:
1033                                         token ();
1034
1035                                         //
1036                                         // Expression inside parens is lambda, (int i) =>
1037                                         //
1038                                         if (current_token == Token.ARROW)
1039                                                 return Token.OPEN_PARENS_LAMBDA;
1040
1041                                         //
1042                                         // Expression inside parens is single type, (int[])
1043                                         //
1044                                         if (is_type) {
1045                                                 if (current_token == Token.SEMICOLON)
1046                                                         return Token.OPEN_PARENS;
1047
1048                                                 return Token.OPEN_PARENS_CAST;
1049                                         }
1050
1051                                         //
1052                                         // Expression is possible cast, look at next token, (T)null
1053                                         //
1054                                         if (can_be_type) {
1055                                                 switch (current_token) {
1056                                                 case Token.OPEN_PARENS:
1057                                                 case Token.BANG:
1058                                                 case Token.TILDE:
1059                                                 case Token.IDENTIFIER:
1060                                                 case Token.LITERAL:
1061                                                 case Token.BASE:
1062                                                 case Token.CHECKED:
1063                                                 case Token.DELEGATE:
1064                                                 case Token.FALSE:
1065                                                 case Token.FIXED:
1066                                                 case Token.NEW:
1067                                                 case Token.NULL:
1068                                                 case Token.SIZEOF:
1069                                                 case Token.THIS:
1070                                                 case Token.THROW:
1071                                                 case Token.TRUE:
1072                                                 case Token.TYPEOF:
1073                                                 case Token.UNCHECKED:
1074                                                 case Token.UNSAFE:
1075                                                 case Token.DEFAULT:
1076                                                 case Token.AWAIT:
1077
1078                                                 //
1079                                                 // These can be part of a member access
1080                                                 //
1081                                                 case Token.INT:
1082                                                 case Token.UINT:
1083                                                 case Token.SHORT:
1084                                                 case Token.USHORT:
1085                                                 case Token.LONG:
1086                                                 case Token.ULONG:
1087                                                 case Token.DOUBLE:
1088                                                 case Token.FLOAT:
1089                                                 case Token.CHAR:
1090                                                 case Token.BYTE:
1091                                                 case Token.DECIMAL:
1092                                                 case Token.BOOL:
1093                                                 case Token.STRING:
1094                                                         return Token.OPEN_PARENS_CAST;
1095                                                 }
1096                                         }
1097                                         return Token.OPEN_PARENS;
1098
1099                                 case Token.DOT:
1100                                 case Token.DOUBLE_COLON:
1101                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1102                                                 goto default;
1103
1104                                         continue;
1105
1106                                 case Token.IDENTIFIER:
1107                                 case Token.AWAIT:
1108                                         switch (ptoken) {
1109                                         case Token.DOT:
1110                                                 if (bracket_level == 0) {
1111                                                         is_type = false;
1112                                                         can_be_type = true;
1113                                                 }
1114
1115                                                 continue;
1116                                         case Token.OP_GENERICS_LT:
1117                                         case Token.COMMA:
1118                                         case Token.DOUBLE_COLON:
1119                                         case -1:
1120                                                 if (bracket_level == 0)
1121                                                         can_be_type = true;
1122                                                 continue;
1123                                         default:
1124                                                 can_be_type = is_type = false;
1125                                                 continue;
1126                                         }
1127
1128                                 case Token.OBJECT:
1129                                 case Token.STRING:
1130                                 case Token.BOOL:
1131                                 case Token.DECIMAL:
1132                                 case Token.FLOAT:
1133                                 case Token.DOUBLE:
1134                                 case Token.SBYTE:
1135                                 case Token.BYTE:
1136                                 case Token.SHORT:
1137                                 case Token.USHORT:
1138                                 case Token.INT:
1139                                 case Token.UINT:
1140                                 case Token.LONG:
1141                                 case Token.ULONG:
1142                                 case Token.CHAR:
1143                                 case Token.VOID:
1144                                         if (bracket_level == 0)
1145                                                 is_type = true;
1146                                         continue;
1147
1148                                 case Token.COMMA:
1149                                         if (bracket_level == 0) {
1150                                                 bracket_level = 100;
1151                                                 can_be_type = is_type = false;
1152                                         }
1153                                         continue;
1154
1155                                 case Token.OP_GENERICS_LT:
1156                                 case Token.OPEN_BRACKET:
1157                                         if (bracket_level++ == 0)
1158                                                 is_type = true;
1159                                         continue;
1160
1161                                 case Token.OP_GENERICS_GT:
1162                                 case Token.CLOSE_BRACKET:
1163                                         --bracket_level;
1164                                         continue;
1165
1166                                 case Token.INTERR_NULLABLE:
1167                                 case Token.STAR:
1168                                         if (bracket_level == 0)
1169                                                 is_type = true;
1170                                         continue;
1171
1172                                 case Token.REF:
1173                                 case Token.OUT:
1174                                         can_be_type = is_type = false;
1175                                         continue;
1176
1177                                 default:
1178                                         return Token.OPEN_PARENS;
1179                                 }
1180                         }
1181                 }
1182
1183                 public static bool IsValidIdentifier (string s)
1184                 {
1185                         if (s == null || s.Length == 0)
1186                                 return false;
1187
1188                         if (!is_identifier_start_character (s [0]))
1189                                 return false;
1190
1191                         for (int i = 1; i < s.Length; i ++)
1192                                 if (! is_identifier_part_character (s [i]))
1193                                         return false;
1194
1195                         return true;
1196                 }
1197
1198                 bool parse_less_than (ref int genericDimension)
1199                 {
1200                 start:
1201                         int the_token = token ();
1202                         if (the_token == Token.OPEN_BRACKET) {
1203                                 while (true) {
1204                                         the_token = token ();
1205                                         if (the_token == Token.EOF)
1206                                                 return true;
1207
1208                                         if (the_token == Token.CLOSE_BRACKET)
1209                                                 break;
1210                                 }
1211                                 the_token = token ();
1212                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1213                                 the_token = token ();
1214                         }
1215                         switch (the_token) {
1216                         case Token.IDENTIFIER:
1217                         case Token.OBJECT:
1218                         case Token.STRING:
1219                         case Token.BOOL:
1220                         case Token.DECIMAL:
1221                         case Token.FLOAT:
1222                         case Token.DOUBLE:
1223                         case Token.SBYTE:
1224                         case Token.BYTE:
1225                         case Token.SHORT:
1226                         case Token.USHORT:
1227                         case Token.INT:
1228                         case Token.UINT:
1229                         case Token.LONG:
1230                         case Token.ULONG:
1231                         case Token.CHAR:
1232                         case Token.VOID:
1233                                 break;
1234                         case Token.OP_GENERICS_GT:
1235                                 genericDimension = 1;
1236                                 return true;
1237                         case Token.IN:
1238                         case Token.OUT:
1239                                 return true;
1240                         case Token.COMMA:
1241                                 do {
1242                                         ++genericDimension;
1243                                         the_token = token ();
1244                                 } while (the_token == Token.COMMA);
1245
1246                                 if (the_token == Token.OP_GENERICS_GT) {
1247                                         ++genericDimension;
1248                                         return true;
1249                                 }
1250
1251                                 return false;
1252                         default:
1253                                 return false;
1254                         }
1255                 again:
1256                         the_token = token ();
1257
1258                         if (the_token == Token.OP_GENERICS_GT)
1259                                 return true;
1260                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1261                                 goto start;
1262                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1263                                 goto again;
1264                         else if (the_token == Token.OP_GENERICS_LT) {
1265                                 if (!parse_less_than (ref genericDimension))
1266                                         return false;
1267                                 goto again;
1268                         } else if (the_token == Token.OPEN_BRACKET) {
1269                         rank_specifiers:
1270                                 the_token = token ();
1271                                 if (the_token == Token.CLOSE_BRACKET)
1272                                         goto again;
1273                                 else if (the_token == Token.COMMA)
1274                                         goto rank_specifiers;
1275                                 return false;
1276                         }
1277
1278                         return false;
1279                 }
1280
1281                 public int peek_token ()
1282                 {
1283                         int the_token;
1284
1285                         PushPosition ();
1286                         the_token = token ();
1287                         PopPosition ();
1288
1289                         return the_token;
1290                 }
1291
1292                 //
1293                 // Tonizes `?' using custom disambiguous rules to return one
1294                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1295                 //
1296                 // Tricky expression looks like:
1297                 //
1298                 // Foo ? a = x ? b : c;
1299                 //
1300                 int TokenizePossibleNullableType ()
1301                 {
1302                         if (parsing_block == 0 || parsing_type > 0)
1303                                 return Token.INTERR_NULLABLE;
1304
1305                         int d = peek_char ();
1306                         if (d == '?') {
1307                                 get_char ();
1308                                 return Token.OP_COALESCING;
1309                         }
1310
1311                         if (d == '.') {
1312                                 return Token.INTERR_OPERATOR;
1313                         }
1314
1315                         if (d != ' ') {
1316                                 if (d == ',' || d == ';' || d == '>')
1317                                         return Token.INTERR_NULLABLE;
1318                                 if (d == '*' || (d >= '0' && d <= '9'))
1319                                         return Token.INTERR;
1320                         }
1321
1322                         PushPosition ();
1323                         current_token = Token.NONE;
1324                         int next_token;
1325                         int parens = 0;
1326                         int generics = 0;
1327                         int brackets = 0;
1328
1329                         var nt = xtoken ();
1330                         switch (nt) {
1331                         case Token.DOT:
1332                         case Token.OPEN_BRACKET_EXPR:
1333                                 next_token = Token.INTERR_OPERATOR;
1334                                 break;
1335                         case Token.LITERAL:
1336                         case Token.TRUE:
1337                         case Token.FALSE:
1338                         case Token.NULL:
1339                         case Token.THIS:
1340                         case Token.NEW:
1341                         case Token.INTERPOLATED_STRING:
1342                                 next_token = Token.INTERR;
1343                                 break;
1344
1345                         case Token.SEMICOLON:
1346                         case Token.COMMA:
1347                         case Token.CLOSE_PARENS:
1348                         case Token.OPEN_BRACKET:
1349                         case Token.OP_GENERICS_GT:
1350                         case Token.INTERR:
1351                         case Token.OP_COALESCING:
1352                         case Token.COLON:
1353                                 next_token = Token.INTERR_NULLABLE;
1354                                 break;
1355
1356                         case Token.OPEN_PARENS:
1357                         case Token.OPEN_PARENS_CAST:
1358                         case Token.OPEN_PARENS_LAMBDA:
1359                                 next_token = -1;
1360                                 ++parens;
1361                                 break;
1362
1363                         case Token.OP_GENERICS_LT:
1364                         case Token.OP_GENERICS_LT_DECL:
1365                         case Token.GENERIC_DIMENSION:
1366                                 next_token = -1;
1367                                 ++generics;
1368                                 break;
1369
1370                         default:
1371                                 next_token = -1;
1372                                 break;
1373                         }
1374
1375                         if (next_token == -1) {
1376                                 switch (xtoken ()) {
1377                                 case Token.COMMA:
1378                                 case Token.SEMICOLON:
1379                                 case Token.OPEN_BRACE:
1380                                 case Token.IN:
1381                                         next_token = Token.INTERR_NULLABLE;
1382                                         break;
1383
1384                                 case Token.COLON:
1385                                         next_token = Token.INTERR;
1386                                         break;
1387
1388                                 case Token.OPEN_PARENS:
1389                                 case Token.OPEN_PARENS_CAST:
1390                                 case Token.OPEN_PARENS_LAMBDA:
1391                                         ++parens;
1392                                         goto default;
1393
1394                                 case Token.OPEN_BRACKET:
1395                                 case Token.OPEN_BRACKET_EXPR:
1396                                         ++brackets;
1397                                         goto default;
1398
1399                                 case Token.CLOSE_PARENS:
1400                                         --parens;
1401                                         goto default;
1402
1403                                 case Token.OP_GENERICS_LT:
1404                                 case Token.OP_GENERICS_LT_DECL:
1405                                 case Token.GENERIC_DIMENSION:
1406                                         ++generics;
1407                                         goto default;
1408
1409                                 default:
1410                                         int ntoken;
1411                                         int interrs = 1;
1412                                         int colons = 0;
1413                                         int braces = 0;
1414                                         //
1415                                         // All shorcuts failed, do it hard way
1416                                         //
1417                                         while ((ntoken = xtoken ()) != Token.EOF) {
1418                                                 switch (ntoken) {
1419                                                 case Token.OPEN_BRACE:
1420                                                         ++braces;
1421                                                         continue;
1422                                                 case Token.OPEN_PARENS:
1423                                                 case Token.OPEN_PARENS_CAST:
1424                                                 case Token.OPEN_PARENS_LAMBDA:
1425                                                         ++parens;
1426                                                         continue;
1427                                                 case Token.CLOSE_BRACE:
1428                                                         --braces;
1429                                                         continue;
1430                                                 case Token.OP_GENERICS_LT:
1431                                                 case Token.OP_GENERICS_LT_DECL:
1432                                                 case Token.GENERIC_DIMENSION:
1433                                                         ++generics;
1434                                                         continue;
1435                                                 case Token.OPEN_BRACKET:
1436                                                 case Token.OPEN_BRACKET_EXPR:
1437                                                         ++brackets;
1438                                                         continue;
1439                                                 case Token.CLOSE_BRACKET:
1440                                                         --brackets;
1441                                                         continue;
1442                                                 case Token.CLOSE_PARENS:
1443                                                         if (parens > 0) {
1444                                                                 --parens;
1445                                                                 continue;
1446                                                         }
1447
1448                                                         PopPosition ();
1449                                                         return Token.INTERR_NULLABLE;
1450
1451                                                 case Token.OP_GENERICS_GT:
1452                                                         if (generics > 0) {
1453                                                                 --generics;
1454                                                                 continue;
1455                                                         }
1456
1457                                                         PopPosition ();
1458                                                         return Token.INTERR_NULLABLE;
1459                                                 }
1460
1461                                                 if (braces != 0)
1462                                                         continue;
1463
1464                                                 if (ntoken == Token.SEMICOLON)
1465                                                         break;
1466
1467                                                 if (parens != 0)
1468                                                         continue;
1469
1470                                                 if (ntoken == Token.COMMA) {
1471                                                         if (generics != 0 || brackets != 0)
1472                                                                 continue;
1473
1474                                                         PopPosition ();
1475                                                         return Token.INTERR_NULLABLE;
1476                                                 }
1477
1478                                                 if (ntoken == Token.COLON) {
1479                                                         if (++colons == interrs)
1480                                                                 break;
1481                                                         continue;
1482                                                 }
1483
1484                                                 if (ntoken == Token.INTERR) {
1485                                                         ++interrs;
1486                                                         continue;
1487                                                 }
1488                                         }
1489
1490                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1491                                         break;
1492                                 }
1493                         }
1494
1495                         PopPosition ();
1496                         return next_token;
1497                 }
1498
1499                 bool decimal_digits (int c)
1500                 {
1501                         int d;
1502                         bool seen_digits = false;
1503
1504                         if (c != -1){
1505                                 if (number_pos == MaxNumberLength)
1506                                         Error_NumericConstantTooLong ();
1507                                 number_builder [number_pos++] = (char) c;
1508                         }
1509
1510                         //
1511                         // We use peek_char2, because decimal_digits needs to do a
1512                         // 2-character look-ahead (5.ToString for example).
1513                         //
1514                         while ((d = peek_char2 ()) != -1){
1515                                 if (d >= '0' && d <= '9'){
1516                                         if (number_pos == MaxNumberLength)
1517                                                 Error_NumericConstantTooLong ();
1518                                         number_builder [number_pos++] = (char) d;
1519                                         get_char ();
1520                                         seen_digits = true;
1521                                 } else
1522                                         break;
1523                         }
1524
1525                         return seen_digits;
1526                 }
1527
1528                 static bool is_hex (int e)
1529                 {
1530                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1531                 }
1532
1533                 static TypeCode real_type_suffix (int c)
1534                 {
1535                         switch (c){
1536                         case 'F': case 'f':
1537                                 return TypeCode.Single;
1538                         case 'D': case 'd':
1539                                 return TypeCode.Double;
1540                         case 'M': case 'm':
1541                                 return TypeCode.Decimal;
1542                         default:
1543                                 return TypeCode.Empty;
1544                         }
1545                 }
1546
1547                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1548                 {
1549                         bool is_unsigned = false;
1550                         bool is_long = false;
1551
1552                         if (c != -1){
1553                                 bool scanning = true;
1554                                 do {
1555                                         switch (c){
1556                                         case 'U': case 'u':
1557                                                 if (is_unsigned)
1558                                                         scanning = false;
1559                                                 is_unsigned = true;
1560                                                 get_char ();
1561                                                 break;
1562
1563                                         case 'l':
1564                                                 if (!is_unsigned){
1565                                                         //
1566                                                         // if we have not seen anything in between
1567                                                         // report this error
1568                                                         //
1569                                                         Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1570                                                 }
1571
1572                                                 goto case 'L';
1573
1574                                         case 'L':
1575                                                 if (is_long)
1576                                                         scanning = false;
1577                                                 is_long = true;
1578                                                 get_char ();
1579                                                 break;
1580
1581                                         default:
1582                                                 scanning = false;
1583                                                 break;
1584                                         }
1585                                         c = peek_char ();
1586                                 } while (scanning);
1587                         }
1588
1589                         if (is_long && is_unsigned){
1590                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1591                         }
1592
1593                         if (is_unsigned){
1594                                 // uint if possible, or ulong else.
1595
1596                                 if ((ul & 0xffffffff00000000) == 0)
1597                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1598                                 else
1599                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1600                         } else if (is_long){
1601                                 // long if possible, ulong otherwise
1602                                 if ((ul & 0x8000000000000000) != 0)
1603                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1604                                 else
1605                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1606                         } else {
1607                                 // int, uint, long or ulong in that order
1608                                 if ((ul & 0xffffffff00000000) == 0){
1609                                         uint ui = (uint) ul;
1610
1611                                         if ((ui & 0x80000000) != 0)
1612                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1613                                         else
1614                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1615                                 } else {
1616                                         if ((ul & 0x8000000000000000) != 0)
1617                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1618                                         else
1619                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1620                                 }
1621                         }
1622                 }
1623
1624                 //
1625                 // given `c' as the next char in the input decide whether
1626                 // we need to convert to a special type, and then choose
1627                 // the best representation for the integer
1628                 //
1629                 ILiteralConstant adjust_int (int c, Location loc)
1630                 {
1631                         try {
1632                                 if (number_pos > 9){
1633                                         ulong ul = (uint) (number_builder [0] - '0');
1634
1635                                         for (int i = 1; i < number_pos; i++){
1636                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1637                                         }
1638
1639                                         return integer_type_suffix (ul, c, loc);
1640                                 } else {
1641                                         uint ui = (uint) (number_builder [0] - '0');
1642
1643                                         for (int i = 1; i < number_pos; i++){
1644                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1645                                         }
1646
1647                                         return integer_type_suffix (ui, c, loc);
1648                                 }
1649                         } catch (OverflowException) {
1650                                 Error_NumericConstantTooLong ();
1651                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1652                         }
1653                         catch (FormatException) {
1654                                 Report.Error (1013, Location, "Invalid number");
1655                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1656                         }
1657                 }
1658
1659                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1660                 {
1661                         string s = new string (number_builder, 0, number_pos);
1662                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1663
1664                         switch (t){
1665                         case TypeCode.Decimal:
1666                                 try {
1667                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1668                                 } catch (OverflowException) {
1669                                         Report.Error (594, Location, error_details, "decimal");
1670                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1671                                 }
1672                         case TypeCode.Single:
1673                                 try {
1674                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1675                                 } catch (OverflowException) {
1676                                         Report.Error (594, Location, error_details, "float");
1677                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1678                                 }
1679                         default:
1680                                 try {
1681                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1682                                 } catch (OverflowException) {
1683                                         Report.Error (594, loc, error_details, "double");
1684                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1685                                 }
1686                         }
1687                 }
1688
1689                 ILiteralConstant handle_hex (Location loc)
1690                 {
1691                         int d;
1692                         ulong ul;
1693
1694                         get_char ();
1695                         while ((d = peek_char ()) != -1){
1696                                 if (is_hex (d)){
1697                                         number_builder [number_pos++] = (char) d;
1698                                         get_char ();
1699                                 } else
1700                                         break;
1701                         }
1702
1703                         string s = new String (number_builder, 0, number_pos);
1704
1705                         try {
1706                                 if (number_pos <= 8)
1707                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1708                                 else
1709                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1710
1711                                 return integer_type_suffix (ul, peek_char (), loc);
1712                         } catch (OverflowException){
1713                                 Error_NumericConstantTooLong ();
1714                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1715                         }
1716                         catch (FormatException) {
1717                                 Report.Error (1013, Location, "Invalid number");
1718                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1719                         }
1720                 }
1721
1722                 //
1723                 // Invoked if we know we have .digits or digits
1724                 //
1725                 int is_number (int c, bool dotLead)
1726                 {
1727                         ILiteralConstant res;
1728
1729 #if FULL_AST
1730                         int read_start = reader.Position - 1;
1731                         if (dotLead) {
1732                                 //
1733                                 // Caller did peek_char
1734                                 //
1735                                 --read_start;
1736                         }
1737 #endif
1738                         number_pos = 0;
1739                         var loc = Location;
1740
1741                         if (!dotLead){
1742                                 if (c == '0'){
1743                                         int peek = peek_char ();
1744
1745                                         if (peek == 'x' || peek == 'X') {
1746                                                 val = res = handle_hex (loc);
1747 #if FULL_AST
1748                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1749 #endif
1750
1751                                                 return Token.LITERAL;
1752                                         }
1753                                 }
1754                                 decimal_digits (c);
1755                                 c = peek_char ();
1756                         }
1757
1758                         //
1759                         // We need to handle the case of
1760                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1761                         //
1762                         bool is_real = false;
1763                         if (c == '.'){
1764                                 if (!dotLead)
1765                                         get_char ();
1766
1767                                 if (decimal_digits ('.')){
1768                                         is_real = true;
1769                                         c = peek_char ();
1770                                 } else {
1771                                         putback ('.');
1772                                         number_pos--;
1773                                         val = res = adjust_int (-1, loc);
1774
1775 #if FULL_AST
1776                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1777 #endif
1778                                         return Token.LITERAL;
1779                                 }
1780                         }
1781
1782                         if (c == 'e' || c == 'E'){
1783                                 is_real = true;
1784                                 get_char ();
1785                                 if (number_pos == MaxNumberLength)
1786                                         Error_NumericConstantTooLong ();
1787                                 number_builder [number_pos++] = (char) c;
1788                                 c = get_char ();
1789
1790                                 if (c == '+'){
1791                                         if (number_pos == MaxNumberLength)
1792                                                 Error_NumericConstantTooLong ();
1793                                         number_builder [number_pos++] = '+';
1794                                         c = -1;
1795                                 } else if (c == '-') {
1796                                         if (number_pos == MaxNumberLength)
1797                                                 Error_NumericConstantTooLong ();
1798                                         number_builder [number_pos++] = '-';
1799                                         c = -1;
1800                                 } else {
1801                                         if (number_pos == MaxNumberLength)
1802                                                 Error_NumericConstantTooLong ();
1803                                         number_builder [number_pos++] = '+';
1804                                 }
1805
1806                                 decimal_digits (c);
1807                                 c = peek_char ();
1808                         }
1809
1810                         var type = real_type_suffix (c);
1811                         if (type == TypeCode.Empty && !is_real) {
1812                                 res = adjust_int (c, loc);
1813                         } else {
1814                                 is_real = true;
1815
1816                                 if (type != TypeCode.Empty) {
1817                                         get_char ();
1818                                 }
1819
1820                                 res = adjust_real (type, loc);
1821                         }
1822
1823                         val = res;
1824
1825 #if FULL_AST
1826                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1827                         if (chars[chars.Length - 1] == '\r')
1828                                 Array.Resize (ref chars, chars.Length - 1);
1829                         res.ParsedValue = chars;
1830 #endif
1831
1832                         return Token.LITERAL;
1833                 }
1834
1835                 //
1836                 // Accepts exactly count (4 or 8) hex, no more no less
1837                 //
1838                 int getHex (int count, out int surrogate, out bool error)
1839                 {
1840                         int i;
1841                         int total = 0;
1842                         int c;
1843                         int top = count != -1 ? count : 4;
1844
1845                         get_char ();
1846                         error = false;
1847                         surrogate = 0;
1848                         for (i = 0; i < top; i++){
1849                                 c = get_char ();
1850
1851                                 if (c >= '0' && c <= '9')
1852                                         c = (int) c - (int) '0';
1853                                 else if (c >= 'A' && c <= 'F')
1854                                         c = (int) c - (int) 'A' + 10;
1855                                 else if (c >= 'a' && c <= 'f')
1856                                         c = (int) c - (int) 'a' + 10;
1857                                 else {
1858                                         error = true;
1859                                         return 0;
1860                                 }
1861
1862                                 total = (total * 16) + c;
1863                                 if (count == -1){
1864                                         int p = peek_char ();
1865                                         if (p == -1)
1866                                                 break;
1867                                         if (!is_hex ((char)p))
1868                                                 break;
1869                                 }
1870                         }
1871
1872                         if (top == 8) {
1873                                 if (total > 0x0010FFFF) {
1874                                         error = true;
1875                                         return 0;
1876                                 }
1877
1878                                 if (total >= 0x00010000) {
1879                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1880                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1881                                 }
1882                         }
1883
1884                         return total;
1885                 }
1886
1887                 int escape (int c, out int surrogate)
1888                 {
1889                         bool error;
1890                         int d;
1891                         int v;
1892
1893                         d = peek_char ();
1894                         if (c != '\\') {
1895                                 surrogate = 0;
1896                                 return c;
1897                         }
1898
1899                         switch (d){
1900                         case 'a':
1901                                 v = '\a'; break;
1902                         case 'b':
1903                                 v = '\b'; break;
1904                         case 'n':
1905                                 v = '\n'; break;
1906                         case 't':
1907                                 v = '\t'; break;
1908                         case 'v':
1909                                 v = '\v'; break;
1910                         case 'r':
1911                                 v = '\r'; break;
1912                         case '\\':
1913                                 v = '\\'; break;
1914                         case 'f':
1915                                 v = '\f'; break;
1916                         case '0':
1917                                 v = 0; break;
1918                         case '"':
1919                                 v = '"'; break;
1920                         case '\'':
1921                                 v = '\''; break;
1922                         case 'x':
1923                                 v = getHex (-1, out surrogate, out error);
1924                                 if (error)
1925                                         goto default;
1926                                 return v;
1927                         case 'u':
1928                         case 'U':
1929                                 return EscapeUnicode (d, out surrogate);
1930                         default:
1931                                 surrogate = 0;
1932                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1933                                 return d;
1934                         }
1935
1936                         get_char ();
1937                         surrogate = 0;
1938                         return v;
1939                 }
1940
1941                 int EscapeUnicode (int ch, out int surrogate)
1942                 {
1943                         bool error;
1944                         if (ch == 'U') {
1945                                 ch = getHex (8, out surrogate, out error);
1946                         } else {
1947                                 ch = getHex (4, out surrogate, out error);
1948                         }
1949
1950                         if (error)
1951                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1952
1953                         return ch;
1954                 }
1955
1956                 int get_char ()
1957                 {
1958                         int x;
1959                         if (putback_char != -1) {
1960                                 x = putback_char;
1961                                 putback_char = -1;
1962                         } else {
1963                                 x = reader.Read ();
1964                         }
1965
1966                         if (x <= 13) {
1967                                 if (x == '\r') {
1968                                         if (peek_char () == '\n') {
1969                                                 putback_char = -1;
1970                                         }
1971
1972                                         x = '\n';
1973                                         advance_line ();
1974                                 } else if (x == '\n') {
1975                                         advance_line ();
1976                                 } else {
1977                                         col++;
1978                                 }
1979                         } else if (x >= UnicodeLS && x <= UnicodePS) {
1980                                 advance_line ();
1981                         } else {
1982                                 col++;
1983                         }
1984
1985                         return x;
1986                 }
1987
1988                 void advance_line ()
1989                 {
1990                         line++;
1991                         ref_line++;
1992                         previous_col = col;
1993                         col = 0;
1994                 }
1995
1996                 int peek_char ()
1997                 {
1998                         if (putback_char == -1)
1999                                 putback_char = reader.Read ();
2000                         return putback_char;
2001                 }
2002
2003                 int peek_char2 ()
2004                 {
2005                         if (putback_char != -1)
2006                                 return putback_char;
2007                         return reader.Peek ();
2008                 }
2009
2010                 public void putback (int c)
2011                 {
2012                         if (putback_char != -1) {
2013                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
2014                         }
2015
2016                         if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
2017                                 // It won't happen though.
2018                                 line--;
2019                                 ref_line--;
2020                                 col = previous_col;
2021                         }
2022                         else
2023                                 col--;
2024                         putback_char = c;
2025                 }
2026
2027                 public bool advance ()
2028                 {
2029                         return peek_char () != -1 || CompleteOnEOF;
2030                 }
2031
2032                 public Object Value {
2033                         get {
2034                                 return val;
2035                         }
2036                 }
2037
2038                 public Object value ()
2039                 {
2040                         return val;
2041                 }
2042
2043                 public int token ()
2044                 {
2045                         current_token = xtoken ();
2046                         return current_token;
2047                 }
2048
2049                 int TokenizePreprocessorKeyword (out int c)
2050                 {
2051                         // skip over white space
2052                         do {
2053                                 c = get_char ();
2054                         } while (c == ' ' || c == '\t');
2055
2056
2057                         int pos = 0;
2058                         while (c != -1 && c >= 'a' && c <= 'z') {
2059                                 id_builder[pos++] = (char) c;
2060                                 c = get_char ();
2061                                 if (c == '\\') {
2062                                         int peek = peek_char ();
2063                                         if (peek == 'U' || peek == 'u') {
2064                                                 int surrogate;
2065                                                 c = EscapeUnicode (c, out surrogate);
2066                                                 if (surrogate != 0) {
2067                                                         if (is_identifier_part_character ((char) c)) {
2068                                                                 id_builder[pos++] = (char) c;
2069                                                         }
2070                                                         c = surrogate;
2071                                                 }
2072                                         }
2073                                 }
2074                         }
2075
2076                         return pos;
2077                 }
2078
2079                 PreprocessorDirective get_cmd_arg (out string arg)
2080                 {
2081                         int c;
2082
2083                         tokens_seen = false;
2084                         arg = "";
2085
2086                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorKeyword (out c));
2087
2088                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
2089                                 return cmd;
2090
2091                         // skip over white space
2092                         while (c == ' ' || c == '\t')
2093                                 c = get_char ();
2094
2095                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
2096                         int pos = 0;
2097
2098                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2099                                 if (c == '\\' && has_identifier_argument >= 0) {
2100                                         if (has_identifier_argument != 0) {
2101                                                 has_identifier_argument = 1;
2102
2103                                                 int peek = peek_char ();
2104                                                 if (peek == 'U' || peek == 'u') {
2105                                                         int surrogate;
2106                                                         c = EscapeUnicode (c, out surrogate);
2107                                                         if (surrogate != 0) {
2108                                                                 if (is_identifier_part_character ((char) c)) {
2109                                                                         if (pos == value_builder.Length)
2110                                                                                 Array.Resize (ref value_builder, pos * 2);
2111
2112                                                                         value_builder[pos++] = (char) c;
2113                                                                 }
2114                                                                 c = surrogate;
2115                                                         }
2116                                                 }
2117                                         } else {
2118                                                 has_identifier_argument = -1;
2119                                         }
2120                                 } else if (c == '/' && peek_char () == '/') {
2121                                         //
2122                                         // Eat single-line comments
2123                                         //
2124                                         get_char ();
2125                                         ReadToEndOfLine ();
2126                                         break;
2127                                 }
2128
2129                                 if (pos == value_builder.Length)
2130                                         Array.Resize (ref value_builder, pos * 2);
2131
2132                                 value_builder[pos++] = (char) c;
2133                                 c = get_char ();
2134                         }
2135
2136                         if (pos != 0) {
2137                                 if (pos > MaxIdentifierLength)
2138                                         arg = new string (value_builder, 0, pos);
2139                                 else
2140                                         arg = InternIdentifier (value_builder, pos);
2141
2142                                 // Eat any trailing whitespaces
2143                                 arg = arg.Trim (simple_whitespaces);
2144                         }
2145
2146                         return cmd;
2147                 }
2148
2149                 //
2150                 // Handles the #line directive
2151                 //
2152                 bool PreProcessLine ()
2153                 {
2154                         Location loc = Location;
2155
2156                         int c;
2157
2158                         int length = TokenizePreprocessorKeyword (out c);
2159                         if (length == line_default.Length) {
2160                                 if (!IsTokenIdentifierEqual (line_default))
2161                                         return false;
2162
2163                                 current_source = source_file.SourceFile;
2164                                 if (!hidden_block_start.IsNull) {
2165                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
2166                                         hidden_block_start = Location.Null;
2167                                 }
2168
2169                                 ref_line = line;
2170                                 return true;
2171                         }
2172
2173                         if (length == line_hidden.Length) {
2174                                 if (!IsTokenIdentifierEqual (line_hidden))
2175                                         return false;
2176
2177                                 if (hidden_block_start.IsNull)
2178                                         hidden_block_start = loc;
2179
2180                                 return true;
2181                         }
2182
2183                         if (length != 0 || c < '0' || c > '9') {
2184                                 //
2185                                 // Eat any remaining characters to continue parsing on next line
2186                                 //
2187                                 ReadToEndOfLine ();
2188                                 return false;
2189                         }
2190
2191                         int new_line = TokenizeNumber (c);
2192                         if (new_line < 1) {
2193                                 //
2194                                 // Eat any remaining characters to continue parsing on next line
2195                                 //
2196                                 ReadToEndOfLine ();
2197                                 return new_line != 0;
2198                         }
2199
2200                         c = get_char ();
2201                         if (c == ' ') {
2202                                 // skip over white space
2203                                 do {
2204                                         c = get_char ();
2205                                 } while (c == ' ' || c == '\t');
2206                         } else if (c == '"') {
2207                                 c = 0;
2208                         }
2209
2210                         if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
2211                                 //
2212                                 // Eat any remaining characters to continue parsing on next line
2213                                 //
2214                                 ReadToEndOfLine ();
2215
2216                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2217                                 return true;
2218                         }
2219
2220                         string new_file_name = null;
2221                         if (c == '"') {
2222                                 new_file_name = TokenizeFileName (ref c);
2223
2224                                 // skip over white space
2225                                 while (c == ' ' || c == '\t') {
2226                                         c = get_char ();
2227                                 }
2228                         }
2229
2230                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2231
2232                         } else if (c == '/') {
2233                                 ReadSingleLineComment ();
2234                         } else {
2235                                 //
2236                                 // Eat any remaining characters to continue parsing on next line
2237                                 //
2238                                 ReadToEndOfLine ();
2239
2240                                 Error_EndLineExpected ();
2241                                 return true;
2242                         }
2243
2244                         if (new_file_name != null) {
2245                                 current_source = context.LookupFile (source_file, new_file_name);
2246                                 source_file.AddIncludeFile (current_source);
2247                         }
2248
2249                         if (!hidden_block_start.IsNull) {
2250                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2251                                 hidden_block_start = Location.Null;
2252                         }
2253
2254                         ref_line = new_line;
2255                         return true;
2256                 }
2257
2258                 //
2259                 // Handles #define and #undef
2260                 //
2261                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2262                 {
2263                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2264                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2265                                 return;
2266                         }
2267
2268                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2269                                 Error_EndLineExpected ();
2270                                 return;
2271                         }
2272
2273                         if (!is_identifier_start_character (ident [0]))
2274                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2275
2276                         foreach (char c in ident.Substring (1)){
2277                                 if (!is_identifier_part_character (c)){
2278                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2279                                         return;
2280                                 }
2281                         }
2282
2283                         if (!caller_is_taking)
2284                                 return;
2285
2286                         if (is_define) {
2287                                 //
2288                                 // #define ident
2289                                 //
2290                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2291                                         return;
2292
2293                                 source_file.AddDefine (ident);
2294                         } else {
2295                                 //
2296                                 // #undef ident
2297                                 //
2298                                 source_file.AddUndefine (ident);
2299                         }
2300                 }
2301
2302                 byte read_hex (out bool error)
2303                 {
2304                         int total;
2305                         int c = get_char ();
2306
2307                         if ((c >= '0') && (c <= '9'))
2308                                 total = (int) c - (int) '0';
2309                         else if ((c >= 'A') && (c <= 'F'))
2310                                 total = (int) c - (int) 'A' + 10;
2311                         else if ((c >= 'a') && (c <= 'f'))
2312                                 total = (int) c - (int) 'a' + 10;
2313                         else {
2314                                 error = true;
2315                                 return 0;
2316                         }
2317
2318                         total *= 16;
2319                         c = get_char ();
2320
2321                         if ((c >= '0') && (c <= '9'))
2322                                 total += (int) c - (int) '0';
2323                         else if ((c >= 'A') && (c <= 'F'))
2324                                 total += (int) c - (int) 'A' + 10;
2325                         else if ((c >= 'a') && (c <= 'f'))
2326                                 total += (int) c - (int) 'a' + 10;
2327                         else {
2328                                 error = true;
2329                                 return 0;
2330                         }
2331
2332                         error = false;
2333                         return (byte) total;
2334                 }
2335
2336                 //
2337                 // Parses #pragma checksum
2338                 //
2339                 bool ParsePragmaChecksum ()
2340                 {
2341                         //
2342                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2343                         //
2344                         // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
2345                         //
2346                         int c = get_char ();
2347
2348                         if (c != '"')
2349                                 return false;
2350
2351                         string file_name = TokenizeFileName (ref c);
2352
2353                         // TODO: Any white-spaces count
2354                         if (c != ' ')
2355                                 return false;
2356
2357                         SourceFile file = context.LookupFile (source_file, file_name);
2358
2359                         if (get_char () != '"' || get_char () != '{')
2360                                 return false;
2361
2362                         bool error;
2363                         byte[] guid_bytes = new byte [16];
2364                         int i = 0;
2365
2366                         for (; i < 4; i++) {
2367                                 guid_bytes [i] = read_hex (out error);
2368                                 if (error)
2369                                         return false;
2370                         }
2371
2372                         if (get_char () != '-')
2373                                 return false;
2374
2375                         for (; i < 10; i++) {
2376                                 guid_bytes [i] = read_hex (out error);
2377                                 if (error)
2378                                         return false;
2379
2380                                 guid_bytes [i++] = read_hex (out error);
2381                                 if (error)
2382                                         return false;
2383
2384                                 if (get_char () != '-')
2385                                         return false;
2386                         }
2387
2388                         for (; i < 16; i++) {
2389                                 guid_bytes [i] = read_hex (out error);
2390                                 if (error)
2391                                         return false;
2392                         }
2393
2394                         if (get_char () != '}' || get_char () != '"')
2395                                 return false;
2396
2397                         // TODO: Any white-spaces count
2398                         c = get_char ();
2399                         if (c != ' ')
2400                                 return false;
2401
2402                         if (get_char () != '"')
2403                                 return false;
2404
2405                         // Any length of checksum
2406                         List<byte> checksum_bytes = new List<byte> (16);
2407
2408                         var checksum_location = Location;
2409                         c = peek_char ();
2410                         while (c != '"' && c != -1) {
2411                                 checksum_bytes.Add (read_hex (out error));
2412                                 if (error)
2413                                         return false;
2414
2415                                 c = peek_char ();
2416                         }
2417
2418                         if (c == '/') {
2419                                 ReadSingleLineComment ();
2420                         } else if (get_char () != '"') {
2421                                 return false;
2422                         }
2423
2424                         if (context.Settings.GenerateDebugInfo) {
2425                                 var chsum = checksum_bytes.ToArray ();
2426
2427                                 if (file.HasChecksum) {
2428                                         if (!ArrayComparer.IsEqual (file.Checksum, chsum)) {
2429                                                 // TODO: Report.SymbolRelatedToPreviousError
2430                                                 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name);
2431                                         }
2432                                 }
2433
2434                                 file.SetChecksum (guid_bytes, chsum);
2435                                 current_source.AutoGenerated = true;
2436                         }
2437
2438                         return true;
2439                 }
2440
2441                 bool IsTokenIdentifierEqual (char[] identifier)
2442                 {
2443                         for (int i = 0; i < identifier.Length; ++i) {
2444                                 if (identifier[i] != id_builder[i])
2445                                         return false;
2446                         }
2447
2448                         return true;
2449                 }
2450
2451                 bool ScanClosingInterpolationBrace ()
2452                 {
2453                         PushPosition ();
2454
2455                         bool? res = null;
2456                         int str_quote = 0;
2457                         do {
2458                                 var c = reader.Read ();
2459                                 switch (c) {
2460                                 case '\"':
2461                                         ++str_quote;
2462                                         break;
2463                                 case -1:
2464                                         res = false;
2465                                         break;
2466                                 case '}':
2467                                         if (str_quote % 2 == 1) {
2468                                                 res = true;
2469                                         }
2470
2471                                         break;
2472                                 }
2473                         } while (res == null);
2474
2475                         PopPosition ();
2476                         return res.Value;
2477                 }
2478
2479                 int TokenizeNumber (int value)
2480                 {
2481                         number_pos = 0;
2482
2483                         decimal_digits (value);
2484                         uint ui = (uint) (number_builder[0] - '0');
2485
2486                         try {
2487                                 for (int i = 1; i < number_pos; i++) {
2488                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2489                                 }
2490
2491                                 return (int) ui;
2492                         } catch (OverflowException) {
2493                                 Error_NumericConstantTooLong ();
2494                                 return -1;
2495                         }
2496                 }
2497
2498                 string TokenizeFileName (ref int c)
2499                 {
2500                         var string_builder = new StringBuilder ();
2501                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2502                                 c = get_char ();
2503                                 if (c == '"') {
2504                                         c = get_char ();
2505                                         break;
2506                                 }
2507
2508                                 string_builder.Append ((char) c);
2509                         }
2510
2511                         if (string_builder.Length == 0) {
2512                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2513                         }
2514
2515
2516                         return string_builder.ToString ();
2517                 }
2518
2519                 int TokenizePragmaWarningIdentifier (ref int c, ref bool identifier)
2520                 {
2521
2522                         if ((c >= '0' && c <= '9') || is_identifier_start_character (c)) {
2523                                 int number;
2524
2525                                 if (c >= '0' && c <= '9') {
2526                                         number_pos = 0;
2527                                         number = TokenizeNumber (c);
2528
2529                                         c = get_char ();
2530
2531                                         if (c != ' ' && c != '\t' && c != ',' && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS) {
2532                                                 return ReadPragmaWarningComment (c);
2533                                         }
2534                                 } else {
2535                                         //
2536                                         // LAMESPEC v6: No spec what identifier really is in this context, it seems keywords are allowed too
2537                                         //
2538                                         int pos = 0;
2539                                         number = -1;
2540                                         id_builder [pos++] = (char)c;
2541                                         while (c < MaxIdentifierLength) {
2542                                                 c = reader.Read ();
2543                                                 id_builder [pos] = (char)c;
2544
2545                                                 if (c >= '0' && c <= '9') {
2546                                                         if (pos == 6 && id_builder [0] == 'C' && id_builder [1] == 'S') {
2547                                                                 // Recognize CSXXXX as C# XXXX warning
2548                                                                 number = 0;
2549                                                                 int pow = 1000;
2550                                                                 for (int i = 0; i < 4; ++i) {
2551                                                                         var ch = id_builder [i + 2];
2552                                                                         if (ch < '0' || ch > '9') {
2553                                                                                 number = -1;
2554                                                                                 break;
2555                                                                         }
2556
2557                                                                         number += (ch - '0') * pow;
2558                                                                         pow /= 10;
2559                                                                 }
2560                                                         }
2561                                                 } else if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '_') {
2562                                                         break;
2563                                                 }
2564
2565                                                 ++pos;
2566                                         }
2567
2568                                         if (number < 0) {
2569                                                 identifier = true;
2570                                                 number = pos;
2571                                         }
2572                                 }
2573
2574                                 // skip over white space
2575                                 while (c == ' ' || c == '\t')
2576                                         c = get_char ();
2577
2578                                 if (c == ',') {
2579                                         c = get_char ();
2580                                 }
2581
2582                                 // skip over white space
2583                                 while (c == ' ' || c == '\t')
2584                                         c = get_char ();
2585
2586                                 return number;
2587                         }
2588
2589                         return ReadPragmaWarningComment (c);
2590                 }
2591
2592                 int ReadPragmaWarningComment (int c)
2593                 {
2594                         if (c == '/') {
2595                                 ReadSingleLineComment ();
2596                         } else {
2597                                 Report.Warning (1692, 1, Location, "Invalid number");
2598
2599                                 // Read everything till the end of the line or file
2600                                 ReadToEndOfLine ();
2601                         }
2602
2603                         return -1;
2604                 }
2605
2606                 void ReadToEndOfLine ()
2607                 {
2608                         int c;
2609                         do {
2610                                 c = get_char ();
2611                         } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
2612                 }
2613
2614                 void ReadSingleLineComment ()
2615                 {
2616                         if (peek_char () != '/')
2617                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2618
2619                         // Read everything till the end of the line or file
2620                         ReadToEndOfLine ();
2621                 }
2622
2623                 /// <summary>
2624                 /// Handles #pragma directive
2625                 /// </summary>
2626                 void ParsePragmaDirective ()
2627                 {
2628                         int c;
2629                         int length = TokenizePreprocessorKeyword (out c);
2630                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2631                                 length = TokenizePreprocessorKeyword (out c);
2632
2633                                 //
2634                                 // #pragma warning disable
2635                                 // #pragma warning restore
2636                                 //
2637                                 if (length == pragma_warning_disable.Length) {
2638                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2639                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2640                                                 // skip over white space
2641                                                 while (c == ' ' || c == '\t')
2642                                                         c = get_char ();
2643
2644                                                 var loc = Location;
2645
2646                                                 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
2647                                                         if (c == '/')
2648                                                                 ReadSingleLineComment ();
2649
2650                                                         //
2651                                                         // Disable/Restore all warnings
2652                                                         //
2653                                                         if (disable) {
2654                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2655                                                         } else {
2656                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2657                                                         }
2658                                                 } else {
2659                                                         //
2660                                                         // Disable/Restore a warning or group of warnings
2661                                                         //
2662                                                         int code;
2663                                                         do {
2664                                                                 bool identifier = false;
2665                                                                 code = TokenizePragmaWarningIdentifier (ref c, ref identifier);
2666                                                                 if (code > 0) {
2667                                                                         if (identifier) {
2668                                                                                 // no-op, custom warnings cannot occur in mcs
2669                                                                         } else if (disable) {
2670                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2671                                                                         } else {
2672                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2673                                                                         }
2674                                                                 }
2675                                                         } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
2676                                                 }
2677
2678                                                 return;
2679                                         }
2680                                 }
2681
2682                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2683
2684                                 // Eat any remaining characters on the line
2685                                 ReadToEndOfLine ();
2686
2687                                 return;
2688                         }
2689
2690                         //
2691                         // #pragma checksum
2692                         //
2693                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2694                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2695                                         Report.Warning (1695, 1, Location,
2696                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2697                                 }
2698
2699                                 return;
2700                         }
2701
2702                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2703
2704                         // Eat any remaining characters on the line
2705                         ReadToEndOfLine ();
2706                 }
2707
2708                 bool eval_val (string s)
2709                 {
2710                         if (s == "true")
2711                                 return true;
2712                         if (s == "false")
2713                                 return false;
2714
2715                         return source_file.IsConditionalDefined (s);
2716                 }
2717
2718                 bool pp_primary (ref string s)
2719                 {
2720                         s = s.Trim ();
2721                         int len = s.Length;
2722
2723                         if (len > 0){
2724                                 char c = s [0];
2725
2726                                 if (c == '('){
2727                                         s = s.Substring (1);
2728                                         bool val = pp_expr (ref s, false);
2729                                         if (s.Length > 0 && s [0] == ')'){
2730                                                 s = s.Substring (1);
2731                                                 return val;
2732                                         }
2733                                         Error_InvalidDirective ();
2734                                         return false;
2735                                 }
2736
2737                                 if (is_identifier_start_character (c)){
2738                                         int j = 1;
2739
2740                                         while (j < len){
2741                                                 c = s [j];
2742
2743                                                 if (is_identifier_part_character (c)){
2744                                                         j++;
2745                                                         continue;
2746                                                 }
2747                                                 bool v = eval_val (s.Substring (0, j));
2748                                                 s = s.Substring (j);
2749                                                 return v;
2750                                         }
2751                                         bool vv = eval_val (s);
2752                                         s = "";
2753                                         return vv;
2754                                 }
2755                         }
2756                         Error_InvalidDirective ();
2757                         return false;
2758                 }
2759
2760                 bool pp_unary (ref string s)
2761                 {
2762                         s = s.Trim ();
2763                         int len = s.Length;
2764
2765                         if (len > 0){
2766                                 if (s [0] == '!'){
2767                                         if (len > 1 && s [1] == '='){
2768                                                 Error_InvalidDirective ();
2769                                                 return false;
2770                                         }
2771                                         s = s.Substring (1);
2772                                         return ! pp_primary (ref s);
2773                                 } else
2774                                         return pp_primary (ref s);
2775                         } else {
2776                                 Error_InvalidDirective ();
2777                                 return false;
2778                         }
2779                 }
2780
2781                 bool pp_eq (ref string s)
2782                 {
2783                         bool va = pp_unary (ref s);
2784
2785                         s = s.Trim ();
2786                         int len = s.Length;
2787                         if (len > 0){
2788                                 if (s [0] == '='){
2789                                         if (len > 2 && s [1] == '='){
2790                                                 s = s.Substring (2);
2791                                                 return va == pp_unary (ref s);
2792                                         } else {
2793                                                 Error_InvalidDirective ();
2794                                                 return false;
2795                                         }
2796                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2797                                         s = s.Substring (2);
2798
2799                                         return va != pp_unary (ref s);
2800
2801                                 }
2802                         }
2803
2804                         return va;
2805
2806                 }
2807
2808                 bool pp_and (ref string s)
2809                 {
2810                         bool va = pp_eq (ref s);
2811
2812                         s = s.Trim ();
2813                         int len = s.Length;
2814                         if (len > 0){
2815                                 if (s [0] == '&'){
2816                                         if (len > 2 && s [1] == '&'){
2817                                                 s = s.Substring (2);
2818                                                 return (va & pp_and (ref s));
2819                                         } else {
2820                                                 Error_InvalidDirective ();
2821                                                 return false;
2822                                         }
2823                                 }
2824                         }
2825                         return va;
2826                 }
2827
2828                 //
2829                 // Evaluates an expression for `#if' or `#elif'
2830                 //
2831                 bool pp_expr (ref string s, bool isTerm)
2832                 {
2833                         bool va = pp_and (ref s);
2834                         s = s.Trim ();
2835                         int len = s.Length;
2836                         if (len > 0){
2837                                 char c = s [0];
2838
2839                                 if (c == '|'){
2840                                         if (len > 2 && s [1] == '|'){
2841                                                 s = s.Substring (2);
2842                                                 return va | pp_expr (ref s, isTerm);
2843                                         } else {
2844                                                 Error_InvalidDirective ();
2845                                                 return false;
2846                                         }
2847                                 }
2848                                 if (isTerm) {
2849                                         Error_EndLineExpected ();
2850                                         return false;
2851                                 }
2852                         }
2853
2854                         return va;
2855                 }
2856
2857                 bool eval (string s)
2858                 {
2859                         bool v = pp_expr (ref s, true);
2860                         s = s.Trim ();
2861                         if (s.Length != 0){
2862                                 return false;
2863                         }
2864
2865                         return v;
2866                 }
2867
2868                 void Error_NumericConstantTooLong ()
2869                 {
2870                         Report.Error (1021, Location, "Integral constant is too large");
2871                 }
2872
2873                 void Error_InvalidDirective ()
2874                 {
2875                         Report.Error (1517, Location, "Invalid preprocessor directive");
2876                 }
2877
2878                 void Error_UnexpectedDirective (string extra)
2879                 {
2880                         Report.Error (
2881                                 1028, Location,
2882                                 "Unexpected processor directive ({0})", extra);
2883                 }
2884
2885                 void Error_TokensSeen ()
2886                 {
2887                         Report.Error (1032, Location,
2888                                 "Cannot define or undefine preprocessor symbols after first token in file");
2889                 }
2890
2891                 void Eror_WrongPreprocessorLocation ()
2892                 {
2893                         Report.Error (1040, Location,
2894                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2895                 }
2896
2897                 void Error_EndLineExpected ()
2898                 {
2899                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2900                 }
2901
2902                 //
2903                 // Raises a warning when tokenizer found documentation comment
2904                 // on unexpected place
2905                 //
2906                 void WarningMisplacedComment (Location loc)
2907                 {
2908                         if (doc_state != XmlCommentState.Error) {
2909                                 doc_state = XmlCommentState.Error;
2910                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2911                         }
2912                 }
2913
2914                 //
2915                 // if true, then the code continues processing the code
2916                 // if false, the code stays in a loop until another directive is
2917                 // reached.
2918                 // When caller_is_taking is false we ignore all directives except the ones
2919                 // which can help us to identify where the #if block ends
2920                 bool ParsePreprocessingDirective (bool caller_is_taking)
2921                 {
2922                         string arg;
2923                         bool region_directive = false;
2924
2925                         var directive = get_cmd_arg (out arg);
2926
2927                         //
2928                         // The first group of pre-processing instructions is always processed
2929                         //
2930                         switch (directive) {
2931                         case PreprocessorDirective.Region:
2932                                 region_directive = true;
2933                                 arg = "true";
2934                                 goto case PreprocessorDirective.If;
2935
2936                         case PreprocessorDirective.Endregion:
2937                                 if (ifstack == null || ifstack.Count == 0){
2938                                         Error_UnexpectedDirective ("no #region for this #endregion");
2939                                         return true;
2940                                 }
2941                                 int pop = ifstack.Pop ();
2942
2943                                 if ((pop & REGION) == 0)
2944                                         Report.Error (1027, Location, "Expected `#endif' directive");
2945
2946                                 return caller_is_taking;
2947
2948                         case PreprocessorDirective.If:
2949                                 if (ifstack == null)
2950                                         ifstack = new Stack<int> (2);
2951
2952                                 int flags = region_directive ? REGION : 0;
2953                                 if (ifstack.Count == 0){
2954                                         flags |= PARENT_TAKING;
2955                                 } else {
2956                                         int state = ifstack.Peek ();
2957                                         if ((state & TAKING) != 0) {
2958                                                 flags |= PARENT_TAKING;
2959                                         }
2960                                 }
2961
2962                                 if (eval (arg) && caller_is_taking) {
2963                                         ifstack.Push (flags | TAKING);
2964                                         return true;
2965                                 }
2966                                 ifstack.Push (flags);
2967                                 return false;
2968
2969                         case PreprocessorDirective.Endif:
2970                                 if (ifstack == null || ifstack.Count == 0){
2971                                         Error_UnexpectedDirective ("no #if for this #endif");
2972                                         return true;
2973                                 } else {
2974                                         pop = ifstack.Pop ();
2975
2976                                         if ((pop & REGION) != 0)
2977                                                 Report.Error (1038, Location, "#endregion directive expected");
2978
2979                                         if (arg.Length != 0) {
2980                                                 Error_EndLineExpected ();
2981                                         }
2982
2983                                         if (ifstack.Count == 0)
2984                                                 return true;
2985
2986                                         int state = ifstack.Peek ();
2987                                         return (state & TAKING) != 0;
2988                                 }
2989
2990                         case PreprocessorDirective.Elif:
2991                                 if (ifstack == null || ifstack.Count == 0){
2992                                         Error_UnexpectedDirective ("no #if for this #elif");
2993                                         return true;
2994                                 } else {
2995                                         int state = ifstack.Pop ();
2996
2997                                         if ((state & REGION) != 0) {
2998                                                 Report.Error (1038, Location, "#endregion directive expected");
2999                                                 return true;
3000                                         }
3001
3002                                         if ((state & ELSE_SEEN) != 0){
3003                                                 Error_UnexpectedDirective ("#elif not valid after #else");
3004                                                 return true;
3005                                         }
3006
3007                                         if ((state & TAKING) != 0) {
3008                                                 ifstack.Push (0);
3009                                                 return false;
3010                                         }
3011
3012                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
3013                                                 ifstack.Push (state | TAKING);
3014                                                 return true;
3015                                         }
3016
3017                                         ifstack.Push (state);
3018                                         return false;
3019                                 }
3020
3021                         case PreprocessorDirective.Else:
3022                                 if (ifstack == null || ifstack.Count == 0){
3023                                         Error_UnexpectedDirective ("no #if for this #else");
3024                                         return true;
3025                                 } else {
3026                                         int state = ifstack.Peek ();
3027
3028                                         if ((state & REGION) != 0) {
3029                                                 Report.Error (1038, Location, "#endregion directive expected");
3030                                                 return true;
3031                                         }
3032
3033                                         if ((state & ELSE_SEEN) != 0){
3034                                                 Error_UnexpectedDirective ("#else within #else");
3035                                                 return true;
3036                                         }
3037
3038                                         ifstack.Pop ();
3039
3040                                         if (arg.Length != 0) {
3041                                                 Error_EndLineExpected ();
3042                                                 return true;
3043                                         }
3044
3045                                         bool ret = false;
3046                                         if ((state & PARENT_TAKING) != 0) {
3047                                                 ret = (state & TAKING) == 0;
3048
3049                                                 if (ret)
3050                                                         state |= TAKING;
3051                                                 else
3052                                                         state &= ~TAKING;
3053                                         }
3054
3055                                         ifstack.Push (state | ELSE_SEEN);
3056
3057                                         return ret;
3058                                 }
3059                         case PreprocessorDirective.Define:
3060                                 if (any_token_seen){
3061                                         if (caller_is_taking)
3062                                                 Error_TokensSeen ();
3063                                         return caller_is_taking;
3064                                 }
3065                                 PreProcessDefinition (true, arg, caller_is_taking);
3066                                 return caller_is_taking;
3067
3068                         case PreprocessorDirective.Undef:
3069                                 if (any_token_seen){
3070                                         if (caller_is_taking)
3071                                                 Error_TokensSeen ();
3072                                         return caller_is_taking;
3073                                 }
3074                                 PreProcessDefinition (false, arg, caller_is_taking);
3075                                 return caller_is_taking;
3076
3077                         case PreprocessorDirective.Invalid:
3078                                 Report.Error (1024, Location, "Wrong preprocessor directive");
3079                                 return true;
3080                         }
3081
3082                         //
3083                         // These are only processed if we are in a `taking' block
3084                         //
3085                         if (!caller_is_taking)
3086                                 return false;
3087
3088                         switch (directive){
3089                         case PreprocessorDirective.Error:
3090                                 Report.Error (1029, Location, "#error: '{0}'", arg);
3091                                 return true;
3092
3093                         case PreprocessorDirective.Warning:
3094                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
3095                                 return true;
3096
3097                         case PreprocessorDirective.Pragma:
3098                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
3099                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
3100                                 }
3101
3102                                 ParsePragmaDirective ();
3103                                 return true;
3104
3105                         case PreprocessorDirective.Line:
3106                                 Location loc = Location;
3107                                 if (!PreProcessLine ())
3108                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
3109
3110                                 return caller_is_taking;
3111                         }
3112
3113                         throw new NotImplementedException (directive.ToString ());
3114                 }
3115
3116                 int consume_string (bool quoted)
3117                 {
3118                         int c;
3119                         int pos = 0;
3120                         Location start_location = Location;
3121                         if (quoted)
3122                                 start_location = start_location - 1;
3123
3124 #if FULL_AST
3125                         int reader_pos = reader.Position;
3126 #endif
3127
3128                         while (true){
3129                                 // Cannot use get_char because of \r in quoted strings
3130                                 if (putback_char != -1) {
3131                                         c = putback_char;
3132                                         putback_char = -1;
3133                                 } else {
3134                                         c = reader.Read ();
3135                                 }
3136
3137                                 if (c == '"') {
3138                                         ++col;
3139
3140                                         if (quoted && peek_char () == '"') {
3141                                                 if (pos == value_builder.Length)
3142                                                         Array.Resize (ref value_builder, pos * 2);
3143
3144                                                 value_builder[pos++] = (char) c;
3145                                                 get_char ();
3146                                                 continue;
3147                                         }
3148
3149                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3150                                         val = res;
3151 #if FULL_AST
3152                                         res.ParsedValue = quoted ?
3153                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
3154                                                 reader.ReadChars (reader_pos - 1, reader.Position);
3155 #endif
3156
3157                                         return Token.LITERAL;
3158                                 }
3159
3160                                 if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3161                                         if (!quoted) {
3162                                                 Report.Error (1010, Location, "Newline in constant");
3163
3164                                                 advance_line ();
3165
3166                                                 // Don't add \r to string literal
3167                                                 if (pos > 1 && value_builder [pos - 1] == '\r')
3168                                                         --pos;
3169
3170                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
3171                                                 return Token.LITERAL;
3172                                         }
3173
3174                                         advance_line ();
3175                                 } else if (c == '\\' && !quoted) {
3176                                         ++col;
3177                                         int surrogate;
3178                                         c = escape (c, out surrogate);
3179                                         if (c == -1)
3180                                                 return Token.ERROR;
3181                                         if (surrogate != 0) {
3182                                                 if (pos == value_builder.Length)
3183                                                         Array.Resize (ref value_builder, pos * 2);
3184
3185                                                 value_builder[pos++] = (char) c;
3186                                                 c = surrogate;
3187                                         }
3188                                 } else if (c == -1) {
3189                                         Report.Error (1039, Location, "Unterminated string literal");
3190                                         return Token.EOF;
3191                                 } else {
3192                                         ++col;
3193                                 }
3194
3195                                 if (pos == value_builder.Length)
3196                                         Array.Resize (ref value_builder, pos * 2);
3197
3198                                 value_builder[pos++] = (char) c;
3199                         }
3200                 }
3201
3202                 private int consume_identifier (int s)
3203                 {
3204                         int res = consume_identifier (s, false);
3205
3206                         if (doc_state == XmlCommentState.Allowed)
3207                                 doc_state = XmlCommentState.NotAllowed;
3208
3209                         return res;
3210                 }
3211
3212                 int consume_identifier (int c, bool quoted)
3213                 {
3214                         //
3215                         // This method is very performance sensitive. It accounts
3216                         // for approximately 25% of all parser time
3217                         //
3218
3219                         int pos = 0;
3220                         int column = col;
3221                         if (quoted)
3222                                 --column;
3223
3224                         if (c == '\\') {
3225                                 int surrogate;
3226                                 c = escape (c, out surrogate);
3227                                 if (quoted || is_identifier_start_character (c)) {
3228                                         // it's added bellow
3229                                 } else if (surrogate != 0) {
3230                                         id_builder [pos++] = (char)c;
3231                                         c = surrogate;
3232                                 } else {
3233                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3234                                         return Token.ERROR;
3235                                 }
3236                         }
3237
3238                         id_builder [pos++] = (char) c;
3239
3240                         try {
3241                                 while (true) {
3242                                         c = reader.Read ();
3243
3244                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
3245                                                 id_builder [pos++] = (char) c;
3246                                                 continue;
3247                                         }
3248
3249                                         if (c < 0x80) {
3250                                                 if (c == '\\') {
3251                                                         int surrogate;
3252                                                         c = escape (c, out surrogate);
3253                                                         if (is_identifier_part_character ((char) c))
3254                                                                 id_builder[pos++] = (char) c;
3255                                                         else if (surrogate != 0) {
3256                                                                 c = surrogate;
3257                                                         } else {
3258                                                                 switch (c) {
3259                                                                 // TODO: Probably need more whitespace characters
3260                                                                 case 0xFEFF:
3261                                                                         putback_char = c;
3262                                                                         break;
3263                                                                 default:
3264                                                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3265                                                                         return Token.ERROR;
3266                                                                 }
3267                                                         }
3268
3269                                                         continue;
3270                                                 }
3271                                         } else if (is_identifier_part_character_slow_part ((char) c)) {
3272                                                 id_builder [pos++] = (char) c;
3273                                                 continue;
3274                                         }
3275
3276                                         putback_char = c;
3277                                         break;
3278                                 }
3279                         } catch (IndexOutOfRangeException) {
3280                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
3281                                 --pos;
3282                                 col += pos;
3283                         }
3284
3285                         col += pos - 1;
3286
3287                         //
3288                         // Optimization: avoids doing the keyword lookup
3289                         // on uppercase letters
3290                         //
3291                         if (id_builder [0] >= '_' && !quoted) {
3292                                 int keyword = GetKeyword (id_builder, pos);
3293                                 if (keyword != -1) {
3294                                         val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column);
3295                                         return keyword;
3296                                 }
3297                         }
3298
3299                         string s = InternIdentifier (id_builder, pos);
3300                         val = ltb.Create (s, current_source, ref_line, column);
3301                         if (quoted && parsing_attribute_section)
3302                                 AddEscapedIdentifier (((LocatedToken) val).Location);
3303
3304                         return Token.IDENTIFIER;
3305                 }
3306
3307                 string InternIdentifier (char[] charBuffer, int length)
3308                 {
3309                         //
3310                         // Keep identifiers in an array of hashtables to avoid needless
3311                         // allocations
3312                         //
3313                         var identifiers_group = identifiers[length];
3314                         string s;
3315                         if (identifiers_group != null) {
3316                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3317                                         return s;
3318                                 }
3319                         } else {
3320                                 // TODO: this should be number of files dependant
3321                                 // corlib compilation peaks at 1000 and System.Core at 150
3322                                 int capacity = length > 20 ? 10 : 100;
3323                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3324                                 identifiers[length] = identifiers_group;
3325                         }
3326
3327                         char[] chars = new char[length];
3328                         Array.Copy (charBuffer, chars, length);
3329
3330                         s = new string (charBuffer, 0, length);
3331                         identifiers_group.Add (chars, s);
3332                         return s;
3333                 }
3334
3335                 public int xtoken ()
3336                 {
3337                         if (parsing_interpolation_format) {
3338                                 return TokenizeInterpolationFormat ();
3339                         }
3340
3341                         int d, c;
3342
3343                         // Whether we have seen comments on the current line
3344                         bool comments_seen = false;
3345                         while ((c = get_char ()) != -1) {
3346                                 switch (c) {
3347                                 case '\t':
3348                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3349                                         continue;
3350
3351                                 case ' ':
3352                                 case '\f':
3353                                 case '\v':
3354                                 case 0xa0:
3355                                 case 0:
3356                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3357                                         continue;
3358
3359 /*                              This is required for compatibility with .NET
3360                                 case 0xEF:
3361                                         if (peek_char () == 0xBB) {
3362                                                 PushPosition ();
3363                                                 get_char ();
3364                                                 if (get_char () == 0xBF)
3365                                                         continue;
3366                                                 PopPosition ();
3367                                         }
3368                                         break;
3369 */
3370                                 case '\\':
3371                                         tokens_seen = true;
3372                                         return consume_identifier (c);
3373
3374                                 case '{':
3375                                         val = ltb.Create (current_source, ref_line, col);
3376                                         return Token.OPEN_BRACE;
3377                                 case '}':
3378                                         if (parsing_string_interpolation > 0) {
3379                                                 --parsing_string_interpolation;
3380                                                 bool quoted;
3381                                                 if (parsing_string_interpolation_quoted != null && parsing_string_interpolation_quoted.Count > 0) {
3382                                                         quoted = parsing_string_interpolation_quoted.Pop ();
3383                                                 } else {
3384                                                         quoted = false;
3385                                                 }
3386
3387                                                 return TokenizeInterpolatedString (quoted);
3388                                         }
3389
3390                                         val = ltb.Create (current_source, ref_line, col);
3391                                         return Token.CLOSE_BRACE;
3392                                 case '[':
3393                                         // To block doccomment inside attribute declaration.
3394                                         if (doc_state == XmlCommentState.Allowed)
3395                                                 doc_state = XmlCommentState.NotAllowed;
3396
3397                                         val = ltb.Create (current_source, ref_line, col);
3398
3399                                         if (parsing_block == 0 || lambda_arguments_parsing)
3400                                                 return Token.OPEN_BRACKET;
3401
3402                                         int next = peek_char ();
3403                                         switch (next) {
3404                                         case ']':
3405                                         case ',':
3406                                                 return Token.OPEN_BRACKET;
3407
3408                                         case ' ':
3409                                         case '\f':
3410                                         case '\v':
3411                                         case '\r':
3412                                         case '\n':
3413                                         case UnicodeLS:
3414                                         case UnicodePS:
3415                                         case '/':
3416                                                 next = peek_token ();
3417                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3418                                                         return Token.OPEN_BRACKET;
3419
3420                                                 return Token.OPEN_BRACKET_EXPR;
3421                                         default:
3422                                                 return Token.OPEN_BRACKET_EXPR;
3423                                         }
3424                                 case ']':
3425                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3426                                         return Token.CLOSE_BRACKET;
3427                                 case '(':
3428                                         val = ltb.Create (current_source, ref_line, col);
3429                                         //
3430                                         // An expression versions of parens can appear in block context only
3431                                         //
3432                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3433
3434                                                 //
3435                                                 // Optmize most common case where we know that parens
3436                                                 // is not special
3437                                                 //
3438                                                 switch (current_token) {
3439                                                 case Token.IDENTIFIER:
3440                                                 case Token.IF:
3441                                                 case Token.FOR:
3442                                                 case Token.FOREACH:
3443                                                 case Token.TYPEOF:
3444                                                 case Token.WHILE:
3445                                                 case Token.SWITCH:
3446                                                 case Token.USING:
3447                                                 case Token.DEFAULT:
3448                                                 case Token.DELEGATE:
3449                                                 case Token.OP_GENERICS_GT:
3450                                                         return Token.OPEN_PARENS;
3451                                                 }
3452
3453                                                 // Optimize using peek
3454                                                 int xx = peek_char ();
3455                                                 switch (xx) {
3456                                                 case '(':
3457                                                 case '\'':
3458                                                 case '"':
3459                                                 case '0':
3460                                                 case '1':
3461                                                         return Token.OPEN_PARENS;
3462                                                 }
3463
3464                                                 lambda_arguments_parsing = true;
3465                                                 PushPosition ();
3466                                                 d = TokenizeOpenParens ();
3467                                                 PopPosition ();
3468                                                 lambda_arguments_parsing = false;
3469                                                 return d;
3470                                         }
3471
3472                                         return Token.OPEN_PARENS;
3473                                 case ')':
3474                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3475                                         return Token.CLOSE_PARENS;
3476                                 case ',':
3477                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3478                                         return Token.COMMA;
3479                                 case ';':
3480                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3481                                         return Token.SEMICOLON;
3482                                 case '~':
3483                                         val = ltb.Create (current_source, ref_line, col);
3484                                         return Token.TILDE;
3485                                 case '?':
3486                                         val = ltb.Create (current_source, ref_line, col);
3487                                         return TokenizePossibleNullableType ();
3488                                 case '<':
3489                                         val = ltb.Create (current_source, ref_line, col);
3490                                         if (parsing_generic_less_than++ > 0)
3491                                                 return Token.OP_GENERICS_LT;
3492
3493                                         return TokenizeLessThan ();
3494
3495                                 case '>':
3496                                         val = ltb.Create (current_source, ref_line, col);
3497                                         d = peek_char ();
3498
3499                                         if (d == '='){
3500                                                 get_char ();
3501                                                 return Token.OP_GE;
3502                                         }
3503
3504                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3505                                                 parsing_generic_less_than--;
3506                                                 return Token.OP_GENERICS_GT;
3507                                         }
3508
3509                                         if (d == '>') {
3510                                                 get_char ();
3511                                                 d = peek_char ();
3512
3513                                                 if (d == '=') {
3514                                                         get_char ();
3515                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3516                                                 }
3517                                                 return Token.OP_SHIFT_RIGHT;
3518                                         }
3519
3520                                         return Token.OP_GT;
3521
3522                                 case '+':
3523                                         val = ltb.Create (current_source, ref_line, col);
3524                                         d = peek_char ();
3525                                         if (d == '+') {
3526                                                 d = Token.OP_INC;
3527                                         } else if (d == '=') {
3528                                                 d = Token.OP_ADD_ASSIGN;
3529                                         } else {
3530                                                 return Token.PLUS;
3531                                         }
3532                                         get_char ();
3533                                         return d;
3534
3535                                 case '-':
3536                                         val = ltb.Create (current_source, ref_line, col);
3537                                         d = peek_char ();
3538                                         if (d == '-') {
3539                                                 d = Token.OP_DEC;
3540                                         } else if (d == '=')
3541                                                 d = Token.OP_SUB_ASSIGN;
3542                                         else if (d == '>')
3543                                                 d = Token.OP_PTR;
3544                                         else {
3545                                                 return Token.MINUS;
3546                                         }
3547                                         get_char ();
3548                                         return d;
3549
3550                                 case '!':
3551                                         val = ltb.Create (current_source, ref_line, col);
3552                                         if (peek_char () == '='){
3553                                                 get_char ();
3554                                                 return Token.OP_NE;
3555                                         }
3556                                         return Token.BANG;
3557
3558                                 case '=':
3559                                         val = ltb.Create (current_source, ref_line, col);
3560                                         d = peek_char ();
3561                                         if (d == '='){
3562                                                 get_char ();
3563                                                 return Token.OP_EQ;
3564                                         }
3565                                         if (d == '>'){
3566                                                 get_char ();
3567                                                 return Token.ARROW;
3568                                         }
3569
3570                                         return Token.ASSIGN;
3571
3572                                 case '&':
3573                                         val = ltb.Create (current_source, ref_line, col);
3574                                         d = peek_char ();
3575                                         if (d == '&'){
3576                                                 get_char ();
3577                                                 return Token.OP_AND;
3578                                         }
3579                                         if (d == '='){
3580                                                 get_char ();
3581                                                 return Token.OP_AND_ASSIGN;
3582                                         }
3583                                         return Token.BITWISE_AND;
3584
3585                                 case '|':
3586                                         val = ltb.Create (current_source, ref_line, col);
3587                                         d = peek_char ();
3588                                         if (d == '|'){
3589                                                 get_char ();
3590                                                 return Token.OP_OR;
3591                                         }
3592                                         if (d == '='){
3593                                                 get_char ();
3594                                                 return Token.OP_OR_ASSIGN;
3595                                         }
3596                                         return Token.BITWISE_OR;
3597
3598                                 case '*':
3599                                         val = ltb.Create (current_source, ref_line, col);
3600                                         if (peek_char () == '='){
3601                                                 get_char ();
3602                                                 return Token.OP_MULT_ASSIGN;
3603                                         }
3604                                         return Token.STAR;
3605
3606                                 case '/':
3607                                         d = peek_char ();
3608                                         if (d == '='){
3609                                                 val = ltb.Create (current_source, ref_line, col);
3610                                                 get_char ();
3611                                                 return Token.OP_DIV_ASSIGN;
3612                                         }
3613
3614                                         // Handle double-slash comments.
3615                                         if (d == '/'){
3616                                                 if (parsing_string_interpolation > 0) {
3617                                                         Report.Error (8077, Location, "A single-line comment may not be used in an interpolated string");
3618                                                         goto case '}';
3619                                                 }
3620
3621                                                 get_char ();
3622                                                 if (doc_processing) {
3623                                                         if (peek_char () == '/') {
3624                                                                 get_char ();
3625                                                                 // Don't allow ////.
3626                                                                 if ((d = peek_char ()) != '/') {
3627                                                                         if (doc_state == XmlCommentState.Allowed)
3628                                                                                 handle_one_line_xml_comment ();
3629                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3630                                                                                 WarningMisplacedComment (Location - 3);
3631                                                                 }
3632                                                         } else {
3633                                                                 if (xml_comment_buffer.Length > 0)
3634                                                                         doc_state = XmlCommentState.NotAllowed;
3635                                                         }
3636                                                 }
3637
3638                                                 ReadToEndOfLine ();
3639
3640                                                 any_token_seen |= tokens_seen;
3641                                                 tokens_seen = false;
3642                                                 comments_seen = false;
3643                                                 continue;
3644                                         } else if (d == '*'){
3645                                                 get_char ();
3646                                                 bool docAppend = false;
3647                                                 if (doc_processing && peek_char () == '*') {
3648                                                         get_char ();
3649                                                         // But when it is /**/, just do nothing.
3650                                                         if (peek_char () == '/') {
3651                                                                 get_char ();
3652                                                                 continue;
3653                                                         }
3654                                                         if (doc_state == XmlCommentState.Allowed)
3655                                                                 docAppend = true;
3656                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3657                                                                 WarningMisplacedComment (Location - 2);
3658                                                         }
3659                                                 }
3660
3661                                                 int current_comment_start = 0;
3662                                                 if (docAppend) {
3663                                                         current_comment_start = xml_comment_buffer.Length;
3664                                                         xml_comment_buffer.Append (Environment.NewLine);
3665                                                 }
3666
3667                                                 while ((d = get_char ()) != -1){
3668                                                         if (d == '*' && peek_char () == '/'){
3669                                                                 get_char ();
3670                                                                 comments_seen = true;
3671                                                                 break;
3672                                                         }
3673                                                         if (docAppend)
3674                                                                 xml_comment_buffer.Append ((char) d);
3675
3676                                                         if (d == '\n' || d == UnicodeLS || d == UnicodePS){
3677                                                                 any_token_seen |= tokens_seen;
3678                                                                 tokens_seen = false;
3679                                                                 //
3680                                                                 // Reset 'comments_seen' just to be consistent.
3681                                                                 // It doesn't matter either way, here.
3682                                                                 //
3683                                                                 comments_seen = false;
3684                                                         }
3685                                                 }
3686                                                 if (!comments_seen)
3687                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3688
3689                                                 if (docAppend)
3690                                                         update_formatted_doc_comment (current_comment_start);
3691                                                 continue;
3692                                         }
3693                                         val = ltb.Create (current_source, ref_line, col);
3694                                         return Token.DIV;
3695
3696                                 case '%':
3697                                         val = ltb.Create (current_source, ref_line, col);
3698                                         if (peek_char () == '='){
3699                                                 get_char ();
3700                                                 return Token.OP_MOD_ASSIGN;
3701                                         }
3702                                         return Token.PERCENT;
3703
3704                                 case '^':
3705                                         val = ltb.Create (current_source, ref_line, col);
3706                                         if (peek_char () == '='){
3707                                                 get_char ();
3708                                                 return Token.OP_XOR_ASSIGN;
3709                                         }
3710                                         return Token.CARRET;
3711
3712                                 case ':':
3713                                         val = ltb.Create (current_source, ref_line, col);
3714                                         if (peek_char () == ':') {
3715                                                 get_char ();
3716                                                 return Token.DOUBLE_COLON;
3717                                         }
3718                                         return Token.COLON;
3719
3720                                 case '0': case '1': case '2': case '3': case '4':
3721                                 case '5': case '6': case '7': case '8': case '9':
3722                                         tokens_seen = true;
3723                                         return is_number (c, false);
3724
3725                                 case '\n': // white space
3726                                 case UnicodeLS:
3727                                 case UnicodePS:
3728                                         any_token_seen |= tokens_seen;
3729                                         tokens_seen = false;
3730                                         comments_seen = false;
3731                                         continue;
3732
3733                                 case '.':
3734                                         tokens_seen = true;
3735                                         d = peek_char ();
3736                                         if (d >= '0' && d <= '9')
3737                                                 return is_number (c, true);
3738
3739                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3740                                         return Token.DOT;
3741
3742                                 case '#':
3743                                         if (tokens_seen || comments_seen) {
3744                                                 Eror_WrongPreprocessorLocation ();
3745                                                 return Token.ERROR;
3746                                         }
3747
3748                                         if (ParsePreprocessingDirective (true))
3749                                                 continue;
3750
3751                                         bool directive_expected = false;
3752                                         while ((c = get_char ()) != -1) {
3753                                                 if (col == 1) {
3754                                                         directive_expected = true;
3755                                                 } else if (!directive_expected) {
3756                                                         // TODO: Implement comment support for disabled code and uncomment this code
3757 //                                                      if (c == '#') {
3758 //                                                              Eror_WrongPreprocessorLocation ();
3759 //                                                              return Token.ERROR;
3760 //                                                      }
3761                                                         continue;
3762                                                 }
3763
3764                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
3765                                                         continue;
3766
3767                                                 if (c == '#') {
3768                                                         if (ParsePreprocessingDirective (false))
3769                                                                 break;
3770                                                 }
3771                                                 directive_expected = false;
3772                                         }
3773
3774                                         if (c != -1) {
3775                                                 tokens_seen = false;
3776                                                 continue;
3777                                         }
3778
3779                                         return Token.EOF;
3780
3781                                 case '"':
3782                                         if (parsing_string_interpolation > 0 && !ScanClosingInterpolationBrace ()) {
3783                                                 parsing_string_interpolation = 0;
3784                                                 Report.Error (8076, Location, "Missing close delimiter `}' for interpolated expression");
3785                                                 val = new StringLiteral (context.BuiltinTypes, "", Location);
3786                                                 return Token.INTERPOLATED_STRING_END;
3787                                         }
3788
3789                                         return consume_string (false);
3790
3791                                 case '\'':
3792                                         return TokenizeBackslash ();
3793
3794                                 case '@':
3795                                         c = get_char ();
3796                                         if (c == '"') {
3797                                                 tokens_seen = true;
3798                                                 return consume_string (true);
3799                                         }
3800
3801                                         if (is_identifier_start_character (c)){
3802                                                 return consume_identifier (c, true);
3803                                         }
3804
3805                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3806                                         return Token.ERROR;
3807
3808                                 case '$':
3809                                         switch (peek_char ()) {
3810                                         case '"':
3811                                                 get_char ();
3812                                                 return TokenizeInterpolatedString (false);
3813                                         case '@':
3814                                                 get_char ();
3815                                                 if (peek_char () == '"') {
3816                                                         get_char ();
3817                                                         return TokenizeInterpolatedString (true);
3818                                                 }
3819
3820                                                 break;
3821                                         }
3822
3823                                         break;
3824                                 case EvalStatementParserCharacter:
3825                                         return Token.EVAL_STATEMENT_PARSER;
3826                                 case EvalCompilationUnitParserCharacter:
3827                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3828                                 case EvalUsingDeclarationsParserCharacter:
3829                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3830                                 case DocumentationXref:
3831                                         return Token.DOC_SEE;
3832                                 }
3833
3834                                 if (is_identifier_start_character (c)) {
3835                                         tokens_seen = true;
3836                                         return consume_identifier (c);
3837                                 }
3838
3839                                 if (char.IsWhiteSpace ((char) c))
3840                                         continue;
3841
3842                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3843                         }
3844
3845                         if (CompleteOnEOF){
3846                                 if (generated)
3847                                         return Token.COMPLETE_COMPLETION;
3848
3849                                 generated = true;
3850                                 return Token.GENERATE_COMPLETION;
3851                         }
3852
3853
3854                         return Token.EOF;
3855                 }
3856
3857                 int TokenizeBackslash ()
3858                 {
3859 #if FULL_AST
3860                         int read_start = reader.Position;
3861 #endif
3862                         Location start_location = Location;
3863                         int c = get_char ();
3864                         tokens_seen = true;
3865                         if (c == '\'') {
3866                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3867                                 Report.Error (1011, start_location, "Empty character literal");
3868                                 return Token.LITERAL;
3869                         }
3870
3871                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3872                                 Report.Error (1010, start_location, "Newline in constant");
3873                                 return Token.ERROR;
3874                         }
3875
3876                         int d;
3877                         c = escape (c, out d);
3878                         if (c == -1)
3879                                 return Token.ERROR;
3880                         if (d != 0)
3881                                 throw new NotImplementedException ();
3882
3883                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3884                         val = res;
3885                         c = get_char ();
3886
3887                         if (c != '\'') {
3888                                 Report.Error (1012, start_location, "Too many characters in character literal");
3889
3890                                 // Try to recover, read until newline or next "'"
3891                                 while ((c = get_char ()) != -1) {
3892                                         if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
3893                                                 break;
3894                                 }
3895                         }
3896
3897 #if FULL_AST
3898                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3899 #endif
3900
3901                         return Token.LITERAL;
3902                 }
3903
3904                 int TokenizeLessThan ()
3905                 {
3906                         int d;
3907
3908                         // Save current position and parse next token.
3909                         PushPosition ();
3910                         int generic_dimension = 0;
3911                         if (parse_less_than (ref generic_dimension)) {
3912                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3913                                         d = Token.OP_GENERICS_LT_DECL;
3914                                 } else {
3915                                         if (generic_dimension > 0) {
3916                                                 val = generic_dimension;
3917                                                 DiscardPosition ();
3918                                                 return Token.GENERIC_DIMENSION;
3919                                         }
3920
3921                                         d = Token.OP_GENERICS_LT;
3922                                 }
3923                                 PopPosition ();
3924                                 return d;
3925                         }
3926
3927                         PopPosition ();
3928                         parsing_generic_less_than = 0;
3929
3930                         d = peek_char ();
3931                         if (d == '<') {
3932                                 get_char ();
3933                                 d = peek_char ();
3934
3935                                 if (d == '=') {
3936                                         get_char ();
3937                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3938                                 }
3939                                 return Token.OP_SHIFT_LEFT;
3940                         }
3941
3942                         if (d == '=') {
3943                                 get_char ();
3944                                 return Token.OP_LE;
3945                         }
3946                         return Token.OP_LT;
3947                 }
3948
3949                 int TokenizeInterpolatedString (bool quoted)
3950                 {
3951                         int pos = 0;
3952                         var start_location = Location;
3953
3954                         while (true) {
3955                                 var ch = get_char ();
3956                                 switch (ch) {
3957                                 case '"':
3958                                         if (quoted && peek_char () == '"') {
3959                                                 get_char ();
3960                                                 break;
3961                                         }
3962
3963                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3964                                         return Token.INTERPOLATED_STRING_END;
3965                                 case '{':
3966                                         if (peek_char () == '{') {
3967                                                 value_builder [pos++] = (char)ch;
3968                                                 get_char ();
3969                                                 break;
3970                                         }
3971
3972                                         ++parsing_string_interpolation;
3973                                         if (quoted) {
3974                                                 if (parsing_string_interpolation_quoted == null)
3975                                                         parsing_string_interpolation_quoted = new Stack<bool> ();
3976                                         }
3977
3978                                         if (parsing_string_interpolation_quoted != null) {
3979                                                 parsing_string_interpolation_quoted.Push (quoted);
3980                                         }
3981
3982                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3983                                         return Token.INTERPOLATED_STRING;
3984                                 case '\\':
3985                                         if (quoted)
3986                                                 break;
3987
3988                                         ++col;
3989                                         int surrogate;
3990                                         ch = escape (ch, out surrogate);
3991                                         if (ch == -1)
3992                                                 return Token.ERROR;
3993
3994                                         if (ch == '{' || ch == '}') {
3995                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
3996                                         }
3997
3998                                         if (surrogate != 0) {
3999                                                 if (pos == value_builder.Length)
4000                                                         Array.Resize (ref value_builder, pos * 2);
4001
4002                                                 if (pos == value_builder.Length)
4003                                                         Array.Resize (ref value_builder, pos * 2);
4004
4005                                                 value_builder [pos++] = (char)ch;
4006                                                 ch = surrogate;
4007                                         }
4008
4009                                         break;
4010                                 case -1:
4011                                         return Token.EOF;
4012                                 }
4013
4014                                 ++col;
4015                                 if (pos == value_builder.Length)
4016                                         Array.Resize (ref value_builder, pos * 2);
4017
4018                                 value_builder[pos++] = (char) ch;
4019                         }
4020                 }
4021
4022                 int TokenizeInterpolationFormat ()
4023                 {
4024                         int pos = 0;
4025                         int braces = 0;
4026                         while (true) {
4027                                 var ch = get_char ();
4028                                 switch (ch) {
4029                                 case '{':
4030                                         ++braces;
4031                                         break;
4032                                 case '}':
4033                                         if (braces == 0) {
4034                                                 putback_char = ch;
4035                                                 if (pos == 0) {
4036                                                         Report.Error (8089, Location, "Empty interpolated expression format specifier");
4037                                                 } else if (Array.IndexOf (simple_whitespaces, value_builder [pos - 1]) >= 0) {
4038                                                         Report.Error (8088, Location, "A interpolated expression format specifier may not contain trailing whitespace");
4039                                                 }
4040
4041                                                 val = CreateStringFromBuilder (pos);
4042                                                 return Token.LITERAL;
4043                                         }
4044
4045                                         --braces;
4046                                         break;
4047                                 case '\\':
4048                                         ++col;
4049                                         int surrogate;
4050                                         ch = escape (ch, out surrogate);
4051                                         if (ch == -1)
4052                                                 return Token.ERROR;
4053
4054                                         if (ch == '{' || ch == '}') {
4055                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
4056                                         }
4057
4058                                         if (surrogate != 0) {
4059                                                 if (pos == value_builder.Length)
4060                                                         Array.Resize (ref value_builder, pos * 2);
4061
4062                                                 value_builder [pos++] = (char)ch;
4063                                                 ch = surrogate;
4064                                         }
4065
4066                                         break;
4067                                 case -1:
4068                                         return Token.EOF;
4069                                 }
4070
4071                                 ++col;
4072                                 value_builder[pos++] = (char) ch;
4073                         }
4074                 }
4075
4076                 string CreateStringFromBuilder (int pos)
4077                 {
4078                         if (pos == 0)
4079                                 return string.Empty;
4080                         if (pos <= 4)
4081                                 return InternIdentifier (value_builder, pos);
4082
4083                         return new string (value_builder, 0, pos);
4084                 }
4085
4086                 //
4087                 // Handles one line xml comment
4088                 //
4089                 private void handle_one_line_xml_comment ()
4090                 {
4091                         int c;
4092                         while ((c = peek_char ()) == ' ')
4093                                 get_char (); // skip heading whitespaces.
4094                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
4095                                 xml_comment_buffer.Append ((char) get_char ());
4096                         }
4097                         if (c == '\r' || c == '\n')
4098                                 xml_comment_buffer.Append (Environment.NewLine);
4099                 }
4100
4101                 //
4102                 // Remove heading "*" in Javadoc-like xml documentation.
4103                 //
4104                 private void update_formatted_doc_comment (int current_comment_start)
4105                 {
4106                         int length = xml_comment_buffer.Length - current_comment_start;
4107                         string [] lines = xml_comment_buffer.ToString (
4108                                 current_comment_start,
4109                                 length).Replace ("\r", "").Split ('\n');
4110
4111                         // The first line starts with /**, thus it is not target
4112                         // for the format check.
4113                         for (int i = 1; i < lines.Length; i++) {
4114                                 string s = lines [i];
4115                                 int idx = s.IndexOf ('*');
4116                                 string head = null;
4117                                 if (idx < 0) {
4118                                         if (i < lines.Length - 1)
4119                                                 return;
4120                                         head = s;
4121                                 } else
4122                                         head = s.Substring (0, idx);
4123                                 foreach (char c in head)
4124                                         if (c != ' ')
4125                                                 return;
4126                                 lines [i] = s.Substring (idx + 1);
4127                         }
4128                         xml_comment_buffer.Remove (current_comment_start, length);
4129                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
4130                 }
4131
4132                 //
4133                 // Checks if there was incorrect doc comments and raise
4134                 // warnings.
4135                 //
4136                 public void check_incorrect_doc_comment ()
4137                 {
4138                         if (xml_comment_buffer.Length > 0)
4139                                 WarningMisplacedComment (Location);
4140                 }
4141
4142                 //
4143                 // Consumes the saved xml comment lines (if any)
4144                 // as for current target member or type.
4145                 //
4146                 public string consume_doc_comment ()
4147                 {
4148                         if (xml_comment_buffer.Length > 0) {
4149                                 string ret = xml_comment_buffer.ToString ();
4150                                 reset_doc_comment ();
4151                                 return ret;
4152                         }
4153                         return null;
4154                 }
4155
4156                 void reset_doc_comment ()
4157                 {
4158                         xml_comment_buffer.Length = 0;
4159                 }
4160
4161                 public void cleanup ()
4162                 {
4163                         if (ifstack != null && ifstack.Count >= 1) {
4164                                 int state = ifstack.Pop ();
4165                                 if ((state & REGION) != 0)
4166                                         Report.Error (1038, Location, "#endregion directive expected");
4167                                 else
4168                                         Report.Error (1027, Location, "Expected `#endif' directive");
4169                         }
4170                 }
4171         }
4172
4173         //
4174         // Indicates whether it accepts XML documentation or not.
4175         //
4176         public enum XmlCommentState {
4177                 // comment is allowed in this state.
4178                 Allowed,
4179                 // comment is not allowed in this state.
4180                 NotAllowed,
4181                 // once comments appeared when it is NotAllowed, then the
4182                 // state is changed to it, until the state is changed to
4183                 // .Allowed.
4184                 Error
4185         }
4186 }
4187