mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20 using System.Collections;
  21
  22 namespace Mono.CSharp
  23 {
  24         //
  25         // This class has to be used by parser only, it reuses token
  26         // details once a file is parsed
  27         //
  28         public class LocatedToken
  29         {
  30                 public int row, column;
  31                 public string value;
  32                 public SourceFile file;
  33
  34                 public LocatedToken ()
  35                 {
  36                 }
  37
  38                 public LocatedToken (string value, Location loc)
  39                 {
  40                         this.value = value;
  41                         file = loc.SourceFile;
  42                         row = loc.Row;
  43                         column = loc.Column;
  44                 }
  45
  46                 public override string ToString ()
  47                 {
  48                         return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  49                 }
  50
  51                 public Location Location
  52                 {
  53                         get { return new Location (file, row, column); }
  54                 }
  55
  56                 public string Value
  57                 {
  58                         get { return value; }
  59                 }
  60         }
  61
  62         /// <summary>
  63         ///    Tokenizer for C# source code.
  64         /// </summary>
  65         public class Tokenizer : yyParser.yyInput
  66         {
  67                 class KeywordEntry<T>
  68                 {
  69                         public readonly T Token;
  70                         public KeywordEntry<T> Next;
  71                         public readonly char[] Value;
  72
  73                         public KeywordEntry (string value, T token)
  74                         {
  75                                 this.Value = value.ToCharArray ();
  76                                 this.Token = token;
  77                         }
  78                 }
  79
  80                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  81                 {
  82                         readonly int length;
  83
  84                         public IdentifiersComparer (int length)
  85                         {
  86                                 this.length = length;
  87                         }
  88
  89                         public bool Equals (char[] x, char[] y)
  90                         {
  91                                 for (int i = 0; i < length; ++i)
  92                                         if (x [i] != y [i])
  93                                                 return false;
  94
  95                                 return true;
  96                         }
  97
  98                         public int GetHashCode (char[] obj)
  99                         {
 100                                 int h = 0;
 101                                 for (int i = 0; i < length; ++i)
 102                                         h = (h << 5) - h + obj [i];
 103
 104                                 return h;
 105                         }
 106                 }
 107
 108                 public class LocatedTokenBuffer
 109                 {
 110                         readonly LocatedToken[] buffer;
 111                         public int pos;
 112
 113                         public LocatedTokenBuffer ()
 114                         {
 115                                 buffer = new LocatedToken[0];
 116                         }
 117
 118                         public LocatedTokenBuffer (LocatedToken[] buffer)
 119                         {
 120                                 this.buffer = buffer ?? new LocatedToken[0];
 121                         }
 122
 123                         public LocatedToken Create (SourceFile file, int row, int column)
 124                         {
 125                                 return Create (null, file, row, column);
 126                         }
 127
 128                         public LocatedToken Create (string value, SourceFile file, int row, int column)
 129                         {
 130                                 //
 131                                 // TODO: I am not very happy about the logic but it's the best
 132                                 // what I could come up with for now.
 133                                 // Ideally we should be using just tiny buffer (256 elements) which
 134                                 // is enough to hold all details for currect stack and recycle elements
 135                                 // poped from the stack but there is a trick needed to recycle
 136                                 // them properly.
 137                                 //
 138                                 LocatedToken entry;
 139                                 if (pos >= buffer.Length) {
 140                                         entry = new LocatedToken ();
 141                                 } else {
 142                                         entry = buffer[pos];
 143                                         if (entry == null) {
 144                                                 entry = new LocatedToken ();
 145                                                 buffer[pos] = entry;
 146                                         }
 147
 148                                         ++pos;
 149                                 }
 150                                 entry.value = value;
 151                                 entry.file = file;
 152                                 entry.row = row;
 153                                 entry.column = column;
 154                                 return entry;
 155                         }
 156
 157                         //
 158                         // Used for token not required by expression evaluator
 159                         //
 160                         [Conditional ("FULL_AST")]
 161                         public void CreateOptional (SourceFile file, int row, int col, ref object token)
 162                         {
 163                                 token = Create (file, row, col);
 164                         }
 165                 }
 166
 167                 public enum PreprocessorDirective
 168                 {
 169                         Invalid = 0,
 170
 171                         Region = 1,
 172                         Endregion = 2,
 173                         If = 3 | RequiresArgument,
 174                         Endif = 4,
 175                         Elif = 5 | RequiresArgument,
 176                         Else = 6,
 177                         Define = 7 | RequiresArgument,
 178                         Undef = 8 | RequiresArgument,
 179                         Error = 9,
 180                         Warning = 10,
 181                         Pragma = 11 | CustomArgumentsParsing,
 182                         Line = 12 | CustomArgumentsParsing,
 183
 184                         CustomArgumentsParsing = 1 << 10,
 185                         RequiresArgument = 1 << 11
 186                 }
 187
 188                 readonly SeekableStreamReader reader;
 189                 readonly CompilationSourceFile source_file;
 190                 readonly CompilerContext context;
 191                 readonly Report Report;
 192
 193
 194                 SourceFile current_source;
 195                 Location hidden_block_start;
 196                 int ref_line = 1;
 197                 int line = 1;
 198                 int col = 0;
 199                 int previous_col;
 200                 int current_token;
 201                 readonly int tab_size;
 202                 bool handle_get_set = false;
 203                 bool handle_remove_add = false;
 204                 bool handle_where;
 205                 bool lambda_arguments_parsing;
 206                 List<Location> escaped_identifiers;
 207                 int parsing_generic_less_than;
 208                 readonly bool doc_processing;
 209                 readonly LocatedTokenBuffer ltb;
 210
 211                 //
 212                 // Used mainly for parser optimizations. Some expressions for instance
 213                 // can appear only in block (including initializer, base initializer)
 214                 // scope only
 215                 //
 216                 public int parsing_block;
 217                 internal bool query_parsing;
 218
 219                 //
 220                 // When parsing type only, useful for ambiguous nullable types
 221                 //
 222                 public int parsing_type;
 223
 224                 //
 225                 // Set when parsing generic declaration (type or method header)
 226                 //
 227                 public bool parsing_generic_declaration;
 228                 public bool parsing_generic_declaration_doc;
 229
 230                 //
 231                 // The value indicates that we have not reach any declaration or
 232                 // namespace yet
 233                 //
 234                 public int parsing_declaration;
 235
 236                 public bool parsing_attribute_section;
 237
 238                 public bool parsing_modifiers;
 239
 240                 public bool parsing_catch_when;
 241
 242                 int parsing_string_interpolation;
 243                 int string_interpolation_section;
 244                 Stack<bool> parsing_string_interpolation_quoted;
 245
 246                 public bool parsing_interpolation_format;
 247
 248                 //
 249                 // The special characters to inject on streams to run the unit parser
 250                 // in the special expression mode. Using private characters from
 251                 // Plane Sixteen (U+100000 to U+10FFFD)
 252                 //
 253                 // This character is only tested just before the tokenizer is about to report
 254                 // an error;   So on the regular operation mode, this addition will have no
 255                 // impact on the tokenizer's performance.
 256                 //
 257
 258                 public const int EvalStatementParserCharacter = 0x100000;
 259                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 260                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 261                 public const int DocumentationXref = 0x100003;
 262
 263                 const int UnicodeLS = 0x2028;
 264                 const int UnicodePS = 0x2029;
 265
 266                 //
 267                 // XML documentation buffer. The save point is used to divide
 268                 // comments on types and comments on members.
 269                 //
 270                 StringBuilder xml_comment_buffer;
 271
 272                 //
 273                 // See comment on XmlCommentState enumeration.
 274                 //
 275                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 276
 277                 //
 278                 // Whether tokens have been seen on this line
 279                 //
 280                 bool tokens_seen = false;
 281
 282                 //
 283                 // Set to true once the GENERATE_COMPLETION token has bee
 284                 // returned.   This helps produce one GENERATE_COMPLETION,
 285                 // as many COMPLETE_COMPLETION as necessary to complete the
 286                 // AST tree and one final EOF.
 287                 //
 288                 bool generated;
 289
 290                 //
 291                 // Whether a token has been seen on the file
 292                 // This is needed because `define' is not allowed to be used
 293                 // after a token has been seen.
 294                 //
 295                 bool any_token_seen;
 296
 297                 //
 298                 // Class variables
 299                 //
 300                 static readonly KeywordEntry<int>[][] keywords;
 301                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 302                 static readonly HashSet<string> keyword_strings;
 303                 static readonly NumberStyles styles;
 304                 static readonly NumberFormatInfo csharp_format_info;
 305
 306                 // Pragma arguments
 307                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 308                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 309                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 310                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 311                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 312                 static readonly char[] line_default = "default".ToCharArray ();
 313
 314                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 315
 316                 public bool PropertyParsing {
 317                         get { return handle_get_set; }
 318                         set { handle_get_set = value; }
 319                 }
 320
 321                 public bool EventParsing {
 322                         get { return handle_remove_add; }
 323                         set { handle_remove_add = value; }
 324                 }
 325
 326                 public bool ConstraintsParsing {
 327                         get { return handle_where; }
 328                         set { handle_where = value; }
 329                 }
 330
 331                 public XmlCommentState doc_state {
 332                         get { return xml_doc_state; }
 333                         set {
 334                                 if (value == XmlCommentState.Allowed) {
 335                                         check_incorrect_doc_comment ();
 336                                         reset_doc_comment ();
 337                                 }
 338                                 xml_doc_state = value;
 339                         }
 340                 }
 341
 342                 //
 343                 // This is used to trigger completion generation on the parser
 344                 public bool CompleteOnEOF;
 345
 346                 void AddEscapedIdentifier (Location loc)
 347                 {
 348                         if (escaped_identifiers == null)
 349                                 escaped_identifiers = new List<Location> ();
 350
 351                         escaped_identifiers.Add (loc);
 352                 }
 353
 354                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 355                 {
 356                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 357                 }
 358
 359                 //
 360                 // Values for the associated token returned
 361                 //
 362                 internal int putback_char;      // Used by repl only
 363                 object val;
 364
 365                 //
 366                 // Pre-processor
 367                 //
 368                 const int TAKING        = 1;
 369                 const int ELSE_SEEN     = 4;
 370                 const int PARENT_TAKING = 8;
 371                 const int REGION        = 16;
 372
 373                 //
 374                 // pre-processor if stack state:
 375                 //
 376                 Stack<int> ifstack;
 377
 378                 public const int MaxIdentifierLength = 512;
 379                 public const int MaxNumberLength = 512;
 380
 381                 readonly char[] id_builder;
 382                 readonly Dictionary<char[], string>[] identifiers;
 383                 readonly char[] number_builder;
 384                 int number_pos;
 385
 386                 char[] value_builder = new char[64];
 387
 388                 public int Line {
 389                         get {
 390                                 return ref_line;
 391                         }
 392                 }
 393
 394                 //
 395                 // This is used when the tokenizer needs to save
 396                 // the current position as it needs to do some parsing
 397                 // on its own to deamiguate a token in behalf of the
 398                 // parser.
 399                 //
 400                 Stack<Position> position_stack = new Stack<Position> (2);
 401
 402                 class Position {
 403                         public int position;
 404                         public int line;
 405                         public int ref_line;
 406                         public int col;
 407                         public Location hidden;
 408                         public int putback_char;
 409                         public int previous_col;
 410                         public Stack<int> ifstack;
 411                         public int parsing_generic_less_than;
 412                         public int current_token;
 413                         public object val;
 414                         public int parsing_string_interpolation;
 415                         public int string_interpolation_section;
 416                         public Stack<bool> parsing_string_interpolation_quoted;
 417
 418                         public Position (Tokenizer t)
 419                         {
 420                                 position = t.reader.Position;
 421                                 line = t.line;
 422                                 ref_line = t.ref_line;
 423                                 col = t.col;
 424                                 hidden = t.hidden_block_start;
 425                                 putback_char = t.putback_char;
 426                                 previous_col = t.previous_col;
 427                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 428                                         // There is no simple way to clone Stack<T> all
 429                                         // methods reverse the order
 430                                         var clone = t.ifstack.ToArray ();
 431                                         Array.Reverse (clone);
 432                                         ifstack = new Stack<int> (clone);
 433                                 }
 434                                 parsing_generic_less_than = t.parsing_generic_less_than;
 435                                 string_interpolation_section = t.string_interpolation_section;
 436                                 current_token = t.current_token;
 437                                 val = t.val;
 438                                 parsing_string_interpolation = t.parsing_string_interpolation;
 439                                 string_interpolation_section = t.string_interpolation_section;
 440                                 if (t.parsing_string_interpolation_quoted != null && t.parsing_string_interpolation_quoted.Count != 0) {
 441                                         var clone = t.parsing_string_interpolation_quoted.ToArray ();
 442                                         Array.Reverse (clone);
 443                                         parsing_string_interpolation_quoted = new Stack<bool> (clone);
 444                                 }
 445                         }
 446                 }
 447
 448                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 449                 {
 450                         this.source_file = file;
 451                         this.context = file.Compiler;
 452                         this.current_source = file.SourceFile;
 453                         this.identifiers = session.Identifiers;
 454                         this.id_builder = session.IDBuilder;
 455                         this.number_builder = session.NumberBuilder;
 456                         this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 457                         this.Report = report;
 458
 459                         reader = input;
 460
 461                         putback_char = -1;
 462
 463                         xml_comment_buffer = new StringBuilder ();
 464                         doc_processing = context.Settings.DocumentationFile != null;
 465
 466                         tab_size = context.Settings.TabSize;
 467                 }
 468
 469                 public void PushPosition ()
 470                 {
 471                         position_stack.Push (new Position (this));
 472                 }
 473
 474                 public void PopPosition ()
 475                 {
 476                         Position p = position_stack.Pop ();
 477
 478                         reader.Position = p.position;
 479                         ref_line = p.ref_line;
 480                         line = p.line;
 481                         col = p.col;
 482                         hidden_block_start = p.hidden;
 483                         putback_char = p.putback_char;
 484                         previous_col = p.previous_col;
 485                         ifstack = p.ifstack;
 486                         parsing_generic_less_than = p.parsing_generic_less_than;
 487                         parsing_string_interpolation = p.parsing_string_interpolation;
 488                         parsing_string_interpolation_quoted = p.parsing_string_interpolation_quoted;
 489                         current_token = p.current_token;
 490                         val = p.val;
 491                 }
 492
 493                 // Do not reset the position, ignore it.
 494                 public void DiscardPosition ()
 495                 {
 496                         position_stack.Pop ();
 497                 }
 498
 499                 static void AddKeyword (string kw, int token)
 500                 {
 501                         keyword_strings.Add (kw);
 502
 503                         AddKeyword (keywords, kw, token);
 504                 }
 505
 506                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 507                 {
 508                         AddKeyword (keywords_preprocessor, kw, directive);
 509                 }
 510
 511                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 512                 {
 513                         int length = kw.Length;
 514                         if (keywords[length] == null) {
 515                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 516                         }
 517
 518                         int char_index = kw[0] - '_';
 519                         var kwe = keywords[length][char_index];
 520                         if (kwe == null) {
 521                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 522                                 return;
 523                         }
 524
 525                         while (kwe.Next != null) {
 526                                 kwe = kwe.Next;
 527                         }
 528
 529                         kwe.Next = new KeywordEntry<T> (kw, token);
 530                 }
 531
 532                 //
 533                 // Class initializer
 534                 //
 535                 static Tokenizer ()
 536                 {
 537                         keyword_strings = new HashSet<string> ();
 538
 539                         // 11 is the length of the longest keyword for now
 540                         keywords = new KeywordEntry<int>[11][];
 541
 542                         AddKeyword ("__arglist", Token.ARGLIST);
 543                         AddKeyword ("__makeref", Token.MAKEREF);
 544                         AddKeyword ("__reftype", Token.REFTYPE);
 545                         AddKeyword ("__refvalue", Token.REFVALUE);
 546                         AddKeyword ("abstract", Token.ABSTRACT);
 547                         AddKeyword ("as", Token.AS);
 548                         AddKeyword ("add", Token.ADD);
 549                         AddKeyword ("base", Token.BASE);
 550                         AddKeyword ("bool", Token.BOOL);
 551                         AddKeyword ("break", Token.BREAK);
 552                         AddKeyword ("byte", Token.BYTE);
 553                         AddKeyword ("case", Token.CASE);
 554                         AddKeyword ("catch", Token.CATCH);
 555                         AddKeyword ("char", Token.CHAR);
 556                         AddKeyword ("checked", Token.CHECKED);
 557                         AddKeyword ("class", Token.CLASS);
 558                         AddKeyword ("const", Token.CONST);
 559                         AddKeyword ("continue", Token.CONTINUE);
 560                         AddKeyword ("decimal", Token.DECIMAL);
 561                         AddKeyword ("default", Token.DEFAULT);
 562                         AddKeyword ("delegate", Token.DELEGATE);
 563                         AddKeyword ("do", Token.DO);
 564                         AddKeyword ("double", Token.DOUBLE);
 565                         AddKeyword ("else", Token.ELSE);
 566                         AddKeyword ("enum", Token.ENUM);
 567                         AddKeyword ("event", Token.EVENT);
 568                         AddKeyword ("explicit", Token.EXPLICIT);
 569                         AddKeyword ("extern", Token.EXTERN);
 570                         AddKeyword ("false", Token.FALSE);
 571                         AddKeyword ("finally", Token.FINALLY);
 572                         AddKeyword ("fixed", Token.FIXED);
 573                         AddKeyword ("float", Token.FLOAT);
 574                         AddKeyword ("for", Token.FOR);
 575                         AddKeyword ("foreach", Token.FOREACH);
 576                         AddKeyword ("goto", Token.GOTO);
 577                         AddKeyword ("get", Token.GET);
 578                         AddKeyword ("if", Token.IF);
 579                         AddKeyword ("implicit", Token.IMPLICIT);
 580                         AddKeyword ("in", Token.IN);
 581                         AddKeyword ("int", Token.INT);
 582                         AddKeyword ("interface", Token.INTERFACE);
 583                         AddKeyword ("internal", Token.INTERNAL);
 584                         AddKeyword ("is", Token.IS);
 585                         AddKeyword ("lock", Token.LOCK);
 586                         AddKeyword ("long", Token.LONG);
 587                         AddKeyword ("namespace", Token.NAMESPACE);
 588                         AddKeyword ("new", Token.NEW);
 589                         AddKeyword ("null", Token.NULL);
 590                         AddKeyword ("object", Token.OBJECT);
 591                         AddKeyword ("operator", Token.OPERATOR);
 592                         AddKeyword ("out", Token.OUT);
 593                         AddKeyword ("override", Token.OVERRIDE);
 594                         AddKeyword ("params", Token.PARAMS);
 595                         AddKeyword ("private", Token.PRIVATE);
 596                         AddKeyword ("protected", Token.PROTECTED);
 597                         AddKeyword ("public", Token.PUBLIC);
 598                         AddKeyword ("readonly", Token.READONLY);
 599                         AddKeyword ("ref", Token.REF);
 600                         AddKeyword ("remove", Token.REMOVE);
 601                         AddKeyword ("return", Token.RETURN);
 602                         AddKeyword ("sbyte", Token.SBYTE);
 603                         AddKeyword ("sealed", Token.SEALED);
 604                         AddKeyword ("set", Token.SET);
 605                         AddKeyword ("short", Token.SHORT);
 606                         AddKeyword ("sizeof", Token.SIZEOF);
 607                         AddKeyword ("stackalloc", Token.STACKALLOC);
 608                         AddKeyword ("static", Token.STATIC);
 609                         AddKeyword ("string", Token.STRING);
 610                         AddKeyword ("struct", Token.STRUCT);
 611                         AddKeyword ("switch", Token.SWITCH);
 612                         AddKeyword ("this", Token.THIS);
 613                         AddKeyword ("throw", Token.THROW);
 614                         AddKeyword ("true", Token.TRUE);
 615                         AddKeyword ("try", Token.TRY);
 616                         AddKeyword ("typeof", Token.TYPEOF);
 617                         AddKeyword ("uint", Token.UINT);
 618                         AddKeyword ("ulong", Token.ULONG);
 619                         AddKeyword ("unchecked", Token.UNCHECKED);
 620                         AddKeyword ("unsafe", Token.UNSAFE);
 621                         AddKeyword ("ushort", Token.USHORT);
 622                         AddKeyword ("using", Token.USING);
 623                         AddKeyword ("virtual", Token.VIRTUAL);
 624                         AddKeyword ("void", Token.VOID);
 625                         AddKeyword ("volatile", Token.VOLATILE);
 626                         AddKeyword ("while", Token.WHILE);
 627                         AddKeyword ("partial", Token.PARTIAL);
 628                         AddKeyword ("where", Token.WHERE);
 629
 630                         // LINQ keywords
 631                         AddKeyword ("from", Token.FROM);
 632                         AddKeyword ("join", Token.JOIN);
 633                         AddKeyword ("on", Token.ON);
 634                         AddKeyword ("equals", Token.EQUALS);
 635                         AddKeyword ("select", Token.SELECT);
 636                         AddKeyword ("group", Token.GROUP);
 637                         AddKeyword ("by", Token.BY);
 638                         AddKeyword ("let", Token.LET);
 639                         AddKeyword ("orderby", Token.ORDERBY);
 640                         AddKeyword ("ascending", Token.ASCENDING);
 641                         AddKeyword ("descending", Token.DESCENDING);
 642                         AddKeyword ("into", Token.INTO);
 643
 644                         // Contextual async keywords
 645                         AddKeyword ("async", Token.ASYNC);
 646                         AddKeyword ("await", Token.AWAIT);
 647
 648                         // Contextual filter catch keyword
 649                         AddKeyword ("when", Token.WHEN);
 650
 651                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 652
 653                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 654                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 655                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 656                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 657                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 658                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 659                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 660                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 661                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 662                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 663                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 664                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 665
 666                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 667                         styles = NumberStyles.Float;
 668                 }
 669
 670                 int GetKeyword (char[] id, int id_len)
 671                 {
 672                         //
 673                         // Keywords are stored in an array of arrays grouped by their
 674                         // length and then by the first character
 675                         //
 676                         if (id_len >= keywords.Length || keywords [id_len] == null)
 677                                 return -1;
 678
 679                         int first_index = id [0] - '_';
 680                         if (first_index > 'z' - '_')
 681                                 return -1;
 682
 683                         var kwe = keywords [id_len] [first_index];
 684                         if (kwe == null)
 685                                 return -1;
 686
 687                         int res;
 688                         do {
 689                                 res = kwe.Token;
 690                                 for (int i = 1; i < id_len; ++i) {
 691                                         if (id [i] != kwe.Value [i]) {
 692                                                 res = 0;
 693                                                 kwe = kwe.Next;
 694                                                 break;
 695                                         }
 696                                 }
 697                         } while (res == 0 && kwe != null);
 698
 699                         if (res == 0)
 700                                 return -1;
 701
 702                         int next_token;
 703                         switch (res) {
 704                         case Token.GET:
 705                         case Token.SET:
 706                                 if (!handle_get_set)
 707                                         res = -1;
 708                                 break;
 709                         case Token.REMOVE:
 710                         case Token.ADD:
 711                                 if (!handle_remove_add)
 712                                         res = -1;
 713                                 break;
 714                         case Token.EXTERN:
 715                                 if (parsing_declaration == 0)
 716                                         res = Token.EXTERN_ALIAS;
 717                                 break;
 718                         case Token.DEFAULT:
 719                                 if (peek_token () == Token.COLON) {
 720                                         token ();
 721                                         res = Token.DEFAULT_COLON;
 722                                 }
 723                                 break;
 724                         case Token.WHEN:
 725                                 if (current_token != Token.CATCH && !parsing_catch_when)
 726                                         res = -1;
 727                                 break;
 728                         case Token.WHERE:
 729                                 if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 730                                         res = -1;
 731                                 break;
 732                         case Token.FROM:
 733                                 //
 734                                 // A query expression is any expression that starts with `from identifier'
 735                                 // followed by any token except ; , =
 736                                 //
 737                                 if (!query_parsing) {
 738                                         if (lambda_arguments_parsing || parsing_block == 0) {
 739                                                 res = -1;
 740                                                 break;
 741                                         }
 742
 743                                         PushPosition ();
 744                                         // HACK: to disable generics micro-parser, because PushPosition does not
 745                                         // store identifiers array
 746                                         parsing_generic_less_than = 1;
 747                                         switch (xtoken ()) {
 748                                         case Token.IDENTIFIER:
 749                                         case Token.INT:
 750                                         case Token.BOOL:
 751                                         case Token.BYTE:
 752                                         case Token.CHAR:
 753                                         case Token.DECIMAL:
 754                                         case Token.DOUBLE:
 755                                         case Token.FLOAT:
 756                                         case Token.LONG:
 757                                         case Token.OBJECT:
 758                                         case Token.STRING:
 759                                         case Token.UINT:
 760                                         case Token.ULONG:
 761                                                 next_token = xtoken ();
 762                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 763                                                         goto default;
 764
 765                                                 res = Token.FROM_FIRST;
 766                                                 query_parsing = true;
 767                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 768                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 769                                                 break;
 770                                         case Token.VOID:
 771                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 772                                                 break;
 773                                         default:
 774                                                 PopPosition ();
 775                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 776                                                 // which has been overwritten before we grabbed the identifier
 777                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 778                                                 return -1;
 779                                         }
 780                                         PopPosition ();
 781                                 }
 782                                 break;
 783                         case Token.JOIN:
 784                         case Token.ON:
 785                         case Token.EQUALS:
 786                         case Token.SELECT:
 787                         case Token.GROUP:
 788                         case Token.BY:
 789                         case Token.LET:
 790                         case Token.ORDERBY:
 791                         case Token.ASCENDING:
 792                         case Token.DESCENDING:
 793                         case Token.INTO:
 794                                 if (!query_parsing)
 795                                         res = -1;
 796                                 break;
 797
 798                         case Token.USING:
 799                         case Token.NAMESPACE:
 800                                 // TODO: some explanation needed
 801                                 check_incorrect_doc_comment ();
 802                                 parsing_modifiers = false;
 803                                 break;
 804
 805                         case Token.PARTIAL:
 806                                 if (parsing_block > 0) {
 807                                         res = -1;
 808                                         break;
 809                                 }
 810
 811                                 // Save current position and parse next token.
 812                                 PushPosition ();
 813
 814                                 next_token = token ();
 815                                 bool ok = (next_token == Token.CLASS) ||
 816                                         (next_token == Token.STRUCT) ||
 817                                         (next_token == Token.INTERFACE) ||
 818                                         (next_token == Token.VOID);
 819
 820                                 PopPosition ();
 821
 822                                 if (ok) {
 823                                         if (next_token == Token.VOID) {
 824                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 825                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 826                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 827                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 828
 829                                         return res;
 830                                 }
 831
 832                                 if (next_token < Token.LAST_KEYWORD) {
 833                                         Report.Error (267, Location,
 834                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 835                                         return token ();
 836                                 }
 837
 838                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 839                                 // which has been overwritten before we grabbed the identifier
 840                                 id_builder[0] = 'p';
 841                                 id_builder[1] = 'a';
 842                                 id_builder[2] = 'r';
 843                                 id_builder[3] = 't';
 844                                 id_builder[4] = 'i';
 845                                 id_builder[5] = 'a';
 846                                 id_builder[6] = 'l';
 847                                 res = -1;
 848                                 break;
 849
 850                         case Token.ASYNC:
 851                                 if (parsing_modifiers) {
 852                                         //
 853                                         // Skip attributes section or constructor called async
 854                                         //
 855                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 856                                                 res = -1;
 857                                         } else {
 858                                                 // async is keyword
 859                                         }
 860                                 } else if (parsing_block > 0) {
 861                                         switch (peek_token ()) {
 862                                         case Token.DELEGATE:
 863                                         case Token.OPEN_PARENS_LAMBDA:
 864                                                 // async is keyword
 865                                                 break;
 866                                         case Token.IDENTIFIER:
 867                                                 PushPosition ();
 868                                                 xtoken ();
 869                                                 if (xtoken () != Token.ARROW) {
 870                                                         PopPosition ();
 871                                                         goto default;
 872                                                 }
 873
 874                                                 PopPosition ();
 875                                                 break;
 876                                         default:
 877                                                 // peek_token could overwrite id_buffer
 878                                                 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 879                                                 res = -1;
 880                                                 break;
 881                                         }
 882                                 } else {
 883                                         res = -1;
 884                                 }
 885
 886                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 887                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 888                                 }
 889
 890                                 break;
 891
 892                         case Token.AWAIT:
 893                                 if (parsing_block == 0)
 894                                         res = -1;
 895
 896                                 break;
 897                         case Token.THROW:
 898                                 switch (current_token) {
 899                                 case Token.ARROW:
 900                                 case Token.OP_COALESCING:
 901                                 case Token.INTERR:
 902                                         res = Token.THROW_EXPR;
 903                                         break;
 904                                 }
 905
 906                                 break;
 907                         }
 908
 909
 910                         return res;
 911                 }
 912
 913                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 914                 {
 915                         //
 916                         // Keywords are stored in an array of arrays grouped by their
 917                         // length and then by the first character
 918                         //
 919                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 920                                 return PreprocessorDirective.Invalid;
 921
 922                         int first_index = id[0] - '_';
 923                         if (first_index > 'z' - '_')
 924                                 return PreprocessorDirective.Invalid;
 925
 926                         var kwe = keywords_preprocessor[id_len][first_index];
 927                         if (kwe == null)
 928                                 return PreprocessorDirective.Invalid;
 929
 930                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 931                         do {
 932                                 res = kwe.Token;
 933                                 for (int i = 1; i < id_len; ++i) {
 934                                         if (id[i] != kwe.Value[i]) {
 935                                                 res = 0;
 936                                                 kwe = kwe.Next;
 937                                                 break;
 938                                         }
 939                                 }
 940                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 941
 942                         return res;
 943                 }
 944
 945                 public Location Location {
 946                         get {
 947                                 return new Location (current_source, ref_line, col);
 948                         }
 949                 }
 950
 951                 static bool is_identifier_start_character (int c)
 952                 {
 953                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')
 954                                 return true;
 955
 956                         if (c < 0x80)
 957                                 return false;
 958
 959                         return is_identifier_start_character_slow_part ((char) c);
 960                 }
 961
 962                 static bool is_identifier_part_character (char c)
 963                 {
 964                         if (c >= 'a' && c <= 'z')
 965                                 return true;
 966
 967                         if (c >= 'A' && c <= 'Z')
 968                                 return true;
 969
 970                         if (c == '_' || (c >= '0' && c <= '9'))
 971                                 return true;
 972
 973                         if (c < 0x80)
 974                                 return false;
 975
 976                         return is_identifier_part_character_slow_part (c);
 977                 }
 978
 979                 static bool is_identifier_start_character_slow_part (char c)
 980                 {
 981                         switch (Char.GetUnicodeCategory (c)) {
 982                         case UnicodeCategory.LetterNumber:
 983                         case UnicodeCategory.UppercaseLetter:
 984                         case UnicodeCategory.LowercaseLetter:
 985                         case UnicodeCategory.TitlecaseLetter:
 986                         case UnicodeCategory.ModifierLetter:
 987                         case UnicodeCategory.OtherLetter:
 988                                 return true;
 989                         }
 990                         return false;
 991                 }
 992
 993                 static bool is_identifier_part_character_slow_part (char c)
 994                 {
 995                         switch (Char.GetUnicodeCategory (c)) {
 996                         // connecting-character:  A Unicode character of the class Pc
 997                         case UnicodeCategory.ConnectorPunctuation:
 998
 999                         // combining-character: A Unicode character of classes Mn or Mc
1000                         case UnicodeCategory.NonSpacingMark:
1001                         case UnicodeCategory.SpacingCombiningMark:
1002
1003                         // decimal-digit-character: A Unicode character of the class Nd
1004                         case UnicodeCategory.DecimalDigitNumber:
1005
1006                         // plus is_identifier_start_character_slow_part
1007                         case UnicodeCategory.LetterNumber:
1008                         case UnicodeCategory.UppercaseLetter:
1009                         case UnicodeCategory.LowercaseLetter:
1010                         case UnicodeCategory.TitlecaseLetter:
1011                         case UnicodeCategory.ModifierLetter:
1012                         case UnicodeCategory.OtherLetter:
1013                                 return true;
1014
1015                         // formatting-character: A Unicode character of the class Cf
1016                         case UnicodeCategory.Format:
1017                                 // csc bug compatibility which recognizes it as a whitespace
1018                                 return c != 0xFEFF;
1019                         }
1020
1021                         return false;
1022                 }
1023
1024                 public static bool IsKeyword (string s)
1025                 {
1026                         return keyword_strings.Contains (s);
1027                 }
1028
1029                 //
1030                 // Open parens micro parser
1031                 //
1032                 int TokenizeOpenParens ()
1033                 {
1034                         int ptoken;
1035                         current_token = -1;
1036
1037                         int bracket_level = 0;
1038                         bool is_type = false;
1039                         bool can_be_type = false;
1040                         bool at_least_one_comma = false;
1041
1042                         while (true) {
1043                                 ptoken = current_token;
1044                                 token ();
1045
1046                                 switch (current_token) {
1047                                 case Token.CLOSE_PARENS:
1048                                         token ();
1049
1050                                         //
1051                                         // Expression inside parens is lambda, (int i) =>
1052                                         //
1053                                         if (current_token == Token.ARROW)
1054                                                 return Token.OPEN_PARENS_LAMBDA;
1055
1056                                         //
1057                                         // Expression inside parens is deconstruct expression, (a, x.y) = ...
1058                                         //
1059                                         if (current_token == Token.ASSIGN && at_least_one_comma)
1060                                                 return Token.OPEN_PARENS_DECONSTRUCT;
1061
1062                                         //
1063                                         // Expression inside parens is single type, (int[])
1064                                         //
1065                                         if (is_type) {
1066                                                 if (current_token == Token.SEMICOLON)
1067                                                         return Token.OPEN_PARENS;
1068
1069                                                 return Token.OPEN_PARENS_CAST;
1070                                         }
1071
1072                                         //
1073                                         // Expression is possible cast, look at next token, (T)null
1074                                         //
1075                                         if (can_be_type) {
1076                                                 switch (current_token) {
1077                                                 case Token.OPEN_PARENS:
1078                                                 case Token.BANG:
1079                                                 case Token.TILDE:
1080                                                 case Token.IDENTIFIER:
1081                                                 case Token.LITERAL:
1082                                                 case Token.BASE:
1083                                                 case Token.CHECKED:
1084                                                 case Token.DELEGATE:
1085                                                 case Token.FALSE:
1086                                                 case Token.FIXED:
1087                                                 case Token.NEW:
1088                                                 case Token.NULL:
1089                                                 case Token.SIZEOF:
1090                                                 case Token.THIS:
1091                                                 case Token.THROW:
1092                                                 case Token.TRUE:
1093                                                 case Token.TYPEOF:
1094                                                 case Token.UNCHECKED:
1095                                                 case Token.UNSAFE:
1096                                                 case Token.DEFAULT:
1097                                                 case Token.AWAIT:
1098
1099                                                 //
1100                                                 // These can be part of a member access
1101                                                 //
1102                                                 case Token.INT:
1103                                                 case Token.UINT:
1104                                                 case Token.SHORT:
1105                                                 case Token.USHORT:
1106                                                 case Token.LONG:
1107                                                 case Token.ULONG:
1108                                                 case Token.DOUBLE:
1109                                                 case Token.FLOAT:
1110                                                 case Token.CHAR:
1111                                                 case Token.BYTE:
1112                                                 case Token.DECIMAL:
1113                                                 case Token.BOOL:
1114                                                 case Token.STRING:
1115                                                 case Token.SBYTE:
1116                                                         return Token.OPEN_PARENS_CAST;
1117                                                 }
1118                                         }
1119                                         return Token.OPEN_PARENS;
1120
1121                                 case Token.DOT:
1122                                 case Token.DOUBLE_COLON:
1123                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1124                                                 goto default;
1125
1126                                         continue;
1127
1128                                 case Token.IDENTIFIER:
1129                                 case Token.AWAIT:
1130                                         switch (ptoken) {
1131                                         case Token.DOT:
1132                                                 if (bracket_level == 0) {
1133                                                         is_type = false;
1134                                                         can_be_type = true;
1135                                                 }
1136
1137                                                 continue;
1138                                         case Token.OP_GENERICS_LT:
1139                                         case Token.COMMA:
1140                                         case Token.DOUBLE_COLON:
1141                                         case -1:
1142                                                 if (bracket_level == 0)
1143                                                         can_be_type = true;
1144                                                 continue;
1145                                         default:
1146                                                 can_be_type = is_type = false;
1147                                                 continue;
1148                                         }
1149
1150                                 case Token.OBJECT:
1151                                 case Token.STRING:
1152                                 case Token.BOOL:
1153                                 case Token.DECIMAL:
1154                                 case Token.FLOAT:
1155                                 case Token.DOUBLE:
1156                                 case Token.SBYTE:
1157                                 case Token.BYTE:
1158                                 case Token.SHORT:
1159                                 case Token.USHORT:
1160                                 case Token.INT:
1161                                 case Token.UINT:
1162                                 case Token.LONG:
1163                                 case Token.ULONG:
1164                                 case Token.CHAR:
1165                                 case Token.VOID:
1166                                         if (bracket_level == 0)
1167                                                 is_type = true;
1168                                         continue;
1169
1170                                 case Token.COMMA:
1171                                         if (bracket_level == 0) {
1172                                                 bracket_level = 100;
1173                                                 can_be_type = is_type = false;
1174                                                 at_least_one_comma = true;
1175                                         }
1176                                         continue;
1177
1178                                 case Token.OP_GENERICS_LT:
1179                                 case Token.OPEN_BRACKET:
1180                                         if (bracket_level++ == 0)
1181                                                 is_type = true;
1182                                         continue;
1183
1184                                 case Token.OP_GENERICS_GT:
1185                                 case Token.CLOSE_BRACKET:
1186                                         --bracket_level;
1187                                         continue;
1188
1189                                 case Token.INTERR_NULLABLE:
1190                                 case Token.STAR:
1191                                         if (bracket_level == 0)
1192                                                 is_type = true;
1193                                         continue;
1194
1195                                 case Token.REF:
1196                                 case Token.OUT:
1197                                         can_be_type = is_type = false;
1198                                         continue;
1199
1200                                 default:
1201                                         return Token.OPEN_PARENS;
1202                                 }
1203                         }
1204                 }
1205
1206                 public static bool IsValidIdentifier (string s)
1207                 {
1208                         if (s == null || s.Length == 0)
1209                                 return false;
1210
1211                         if (!is_identifier_start_character (s [0]))
1212                                 return false;
1213
1214                         for (int i = 1; i < s.Length; i ++)
1215                                 if (! is_identifier_part_character (s [i]))
1216                                         return false;
1217
1218                         return true;
1219                 }
1220
1221                 bool parse_less_than (ref int genericDimension)
1222                 {
1223                 start:
1224                         int the_token = token ();
1225                         if (the_token == Token.OPEN_BRACKET) {
1226                                 while (true) {
1227                                         the_token = token ();
1228                                         if (the_token == Token.EOF)
1229                                                 return true;
1230
1231                                         if (the_token == Token.CLOSE_BRACKET)
1232                                                 break;
1233                                 }
1234                                 the_token = token ();
1235                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1236                                 the_token = token ();
1237                         }
1238                         switch (the_token) {
1239                         case Token.IDENTIFIER:
1240                         case Token.OBJECT:
1241                         case Token.STRING:
1242                         case Token.BOOL:
1243                         case Token.DECIMAL:
1244                         case Token.FLOAT:
1245                         case Token.DOUBLE:
1246                         case Token.SBYTE:
1247                         case Token.BYTE:
1248                         case Token.SHORT:
1249                         case Token.USHORT:
1250                         case Token.INT:
1251                         case Token.UINT:
1252                         case Token.LONG:
1253                         case Token.ULONG:
1254                         case Token.CHAR:
1255                         case Token.VOID:
1256                                 break;
1257                         case Token.OP_GENERICS_GT:
1258                                 genericDimension = 1;
1259                                 return true;
1260                         case Token.IN:
1261                         case Token.OUT:
1262                                 return true;
1263                         case Token.COMMA:
1264                                 do {
1265                                         ++genericDimension;
1266                                         the_token = token ();
1267                                 } while (the_token == Token.COMMA);
1268
1269                                 if (the_token == Token.OP_GENERICS_GT) {
1270                                         ++genericDimension;
1271                                         return true;
1272                                 }
1273
1274                                 return false;
1275                         case Token.OPEN_PARENS:
1276                                 if (!parsing_generic_declaration)
1277                                         return false;
1278
1279                                 while (true) {
1280                                         switch (token ()) {
1281                                         case Token.COMMA:
1282                                                 // tuple declaration after <
1283                                                 return true;
1284                                         case Token.OP_GENERICS_GT:
1285                                         case Token.EOF:
1286                                                 return false;
1287                                         }
1288                                 }
1289
1290                         default:
1291                                 return false;
1292                         }
1293                 again:
1294                         the_token = token ();
1295
1296                         if (the_token == Token.OP_GENERICS_GT)
1297                                 return true;
1298                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1299                                 goto start;
1300                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1301                                 goto again;
1302                         else if (the_token == Token.OP_GENERICS_LT) {
1303                                 int unused = 0;
1304                                 if (!parse_less_than (ref unused))
1305                                         return false;
1306                                 goto again;
1307                         } else if (the_token == Token.OPEN_BRACKET) {
1308                         rank_specifiers:
1309                                 the_token = token ();
1310                                 if (the_token == Token.CLOSE_BRACKET)
1311                                         goto again;
1312                                 else if (the_token == Token.COMMA)
1313                                         goto rank_specifiers;
1314                                 return false;
1315                         }
1316
1317                         return false;
1318                 }
1319
1320                 public int peek_token ()
1321                 {
1322                         int the_token;
1323
1324                         PushPosition ();
1325                         the_token = token ();
1326                         PopPosition ();
1327
1328                         return the_token;
1329                 }
1330
1331                 //
1332                 // Tonizes `?' using custom disambiguous rules to return one
1333                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1334                 //
1335                 // Tricky expression looks like:
1336                 //
1337                 // Foo ? a = x ? b : c;
1338                 //
1339                 int TokenizePossibleNullableType ()
1340                 {
1341                         if (parsing_block == 0 || parsing_type > 0)
1342                                 return Token.INTERR_NULLABLE;
1343
1344                         int d = peek_char ();
1345                         if (d == '?') {
1346                                 get_char ();
1347                                 return Token.OP_COALESCING;
1348                         }
1349
1350                         if (d == '.') {
1351                                 d = reader.Peek ();
1352                                 return d >= '0' && d <= '9' ? Token.INTERR : Token.INTERR_OPERATOR;
1353                         }
1354
1355                         if (d != ' ') {
1356                                 if (d == ',' || d == ';' || d == '>')
1357                                         return Token.INTERR_NULLABLE;
1358                                 if (d == '*' || (d >= '0' && d <= '9'))
1359                                         return Token.INTERR;
1360                         }
1361
1362                         PushPosition ();
1363                         current_token = Token.NONE;
1364                         int next_token;
1365                         int parens = 0;
1366                         int generics = 0;
1367                         int brackets = 0;
1368
1369                         var nt = xtoken ();
1370                         switch (nt) {
1371                         case Token.DOT:
1372                         case Token.OPEN_BRACKET_EXPR:
1373                                 next_token = Token.INTERR_OPERATOR;
1374                                 break;
1375                         case Token.LITERAL:
1376                         case Token.TRUE:
1377                         case Token.FALSE:
1378                         case Token.NULL:
1379                         case Token.THIS:
1380                         case Token.NEW:
1381                         case Token.INTERPOLATED_STRING:
1382                         case Token.THROW:
1383                                 next_token = Token.INTERR;
1384                                 break;
1385
1386                         case Token.SEMICOLON:
1387                         case Token.COMMA:
1388                         case Token.CLOSE_PARENS:
1389                         case Token.OPEN_BRACKET:
1390                         case Token.OP_GENERICS_GT:
1391                         case Token.INTERR:
1392                         case Token.OP_COALESCING:
1393                         case Token.COLON:
1394                                 next_token = Token.INTERR_NULLABLE;
1395                                 break;
1396
1397                         case Token.OPEN_PARENS:
1398                         case Token.OPEN_PARENS_CAST:
1399                         case Token.OPEN_PARENS_LAMBDA:
1400                                 next_token = -1;
1401                                 ++parens;
1402                                 break;
1403
1404                         case Token.OP_GENERICS_LT:
1405                         case Token.OP_GENERICS_LT_DECL:
1406                         case Token.GENERIC_DIMENSION:
1407                                 next_token = -1;
1408                                 ++generics;
1409                                 break;
1410
1411                         default:
1412                                 next_token = -1;
1413                                 break;
1414                         }
1415
1416                         if (next_token == -1) {
1417                                 switch (xtoken ()) {
1418                                 case Token.COMMA:
1419                                 case Token.SEMICOLON:
1420                                 case Token.OPEN_BRACE:
1421                                 case Token.IN:
1422                                         next_token = Token.INTERR_NULLABLE;
1423                                         break;
1424
1425                                 case Token.COLON:
1426                                         next_token = Token.INTERR;
1427                                         break;
1428
1429                                 case Token.OPEN_PARENS:
1430                                 case Token.OPEN_PARENS_CAST:
1431                                 case Token.OPEN_PARENS_LAMBDA:
1432                                         ++parens;
1433                                         goto default;
1434
1435                                 case Token.OPEN_BRACKET:
1436                                 case Token.OPEN_BRACKET_EXPR:
1437                                         ++brackets;
1438                                         goto default;
1439
1440                                 case Token.CLOSE_PARENS:
1441                                         --parens;
1442                                         goto default;
1443
1444                                 case Token.OP_GENERICS_LT:
1445                                 case Token.OP_GENERICS_LT_DECL:
1446                                 case Token.GENERIC_DIMENSION:
1447                                         ++generics;
1448                                         goto default;
1449
1450                                 default:
1451                                         int ntoken;
1452                                         int interrs = 1;
1453                                         int colons = 0;
1454                                         int braces = 0;
1455                                         //
1456                                         // All shorcuts failed, do it hard way
1457                                         //
1458                                         while ((ntoken = xtoken ()) != Token.EOF) {
1459                                                 switch (ntoken) {
1460                                                 case Token.OPEN_BRACE:
1461                                                         ++braces;
1462                                                         continue;
1463                                                 case Token.OPEN_PARENS:
1464                                                 case Token.OPEN_PARENS_CAST:
1465                                                 case Token.OPEN_PARENS_LAMBDA:
1466                                                         ++parens;
1467                                                         continue;
1468                                                 case Token.CLOSE_BRACE:
1469                                                         --braces;
1470                                                         continue;
1471                                                 case Token.OP_GENERICS_LT:
1472                                                 case Token.OP_GENERICS_LT_DECL:
1473                                                 case Token.GENERIC_DIMENSION:
1474                                                         ++generics;
1475                                                         continue;
1476                                                 case Token.OPEN_BRACKET:
1477                                                 case Token.OPEN_BRACKET_EXPR:
1478                                                         ++brackets;
1479                                                         continue;
1480                                                 case Token.CLOSE_BRACKET:
1481                                                         --brackets;
1482                                                         continue;
1483                                                 case Token.CLOSE_PARENS:
1484                                                         if (parens > 0) {
1485                                                                 --parens;
1486                                                                 continue;
1487                                                         }
1488
1489                                                         PopPosition ();
1490                                                         return Token.INTERR_NULLABLE;
1491
1492                                                 case Token.OP_GENERICS_GT:
1493                                                         if (generics > 0) {
1494                                                                 --generics;
1495                                                                 continue;
1496                                                         }
1497
1498                                                         PopPosition ();
1499                                                         return Token.INTERR_NULLABLE;
1500                                                 }
1501
1502                                                 if (braces != 0)
1503                                                         continue;
1504
1505                                                 if (ntoken == Token.SEMICOLON)
1506                                                         break;
1507
1508                                                 if (parens != 0)
1509                                                         continue;
1510
1511                                                 if (ntoken == Token.COMMA) {
1512                                                         if (generics != 0 || brackets != 0)
1513                                                                 continue;
1514
1515                                                         PopPosition ();
1516                                                         return Token.INTERR_NULLABLE;
1517                                                 }
1518
1519                                                 if (ntoken == Token.COLON) {
1520                                                         if (++colons == interrs)
1521                                                                 break;
1522                                                         continue;
1523                                                 }
1524
1525                                                 if (ntoken == Token.INTERR) {
1526                                                         ++interrs;
1527                                                         continue;
1528                                                 }
1529                                         }
1530
1531                                         next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1532                                         break;
1533                                 }
1534                         }
1535
1536                         PopPosition ();
1537                         return next_token;
1538                 }
1539
1540                 bool decimal_digits (int c)
1541                 {
1542                         int d;
1543                         bool seen_digits = false;
1544
1545                         if (c != -1){
1546                                 if (number_pos == MaxNumberLength)
1547                                         Error_NumericConstantTooLong ();
1548                                 number_builder [number_pos++] = (char) c;
1549                         }
1550
1551                         //
1552                         // We use peek_char2, because decimal_digits needs to do a
1553                         // 2-character look-ahead (5.ToString for example).
1554                         //
1555                         while ((d = peek_char2 ()) != -1){
1556                                 if (d >= '0' && d <= '9'){
1557                                         if (number_pos == MaxNumberLength)
1558                                                 Error_NumericConstantTooLong ();
1559                                         number_builder [number_pos++] = (char) d;
1560                                         get_char ();
1561                                         seen_digits = true;
1562                                 } else
1563                                         break;
1564                         }
1565
1566                         return seen_digits;
1567                 }
1568
1569                 static bool is_hex (int e)
1570                 {
1571                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1572                 }
1573
1574                 static TypeCode real_type_suffix (int c)
1575                 {
1576                         switch (c){
1577                         case 'F': case 'f':
1578                                 return TypeCode.Single;
1579                         case 'D': case 'd':
1580                                 return TypeCode.Double;
1581                         case 'M': case 'm':
1582                                 return TypeCode.Decimal;
1583                         default:
1584                                 return TypeCode.Empty;
1585                         }
1586                 }
1587
1588                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1589                 {
1590                         bool is_unsigned = false;
1591                         bool is_long = false;
1592
1593                         if (c != -1){
1594                                 bool scanning = true;
1595                                 do {
1596                                         switch (c){
1597                                         case 'U': case 'u':
1598                                                 if (is_unsigned)
1599                                                         scanning = false;
1600                                                 is_unsigned = true;
1601                                                 get_char ();
1602                                                 break;
1603
1604                                         case 'l':
1605                                                 if (!is_unsigned){
1606                                                         //
1607                                                         // if we have not seen anything in between
1608                                                         // report this error
1609                                                         //
1610                                                         Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1611                                                 }
1612
1613                                                 goto case 'L';
1614
1615                                         case 'L':
1616                                                 if (is_long)
1617                                                         scanning = false;
1618                                                 is_long = true;
1619                                                 get_char ();
1620                                                 break;
1621
1622                                         default:
1623                                                 scanning = false;
1624                                                 break;
1625                                         }
1626                                         c = peek_char ();
1627                                 } while (scanning);
1628                         }
1629
1630                         if (is_long && is_unsigned){
1631                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1632                         }
1633
1634                         if (is_unsigned){
1635                                 // uint if possible, or ulong else.
1636
1637                                 if ((ul & 0xffffffff00000000) == 0)
1638                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1639                                 else
1640                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1641                         } else if (is_long){
1642                                 // long if possible, ulong otherwise
1643                                 if ((ul & 0x8000000000000000) != 0)
1644                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1645                                 else
1646                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1647                         } else {
1648                                 // int, uint, long or ulong in that order
1649                                 if ((ul & 0xffffffff00000000) == 0){
1650                                         uint ui = (uint) ul;
1651
1652                                         if ((ui & 0x80000000) != 0)
1653                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1654                                         else
1655                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1656                                 } else {
1657                                         if ((ul & 0x8000000000000000) != 0)
1658                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1659                                         else
1660                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1661                                 }
1662                         }
1663                 }
1664
1665                 //
1666                 // given `c' as the next char in the input decide whether
1667                 // we need to convert to a special type, and then choose
1668                 // the best representation for the integer
1669                 //
1670                 ILiteralConstant adjust_int (int c, Location loc)
1671                 {
1672                         try {
1673                                 if (number_pos > 9){
1674                                         ulong ul = (uint) (number_builder [0] - '0');
1675
1676                                         for (int i = 1; i < number_pos; i++){
1677                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1678                                         }
1679
1680                                         return integer_type_suffix (ul, c, loc);
1681                                 } else {
1682                                         uint ui = (uint) (number_builder [0] - '0');
1683
1684                                         for (int i = 1; i < number_pos; i++){
1685                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1686                                         }
1687
1688                                         return integer_type_suffix (ui, c, loc);
1689                                 }
1690                         } catch (OverflowException) {
1691                                 Error_NumericConstantTooLong ();
1692                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1693                         }
1694                         catch (FormatException) {
1695                                 Report.Error (1013, Location, "Invalid number");
1696                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1697                         }
1698                 }
1699
1700                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1701                 {
1702                         string s = new string (number_builder, 0, number_pos);
1703                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1704
1705                         switch (t){
1706                         case TypeCode.Decimal:
1707                                 try {
1708                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1709                                 } catch (OverflowException) {
1710                                         Report.Error (594, Location, error_details, "decimal");
1711                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1712                                 }
1713                         case TypeCode.Single:
1714                                 try {
1715                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1716                                 } catch (OverflowException) {
1717                                         Report.Error (594, Location, error_details, "float");
1718                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1719                                 }
1720                         default:
1721                                 try {
1722                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1723                                 } catch (OverflowException) {
1724                                         Report.Error (594, loc, error_details, "double");
1725                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1726                                 }
1727                         }
1728                 }
1729
1730                 ILiteralConstant handle_hex (Location loc)
1731                 {
1732                         int d;
1733                         ulong ul;
1734
1735                         get_char ();
1736                         while ((d = peek_char ()) != -1){
1737                                 if (is_hex (d)){
1738                                         number_builder [number_pos++] = (char) d;
1739                                         get_char ();
1740                                 } else
1741                                         break;
1742                         }
1743
1744                         string s = new String (number_builder, 0, number_pos);
1745
1746                         try {
1747                                 if (number_pos <= 8)
1748                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1749                                 else
1750                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1751
1752                                 return integer_type_suffix (ul, peek_char (), loc);
1753                         } catch (OverflowException){
1754                                 Error_NumericConstantTooLong ();
1755                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1756                         }
1757                         catch (FormatException) {
1758                                 Report.Error (1013, Location, "Invalid number");
1759                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1760                         }
1761                 }
1762
1763                 //
1764                 // Invoked if we know we have .digits or digits
1765                 //
1766                 int is_number (int c, bool dotLead)
1767                 {
1768                         ILiteralConstant res;
1769
1770 #if FULL_AST
1771                         int read_start = reader.Position - 1;
1772                         if (dotLead) {
1773                                 //
1774                                 // Caller did peek_char
1775                                 //
1776                                 --read_start;
1777                         }
1778 #endif
1779                         number_pos = 0;
1780                         var loc = Location;
1781
1782                         if (!dotLead){
1783                                 if (c == '0'){
1784                                         int peek = peek_char ();
1785
1786                                         if (peek == 'x' || peek == 'X') {
1787                                                 val = res = handle_hex (loc);
1788 #if FULL_AST
1789                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1790 #endif
1791
1792                                                 return Token.LITERAL;
1793                                         }
1794                                 }
1795                                 decimal_digits (c);
1796                                 c = peek_char ();
1797                         }
1798
1799                         //
1800                         // We need to handle the case of
1801                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1802                         //
1803                         bool is_real = false;
1804                         if (c == '.'){
1805                                 if (!dotLead)
1806                                         get_char ();
1807
1808                                 if (decimal_digits ('.')){
1809                                         is_real = true;
1810                                         c = peek_char ();
1811                                 } else {
1812                                         putback ('.');
1813                                         number_pos--;
1814                                         val = res = adjust_int (-1, loc);
1815
1816 #if FULL_AST
1817                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1818 #endif
1819                                         return Token.LITERAL;
1820                                 }
1821                         }
1822
1823                         if (c == 'e' || c == 'E'){
1824                                 is_real = true;
1825                                 get_char ();
1826                                 if (number_pos == MaxNumberLength)
1827                                         Error_NumericConstantTooLong ();
1828                                 number_builder [number_pos++] = (char) c;
1829                                 c = get_char ();
1830
1831                                 if (c == '+'){
1832                                         if (number_pos == MaxNumberLength)
1833                                                 Error_NumericConstantTooLong ();
1834                                         number_builder [number_pos++] = '+';
1835                                         c = -1;
1836                                 } else if (c == '-') {
1837                                         if (number_pos == MaxNumberLength)
1838                                                 Error_NumericConstantTooLong ();
1839                                         number_builder [number_pos++] = '-';
1840                                         c = -1;
1841                                 } else {
1842                                         if (number_pos == MaxNumberLength)
1843                                                 Error_NumericConstantTooLong ();
1844                                         number_builder [number_pos++] = '+';
1845                                 }
1846
1847                                 decimal_digits (c);
1848                                 c = peek_char ();
1849                         }
1850
1851                         var type = real_type_suffix (c);
1852                         if (type == TypeCode.Empty && !is_real) {
1853                                 res = adjust_int (c, loc);
1854                         } else {
1855                                 is_real = true;
1856
1857                                 if (type != TypeCode.Empty) {
1858                                         get_char ();
1859                                 }
1860
1861                                 res = adjust_real (type, loc);
1862                         }
1863
1864                         val = res;
1865
1866 #if FULL_AST
1867                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1868                         if (chars[chars.Length - 1] == '\r')
1869                                 Array.Resize (ref chars, chars.Length - 1);
1870                         res.ParsedValue = chars;
1871 #endif
1872
1873                         return Token.LITERAL;
1874                 }
1875
1876                 //
1877                 // Accepts exactly count (4 or 8) hex, no more no less
1878                 //
1879                 int getHex (int count, out int surrogate, out bool error)
1880                 {
1881                         int i;
1882                         int total = 0;
1883                         int c;
1884                         int top = count != -1 ? count : 4;
1885
1886                         get_char ();
1887                         error = false;
1888                         surrogate = 0;
1889                         for (i = 0; i < top; i++){
1890                                 c = get_char ();
1891
1892                                 if (c >= '0' && c <= '9')
1893                                         c = (int) c - (int) '0';
1894                                 else if (c >= 'A' && c <= 'F')
1895                                         c = (int) c - (int) 'A' + 10;
1896                                 else if (c >= 'a' && c <= 'f')
1897                                         c = (int) c - (int) 'a' + 10;
1898                                 else {
1899                                         error = true;
1900                                         return 0;
1901                                 }
1902
1903                                 total = (total * 16) + c;
1904                                 if (count == -1){
1905                                         int p = peek_char ();
1906                                         if (p == -1)
1907                                                 break;
1908                                         if (!is_hex ((char)p))
1909                                                 break;
1910                                 }
1911                         }
1912
1913                         if (top == 8) {
1914                                 if (total > 0x0010FFFF) {
1915                                         error = true;
1916                                         return 0;
1917                                 }
1918
1919                                 if (total >= 0x00010000) {
1920                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1921                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1922                                 }
1923                         }
1924
1925                         return total;
1926                 }
1927
1928                 int escape (int c, out int surrogate)
1929                 {
1930                         bool error;
1931                         int d;
1932                         int v;
1933
1934                         d = peek_char ();
1935                         if (c != '\\') {
1936                                 surrogate = 0;
1937                                 return c;
1938                         }
1939
1940                         switch (d){
1941                         case 'a':
1942                                 v = '\a'; break;
1943                         case 'b':
1944                                 v = '\b'; break;
1945                         case 'n':
1946                                 v = '\n'; break;
1947                         case 't':
1948                                 v = '\t'; break;
1949                         case 'v':
1950                                 v = '\v'; break;
1951                         case 'r':
1952                                 v = '\r'; break;
1953                         case '\\':
1954                                 v = '\\'; break;
1955                         case 'f':
1956                                 v = '\f'; break;
1957                         case '0':
1958                                 v = 0; break;
1959                         case '"':
1960                                 v = '"'; break;
1961                         case '\'':
1962                                 v = '\''; break;
1963                         case 'x':
1964                                 v = getHex (-1, out surrogate, out error);
1965                                 if (error)
1966                                         goto default;
1967                                 return v;
1968                         case 'u':
1969                         case 'U':
1970                                 return EscapeUnicode (d, out surrogate);
1971                         default:
1972                                 surrogate = 0;
1973                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1974                                 return d;
1975                         }
1976
1977                         get_char ();
1978                         surrogate = 0;
1979                         return v;
1980                 }
1981
1982                 int EscapeUnicode (int ch, out int surrogate)
1983                 {
1984                         bool error;
1985                         if (ch == 'U') {
1986                                 ch = getHex (8, out surrogate, out error);
1987                         } else {
1988                                 ch = getHex (4, out surrogate, out error);
1989                         }
1990
1991                         if (error)
1992                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1993
1994                         return ch;
1995                 }
1996
1997                 int get_char ()
1998                 {
1999                         int x;
2000                         if (putback_char != -1) {
2001                                 x = putback_char;
2002                                 putback_char = -1;
2003                         } else {
2004                                 x = reader.Read ();
2005                         }
2006
2007                         if (x <= 13) {
2008                                 if (x == '\r') {
2009                                         if (peek_char () == '\n') {
2010                                                 putback_char = -1;
2011                                         }
2012
2013                                         x = '\n';
2014                                         advance_line ();
2015                                 } else if (x == '\n') {
2016                                         advance_line ();
2017                                 } else {
2018                                         col++;
2019                                 }
2020                         } else if (x >= UnicodeLS && x <= UnicodePS) {
2021                                 advance_line ();
2022                         } else {
2023                                 col++;
2024                         }
2025
2026                         return x;
2027                 }
2028
2029                 void advance_line ()
2030                 {
2031                         line++;
2032                         ref_line++;
2033                         previous_col = col;
2034                         col = 0;
2035                 }
2036
2037                 int peek_char ()
2038                 {
2039                         if (putback_char == -1)
2040                                 putback_char = reader.Read ();
2041                         return putback_char;
2042                 }
2043
2044                 int peek_char2 ()
2045                 {
2046                         if (putback_char != -1)
2047                                 return putback_char;
2048                         return reader.Peek ();
2049                 }
2050
2051                 public void putback (int c)
2052                 {
2053                         if (putback_char != -1) {
2054                                 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
2055                         }
2056
2057                         if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
2058                                 // It won't happen though.
2059                                 line--;
2060                                 ref_line--;
2061                                 col = previous_col;
2062                         }
2063                         else
2064                                 col--;
2065                         putback_char = c;
2066                 }
2067
2068                 public bool advance ()
2069                 {
2070                         return peek_char () != -1 || CompleteOnEOF;
2071                 }
2072
2073                 public Object Value {
2074                         get {
2075                                 return val;
2076                         }
2077                 }
2078
2079                 public Object value ()
2080                 {
2081                         return val;
2082                 }
2083
2084                 public int token ()
2085                 {
2086                         current_token = xtoken ();
2087                         return current_token;
2088                 }
2089
2090                 int TokenizePreprocessorKeyword (out int c)
2091                 {
2092                         // skip over white space
2093                         do {
2094                                 c = get_char ();
2095                         } while (c == ' ' || c == '\t');
2096
2097
2098                         int pos = 0;
2099                         while (c != -1 && c >= 'a' && c <= 'z') {
2100                                 id_builder[pos++] = (char) c;
2101                                 c = get_char ();
2102                                 if (c == '\\') {
2103                                         int peek = peek_char ();
2104                                         if (peek == 'U' || peek == 'u') {
2105                                                 int surrogate;
2106                                                 c = EscapeUnicode (c, out surrogate);
2107                                                 if (surrogate != 0) {
2108                                                         if (is_identifier_part_character ((char) c)) {
2109                                                                 id_builder[pos++] = (char) c;
2110                                                         }
2111                                                         c = surrogate;
2112                                                 }
2113                                         }
2114                                 }
2115                         }
2116
2117                         return pos;
2118                 }
2119
2120                 PreprocessorDirective get_cmd_arg (out string arg)
2121                 {
2122                         int c;
2123
2124                         tokens_seen = false;
2125                         arg = "";
2126
2127                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorKeyword (out c));
2128
2129                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
2130                                 return cmd;
2131
2132                         // skip over white space
2133                         while (c == ' ' || c == '\t')
2134                                 c = get_char ();
2135
2136                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
2137                         int pos = 0;
2138
2139                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2140                                 if (c == '\\' && has_identifier_argument >= 0) {
2141                                         if (has_identifier_argument != 0) {
2142                                                 has_identifier_argument = 1;
2143
2144                                                 int peek = peek_char ();
2145                                                 if (peek == 'U' || peek == 'u') {
2146                                                         int surrogate;
2147                                                         c = EscapeUnicode (c, out surrogate);
2148                                                         if (surrogate != 0) {
2149                                                                 if (is_identifier_part_character ((char) c)) {
2150                                                                         if (pos == value_builder.Length)
2151                                                                                 Array.Resize (ref value_builder, pos * 2);
2152
2153                                                                         value_builder[pos++] = (char) c;
2154                                                                 }
2155                                                                 c = surrogate;
2156                                                         }
2157                                                 }
2158                                         } else {
2159                                                 has_identifier_argument = -1;
2160                                         }
2161                                 } else if (c == '/' && peek_char () == '/') {
2162                                         //
2163                                         // Eat single-line comments
2164                                         //
2165                                         get_char ();
2166                                         ReadToEndOfLine ();
2167                                         break;
2168                                 }
2169
2170                                 if (pos == value_builder.Length)
2171                                         Array.Resize (ref value_builder, pos * 2);
2172
2173                                 value_builder[pos++] = (char) c;
2174                                 c = get_char ();
2175                         }
2176
2177                         if (pos != 0) {
2178                                 if (pos > MaxIdentifierLength)
2179                                         arg = new string (value_builder, 0, pos);
2180                                 else
2181                                         arg = InternIdentifier (value_builder, pos);
2182
2183                                 // Eat any trailing whitespaces
2184                                 arg = arg.Trim (simple_whitespaces);
2185                         }
2186
2187                         return cmd;
2188                 }
2189
2190                 //
2191                 // Handles the #line directive
2192                 //
2193                 bool PreProcessLine ()
2194                 {
2195                         Location loc = Location;
2196
2197                         int c;
2198
2199                         int length = TokenizePreprocessorKeyword (out c);
2200                         if (length == line_default.Length) {
2201                                 if (!IsTokenIdentifierEqual (line_default))
2202                                         return false;
2203
2204                                 current_source = source_file.SourceFile;
2205                                 if (!hidden_block_start.IsNull) {
2206                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
2207                                         hidden_block_start = Location.Null;
2208                                 }
2209
2210                                 ref_line = line;
2211                                 return true;
2212                         }
2213
2214                         if (length == line_hidden.Length) {
2215                                 if (!IsTokenIdentifierEqual (line_hidden))
2216                                         return false;
2217
2218                                 if (hidden_block_start.IsNull)
2219                                         hidden_block_start = loc;
2220
2221                                 return true;
2222                         }
2223
2224                         if (length != 0 || c < '0' || c > '9') {
2225                                 //
2226                                 // Eat any remaining characters to continue parsing on next line
2227                                 //
2228                                 ReadToEndOfLine ();
2229                                 return false;
2230                         }
2231
2232                         int new_line = TokenizeNumber (c);
2233                         if (new_line < 1) {
2234                                 //
2235                                 // Eat any remaining characters to continue parsing on next line
2236                                 //
2237                                 ReadToEndOfLine ();
2238                                 return new_line != 0;
2239                         }
2240
2241                         c = get_char ();
2242                         if (c == ' ') {
2243                                 // skip over white space
2244                                 do {
2245                                         c = get_char ();
2246                                 } while (c == ' ' || c == '\t');
2247                         } else if (c == '"') {
2248                                 c = 0;
2249                         }
2250
2251                         if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) {
2252                                 //
2253                                 // Eat any remaining characters to continue parsing on next line
2254                                 //
2255                                 ReadToEndOfLine ();
2256
2257                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2258                                 return true;
2259                         }
2260
2261                         string new_file_name = null;
2262                         if (c == '"') {
2263                                 new_file_name = TokenizeFileName (ref c);
2264
2265                                 // skip over white space
2266                                 while (c == ' ' || c == '\t') {
2267                                         c = get_char ();
2268                                 }
2269                         }
2270
2271                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2272
2273                         } else if (c == '/') {
2274                                 ReadSingleLineComment ();
2275                         } else {
2276                                 //
2277                                 // Eat any remaining characters to continue parsing on next line
2278                                 //
2279                                 ReadToEndOfLine ();
2280
2281                                 Error_EndLineExpected ();
2282                                 return true;
2283                         }
2284
2285                         if (new_file_name != null) {
2286                                 current_source = context.LookupFile (source_file, new_file_name);
2287                                 source_file.AddIncludeFile (current_source);
2288                         }
2289
2290                         if (!hidden_block_start.IsNull) {
2291                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2292                                 hidden_block_start = Location.Null;
2293                         }
2294
2295                         ref_line = new_line;
2296                         return true;
2297                 }
2298
2299                 //
2300                 // Handles #define and #undef
2301                 //
2302                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2303                 {
2304                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2305                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2306                                 return;
2307                         }
2308
2309                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2310                                 Error_EndLineExpected ();
2311                                 return;
2312                         }
2313
2314                         if (!is_identifier_start_character (ident [0]))
2315                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2316
2317                         foreach (char c in ident.Substring (1)){
2318                                 if (!is_identifier_part_character (c)){
2319                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2320                                         return;
2321                                 }
2322                         }
2323
2324                         if (!caller_is_taking)
2325                                 return;
2326
2327                         if (is_define) {
2328                                 //
2329                                 // #define ident
2330                                 //
2331                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2332                                         return;
2333
2334                                 source_file.AddDefine (ident);
2335                         } else {
2336                                 //
2337                                 // #undef ident
2338                                 //
2339                                 source_file.AddUndefine (ident);
2340                         }
2341                 }
2342
2343                 byte read_hex (out bool error)
2344                 {
2345                         int total;
2346                         int c = get_char ();
2347
2348                         if ((c >= '0') && (c <= '9'))
2349                                 total = (int) c - (int) '0';
2350                         else if ((c >= 'A') && (c <= 'F'))
2351                                 total = (int) c - (int) 'A' + 10;
2352                         else if ((c >= 'a') && (c <= 'f'))
2353                                 total = (int) c - (int) 'a' + 10;
2354                         else {
2355                                 error = true;
2356                                 return 0;
2357                         }
2358
2359                         total *= 16;
2360                         c = get_char ();
2361
2362                         if ((c >= '0') && (c <= '9'))
2363                                 total += (int) c - (int) '0';
2364                         else if ((c >= 'A') && (c <= 'F'))
2365                                 total += (int) c - (int) 'A' + 10;
2366                         else if ((c >= 'a') && (c <= 'f'))
2367                                 total += (int) c - (int) 'a' + 10;
2368                         else {
2369                                 error = true;
2370                                 return 0;
2371                         }
2372
2373                         error = false;
2374                         return (byte) total;
2375                 }
2376
2377                 //
2378                 // Parses #pragma checksum
2379                 //
2380                 bool ParsePragmaChecksum ()
2381                 {
2382                         //
2383                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2384                         //
2385                         // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
2386                         //
2387                         int c = get_char ();
2388
2389                         if (c != '"')
2390                                 return false;
2391
2392                         string file_name = TokenizeFileName (ref c);
2393
2394                         // TODO: Any white-spaces count
2395                         if (c != ' ')
2396                                 return false;
2397
2398                         SourceFile file = context.LookupFile (source_file, file_name);
2399
2400                         if (get_char () != '"' || get_char () != '{')
2401                                 return false;
2402
2403                         bool error;
2404                         byte[] guid_bytes = new byte [16];
2405                         int i = 0;
2406
2407                         for (; i < 4; i++) {
2408                                 guid_bytes [i] = read_hex (out error);
2409                                 if (error)
2410                                         return false;
2411                         }
2412
2413                         if (get_char () != '-')
2414                                 return false;
2415
2416                         for (; i < 10; i++) {
2417                                 guid_bytes [i] = read_hex (out error);
2418                                 if (error)
2419                                         return false;
2420
2421                                 guid_bytes [i++] = read_hex (out error);
2422                                 if (error)
2423                                         return false;
2424
2425                                 if (get_char () != '-')
2426                                         return false;
2427                         }
2428
2429                         for (; i < 16; i++) {
2430                                 guid_bytes [i] = read_hex (out error);
2431                                 if (error)
2432                                         return false;
2433                         }
2434
2435                         if (get_char () != '}' || get_char () != '"')
2436                                 return false;
2437
2438                         // TODO: Any white-spaces count
2439                         c = get_char ();
2440                         if (c != ' ')
2441                                 return false;
2442
2443                         if (get_char () != '"')
2444                                 return false;
2445
2446                         // Any length of checksum
2447                         List<byte> checksum_bytes = new List<byte> (16);
2448
2449                         var checksum_location = Location;
2450                         c = peek_char ();
2451                         while (c != '"' && c != -1) {
2452                                 checksum_bytes.Add (read_hex (out error));
2453                                 if (error)
2454                                         return false;
2455
2456                                 c = peek_char ();
2457                         }
2458
2459                         if (c == '/') {
2460                                 ReadSingleLineComment ();
2461                         } else if (get_char () != '"') {
2462                                 return false;
2463                         }
2464
2465                         if (context.Settings.GenerateDebugInfo) {
2466                                 var chsum = checksum_bytes.ToArray ();
2467
2468                                 if (file.HasChecksum) {
2469                                         if (!ArrayComparer.IsEqual (file.Checksum, chsum)) {
2470                                                 // TODO: Report.SymbolRelatedToPreviousError
2471                                                 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name);
2472                                         }
2473                                 }
2474
2475                                 file.SetChecksum (guid_bytes, chsum);
2476                                 current_source.AutoGenerated = true;
2477                         }
2478
2479                         return true;
2480                 }
2481
2482                 bool IsTokenIdentifierEqual (char[] identifier)
2483                 {
2484                         for (int i = 0; i < identifier.Length; ++i) {
2485                                 if (identifier[i] != id_builder[i])
2486                                         return false;
2487                         }
2488
2489                         return true;
2490                 }
2491
2492                 bool ScanClosingInterpolationBrace ()
2493                 {
2494                         PushPosition ();
2495
2496                         bool? res = null;
2497                         int str_quote = 0;
2498                         do {
2499                                 var c = reader.Read ();
2500                                 switch (c) {
2501                                 case '\"':
2502                                         ++str_quote;
2503                                         break;
2504                                 case '\\':
2505                                         // Skip escaped " character
2506                                         c = reader.Read ();
2507                                         if (c == -1)
2508                                                 res = false;
2509                                         break;
2510                                 case -1:
2511                                         res = false;
2512                                         break;
2513                                 case '}':
2514                                         if (str_quote % 2 == 1) {
2515                                                 res = true;
2516                                         }
2517
2518                                         break;
2519                                 }
2520                         } while (res == null);
2521
2522                         PopPosition ();
2523                         return res.Value;
2524                 }
2525
2526                 int TokenizeNumber (int value)
2527                 {
2528                         number_pos = 0;
2529
2530                         decimal_digits (value);
2531                         uint ui = (uint) (number_builder[0] - '0');
2532
2533                         try {
2534                                 for (int i = 1; i < number_pos; i++) {
2535                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2536                                 }
2537
2538                                 return (int) ui;
2539                         } catch (OverflowException) {
2540                                 Error_NumericConstantTooLong ();
2541                                 return -1;
2542                         }
2543                 }
2544
2545                 string TokenizeFileName (ref int c)
2546                 {
2547                         var string_builder = new StringBuilder ();
2548                         while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) {
2549                                 c = get_char ();
2550                                 if (c == '"') {
2551                                         c = get_char ();
2552                                         break;
2553                                 }
2554
2555                                 string_builder.Append ((char) c);
2556                         }
2557
2558                         if (string_builder.Length == 0) {
2559                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2560                         }
2561
2562
2563                         return string_builder.ToString ();
2564                 }
2565
2566                 int TokenizePragmaWarningIdentifier (ref int c, ref bool identifier)
2567                 {
2568                         if ((c >= '0' && c <= '9') || is_identifier_start_character (c)) {
2569                                 int number;
2570
2571                                 if (c >= '0' && c <= '9') {
2572                                         number_pos = 0;
2573                                         number = TokenizeNumber (c);
2574
2575                                         c = get_char ();
2576
2577                                         if (c != ' ' && c != '\t' && c != ',' && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS) {
2578                                                 return ReadPragmaWarningComment (c);
2579                                         }
2580                                 } else {
2581                                         //
2582                                         // LAMESPEC v6: No spec what identifier really is in this context, it seems keywords are allowed too
2583                                         //
2584                                         int pos = 0;
2585                                         number = -1;
2586                                         id_builder [pos++] = (char)c;
2587                                         while (c < MaxIdentifierLength) {
2588                                                 c = reader.Read ();
2589                                                 id_builder [pos] = (char)c;
2590
2591                                                 if (c >= '0' && c <= '9') {
2592                                                         if (pos == 5 && id_builder [0] == 'C' && id_builder [1] == 'S') {
2593                                                                 // Recognize CSXXXX as C# XXXX warning
2594                                                                 number = 0;
2595                                                                 int pow = 1000;
2596                                                                 for (int i = 0; i < 4; ++i) {
2597                                                                         var ch = id_builder [i + 2];
2598                                                                         if (ch < '0' || ch > '9') {
2599                                                                                 number = -1;
2600                                                                                 break;
2601                                                                         }
2602
2603                                                                         number += (ch - '0') * pow;
2604                                                                         pow /= 10;
2605                                                                 }
2606                                                         }
2607                                                 } else if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
2608                                                         advance_line ();
2609                                                         break;
2610                                                 } else if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '_') {
2611                                                         break;
2612                                                 }
2613
2614                                                 ++pos;
2615                                         }
2616
2617                                         if (number < 0) {
2618                                                 identifier = true;
2619                                                 number = pos;
2620                                         }
2621                                 }
2622
2623                                 // skip over white space
2624                                 while (c == ' ' || c == '\t')
2625                                         c = get_char ();
2626
2627                                 if (c == ',') {
2628                                         c = get_char ();
2629                                 }
2630
2631                                 // skip over white space
2632                                 while (c == ' ' || c == '\t')
2633                                         c = get_char ();
2634
2635                                 return number;
2636                         }
2637
2638                         return ReadPragmaWarningComment (c);
2639                 }
2640
2641                 int ReadPragmaWarningComment (int c)
2642                 {
2643                         if (c == '/') {
2644                                 ReadSingleLineComment ();
2645                         } else {
2646                                 Report.Warning (1692, 1, Location, "Invalid number");
2647
2648                                 // Read everything till the end of the line or file
2649                                 ReadToEndOfLine ();
2650                         }
2651
2652                         return -1;
2653                 }
2654
2655                 void ReadToEndOfLine ()
2656                 {
2657                         int c;
2658                         do {
2659                                 c = get_char ();
2660                         } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS);
2661                 }
2662
2663                 void ReadSingleLineComment ()
2664                 {
2665                         if (peek_char () != '/')
2666                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2667
2668                         // Read everything till the end of the line or file
2669                         ReadToEndOfLine ();
2670                 }
2671
2672                 /// <summary>
2673                 /// Handles #pragma directive
2674                 /// </summary>
2675                 void ParsePragmaDirective ()
2676                 {
2677                         int c;
2678                         int length = TokenizePreprocessorKeyword (out c);
2679                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2680                                 length = TokenizePreprocessorKeyword (out c);
2681
2682                                 //
2683                                 // #pragma warning disable
2684                                 // #pragma warning restore
2685                                 //
2686                                 if (length == pragma_warning_disable.Length) {
2687                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2688                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2689                                                 // skip over white space
2690                                                 while (c == ' ' || c == '\t')
2691                                                         c = get_char ();
2692
2693                                                 var loc = Location;
2694
2695                                                 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) {
2696                                                         if (c == '/')
2697                                                                 ReadSingleLineComment ();
2698
2699                                                         //
2700                                                         // Disable/Restore all warnings
2701                                                         //
2702                                                         if (disable) {
2703                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2704                                                         } else {
2705                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2706                                                         }
2707                                                 } else {
2708                                                         //
2709                                                         // Disable/Restore a warning or group of warnings
2710                                                         //
2711                                                         int code;
2712                                                         do {
2713                                                                 bool identifier = false;
2714                                                                 code = TokenizePragmaWarningIdentifier (ref c, ref identifier);
2715                                                                 if (code > 0) {
2716                                                                         if (identifier) {
2717                                                                                 // no-op, custom warnings cannot occur in mcs
2718                                                                         } else if (disable) {
2719                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2720                                                                         } else {
2721                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2722                                                                         }
2723                                                                 }
2724                                                         } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS);
2725                                                 }
2726
2727                                                 return;
2728                                         }
2729                                 }
2730
2731                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2732
2733                                 // Eat any remaining characters on the line
2734                                 ReadToEndOfLine ();
2735
2736                                 return;
2737                         }
2738
2739                         //
2740                         // #pragma checksum
2741                         //
2742                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2743                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2744                                         Report.Warning (1695, 1, Location,
2745                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2746                                 }
2747
2748                                 return;
2749                         }
2750
2751                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2752
2753                         // Eat any remaining characters on the line
2754                         ReadToEndOfLine ();
2755                 }
2756
2757                 bool eval_val (string s)
2758                 {
2759                         if (s == "true")
2760                                 return true;
2761                         if (s == "false")
2762                                 return false;
2763
2764                         return source_file.IsConditionalDefined (s);
2765                 }
2766
2767                 bool pp_primary (ref string s)
2768                 {
2769                         s = s.Trim ();
2770                         int len = s.Length;
2771
2772                         if (len > 0){
2773                                 char c = s [0];
2774
2775                                 if (c == '('){
2776                                         s = s.Substring (1);
2777                                         bool val = pp_expr (ref s, false);
2778                                         if (s.Length > 0 && s [0] == ')'){
2779                                                 s = s.Substring (1);
2780                                                 return val;
2781                                         }
2782                                         Error_InvalidDirective ();
2783                                         return false;
2784                                 }
2785
2786                                 if (is_identifier_start_character (c)){
2787                                         int j = 1;
2788
2789                                         while (j < len){
2790                                                 c = s [j];
2791
2792                                                 if (is_identifier_part_character (c)){
2793                                                         j++;
2794                                                         continue;
2795                                                 }
2796                                                 bool v = eval_val (s.Substring (0, j));
2797                                                 s = s.Substring (j);
2798                                                 return v;
2799                                         }
2800                                         bool vv = eval_val (s);
2801                                         s = "";
2802                                         return vv;
2803                                 }
2804                         }
2805                         Error_InvalidDirective ();
2806                         return false;
2807                 }
2808
2809                 bool pp_unary (ref string s)
2810                 {
2811                         s = s.Trim ();
2812                         int len = s.Length;
2813
2814                         if (len > 0){
2815                                 if (s [0] == '!'){
2816                                         if (len > 1 && s [1] == '='){
2817                                                 Error_InvalidDirective ();
2818                                                 return false;
2819                                         }
2820                                         s = s.Substring (1);
2821                                         return ! pp_primary (ref s);
2822                                 } else
2823                                         return pp_primary (ref s);
2824                         } else {
2825                                 Error_InvalidDirective ();
2826                                 return false;
2827                         }
2828                 }
2829
2830                 bool pp_eq (ref string s)
2831                 {
2832                         bool va = pp_unary (ref s);
2833
2834                         s = s.Trim ();
2835                         int len = s.Length;
2836                         if (len > 0){
2837                                 if (s [0] == '='){
2838                                         if (len > 2 && s [1] == '='){
2839                                                 s = s.Substring (2);
2840                                                 return va == pp_unary (ref s);
2841                                         } else {
2842                                                 Error_InvalidDirective ();
2843                                                 return false;
2844                                         }
2845                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2846                                         s = s.Substring (2);
2847
2848                                         return va != pp_unary (ref s);
2849
2850                                 }
2851                         }
2852
2853                         return va;
2854
2855                 }
2856
2857                 bool pp_and (ref string s)
2858                 {
2859                         bool va = pp_eq (ref s);
2860
2861                         s = s.Trim ();
2862                         int len = s.Length;
2863                         if (len > 0){
2864                                 if (s [0] == '&'){
2865                                         if (len > 2 && s [1] == '&'){
2866                                                 s = s.Substring (2);
2867                                                 return (va & pp_and (ref s));
2868                                         } else {
2869                                                 Error_InvalidDirective ();
2870                                                 return false;
2871                                         }
2872                                 }
2873                         }
2874                         return va;
2875                 }
2876
2877                 //
2878                 // Evaluates an expression for `#if' or `#elif'
2879                 //
2880                 bool pp_expr (ref string s, bool isTerm)
2881                 {
2882                         bool va = pp_and (ref s);
2883                         s = s.Trim ();
2884                         int len = s.Length;
2885                         if (len > 0){
2886                                 char c = s [0];
2887
2888                                 if (c == '|'){
2889                                         if (len > 2 && s [1] == '|'){
2890                                                 s = s.Substring (2);
2891                                                 return va | pp_expr (ref s, isTerm);
2892                                         } else {
2893                                                 Error_InvalidDirective ();
2894                                                 return false;
2895                                         }
2896                                 }
2897                                 if (isTerm) {
2898                                         Error_EndLineExpected ();
2899                                         return false;
2900                                 }
2901                         }
2902
2903                         return va;
2904                 }
2905
2906                 bool eval (string s)
2907                 {
2908                         bool v = pp_expr (ref s, true);
2909                         s = s.Trim ();
2910                         if (s.Length != 0){
2911                                 return false;
2912                         }
2913
2914                         return v;
2915                 }
2916
2917                 void Error_NumericConstantTooLong ()
2918                 {
2919                         Report.Error (1021, Location, "Integral constant is too large");
2920                 }
2921
2922                 void Error_InvalidDirective ()
2923                 {
2924                         Report.Error (1517, Location, "Invalid preprocessor directive");
2925                 }
2926
2927                 void Error_UnexpectedDirective (string extra)
2928                 {
2929                         Report.Error (
2930                                 1028, Location,
2931                                 "Unexpected processor directive ({0})", extra);
2932                 }
2933
2934                 void Error_TokensSeen ()
2935                 {
2936                         Report.Error (1032, Location,
2937                                 "Cannot define or undefine preprocessor symbols after first token in file");
2938                 }
2939
2940                 void Eror_WrongPreprocessorLocation ()
2941                 {
2942                         Report.Error (1040, Location,
2943                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2944                 }
2945
2946                 void Error_EndLineExpected ()
2947                 {
2948                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2949                 }
2950
2951                 //
2952                 // Raises a warning when tokenizer found documentation comment
2953                 // on unexpected place
2954                 //
2955                 void WarningMisplacedComment (Location loc)
2956                 {
2957                         if (doc_state != XmlCommentState.Error) {
2958                                 doc_state = XmlCommentState.Error;
2959                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2960                         }
2961                 }
2962
2963                 //
2964                 // if true, then the code continues processing the code
2965                 // if false, the code stays in a loop until another directive is
2966                 // reached.
2967                 // When caller_is_taking is false we ignore all directives except the ones
2968                 // which can help us to identify where the #if block ends
2969                 bool ParsePreprocessingDirective (bool caller_is_taking)
2970                 {
2971                         string arg;
2972                         bool region_directive = false;
2973
2974                         var directive = get_cmd_arg (out arg);
2975
2976                         //
2977                         // The first group of pre-processing instructions is always processed
2978                         //
2979                         switch (directive) {
2980                         case PreprocessorDirective.Region:
2981                                 region_directive = true;
2982                                 arg = "true";
2983                                 goto case PreprocessorDirective.If;
2984
2985                         case PreprocessorDirective.Endregion:
2986                                 if (ifstack == null || ifstack.Count == 0){
2987                                         Error_UnexpectedDirective ("no #region for this #endregion");
2988                                         return true;
2989                                 }
2990                                 int pop = ifstack.Pop ();
2991
2992                                 if ((pop & REGION) == 0)
2993                                         Report.Error (1027, Location, "Expected `#endif' directive");
2994
2995                                 return caller_is_taking;
2996
2997                         case PreprocessorDirective.If:
2998                                 if (ifstack == null)
2999                                         ifstack = new Stack<int> (2);
3000
3001                                 int flags = region_directive ? REGION : 0;
3002                                 if (ifstack.Count == 0){
3003                                         flags |= PARENT_TAKING;
3004                                 } else {
3005                                         int state = ifstack.Peek ();
3006                                         if ((state & TAKING) != 0) {
3007                                                 flags |= PARENT_TAKING;
3008                                         }
3009                                 }
3010
3011                                 if (eval (arg) && caller_is_taking) {
3012                                         ifstack.Push (flags | TAKING);
3013                                         return true;
3014                                 }
3015                                 ifstack.Push (flags);
3016                                 return false;
3017
3018                         case PreprocessorDirective.Endif:
3019                                 if (ifstack == null || ifstack.Count == 0){
3020                                         Error_UnexpectedDirective ("no #if for this #endif");
3021                                         return true;
3022                                 } else {
3023                                         pop = ifstack.Pop ();
3024
3025                                         if ((pop & REGION) != 0)
3026                                                 Report.Error (1038, Location, "#endregion directive expected");
3027
3028                                         if (arg.Length != 0) {
3029                                                 Error_EndLineExpected ();
3030                                         }
3031
3032                                         if (ifstack.Count == 0)
3033                                                 return true;
3034
3035                                         int state = ifstack.Peek ();
3036                                         return (state & TAKING) != 0;
3037                                 }
3038
3039                         case PreprocessorDirective.Elif:
3040                                 if (ifstack == null || ifstack.Count == 0){
3041                                         Error_UnexpectedDirective ("no #if for this #elif");
3042                                         return true;
3043                                 } else {
3044                                         int state = ifstack.Pop ();
3045
3046                                         if ((state & REGION) != 0) {
3047                                                 Report.Error (1038, Location, "#endregion directive expected");
3048                                                 return true;
3049                                         }
3050
3051                                         if ((state & ELSE_SEEN) != 0){
3052                                                 Error_UnexpectedDirective ("#elif not valid after #else");
3053                                                 return true;
3054                                         }
3055
3056                                         if ((state & TAKING) != 0) {
3057                                                 ifstack.Push (0);
3058                                                 return false;
3059                                         }
3060
3061                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
3062                                                 ifstack.Push (state | TAKING);
3063                                                 return true;
3064                                         }
3065
3066                                         ifstack.Push (state);
3067                                         return false;
3068                                 }
3069
3070                         case PreprocessorDirective.Else:
3071                                 if (ifstack == null || ifstack.Count == 0){
3072                                         Error_UnexpectedDirective ("no #if for this #else");
3073                                         return true;
3074                                 } else {
3075                                         int state = ifstack.Peek ();
3076
3077                                         if ((state & REGION) != 0) {
3078                                                 Report.Error (1038, Location, "#endregion directive expected");
3079                                                 return true;
3080                                         }
3081
3082                                         if ((state & ELSE_SEEN) != 0){
3083                                                 Error_UnexpectedDirective ("#else within #else");
3084                                                 return true;
3085                                         }
3086
3087                                         ifstack.Pop ();
3088
3089                                         if (arg.Length != 0) {
3090                                                 Error_EndLineExpected ();
3091                                                 return true;
3092                                         }
3093
3094                                         bool ret = false;
3095                                         if ((state & PARENT_TAKING) != 0) {
3096                                                 ret = (state & TAKING) == 0;
3097
3098                                                 if (ret)
3099                                                         state |= TAKING;
3100                                                 else
3101                                                         state &= ~TAKING;
3102                                         }
3103
3104                                         ifstack.Push (state | ELSE_SEEN);
3105
3106                                         return ret;
3107                                 }
3108                         case PreprocessorDirective.Define:
3109                                 if (any_token_seen){
3110                                         if (caller_is_taking)
3111                                                 Error_TokensSeen ();
3112                                         return caller_is_taking;
3113                                 }
3114                                 PreProcessDefinition (true, arg, caller_is_taking);
3115                                 return caller_is_taking;
3116
3117                         case PreprocessorDirective.Undef:
3118                                 if (any_token_seen){
3119                                         if (caller_is_taking)
3120                                                 Error_TokensSeen ();
3121                                         return caller_is_taking;
3122                                 }
3123                                 PreProcessDefinition (false, arg, caller_is_taking);
3124                                 return caller_is_taking;
3125
3126                         case PreprocessorDirective.Invalid:
3127                                 Report.Error (1024, Location, "Wrong preprocessor directive");
3128                                 return true;
3129                         }
3130
3131                         //
3132                         // These are only processed if we are in a `taking' block
3133                         //
3134                         if (!caller_is_taking)
3135                                 return false;
3136
3137                         switch (directive){
3138                         case PreprocessorDirective.Error:
3139                                 Report.Error (1029, Location, "#error: '{0}'", arg);
3140                                 return true;
3141
3142                         case PreprocessorDirective.Warning:
3143                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
3144                                 return true;
3145
3146                         case PreprocessorDirective.Pragma:
3147                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
3148                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
3149                                 }
3150
3151                                 ParsePragmaDirective ();
3152                                 return true;
3153
3154                         case PreprocessorDirective.Line:
3155                                 Location loc = Location;
3156                                 if (!PreProcessLine ())
3157                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
3158
3159                                 return caller_is_taking;
3160                         }
3161
3162                         throw new NotImplementedException (directive.ToString ());
3163                 }
3164
3165                 int consume_string (bool quoted)
3166                 {
3167                         int c;
3168                         int pos = 0;
3169                         Location start_location = Location;
3170                         if (quoted)
3171                                 start_location = start_location - 1;
3172
3173 #if FULL_AST
3174                         int reader_pos = reader.Position;
3175 #endif
3176
3177                         while (true){
3178                                 // Cannot use get_char because of \r in quoted strings
3179                                 if (putback_char != -1) {
3180                                         c = putback_char;
3181                                         putback_char = -1;
3182                                 } else {
3183                                         c = reader.Read ();
3184                                 }
3185
3186                                 if (c == '"') {
3187                                         ++col;
3188
3189                                         if (quoted && peek_char () == '"') {
3190                                                 if (pos == value_builder.Length)
3191                                                         Array.Resize (ref value_builder, pos * 2);
3192
3193                                                 value_builder[pos++] = (char) c;
3194                                                 get_char ();
3195                                                 continue;
3196                                         }
3197
3198                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
3199                                         val = res;
3200 #if FULL_AST
3201                                         res.ParsedValue = quoted ?
3202                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
3203                                                 reader.ReadChars (reader_pos - 1, reader.Position);
3204 #endif
3205
3206                                         return Token.LITERAL;
3207                                 }
3208
3209                                 if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3210                                         if (!quoted) {
3211                                                 Report.Error (1010, Location, "Newline in constant");
3212
3213                                                 advance_line ();
3214
3215                                                 // Don't add \r to string literal
3216                                                 if (pos > 1 && value_builder [pos - 1] == '\r')
3217                                                         --pos;
3218
3219                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
3220                                                 return Token.LITERAL;
3221                                         }
3222
3223                                         advance_line ();
3224                                 } else if (c == '\\' && !quoted) {
3225                                         ++col;
3226                                         int surrogate;
3227                                         c = escape (c, out surrogate);
3228                                         if (c == -1)
3229                                                 return Token.ERROR;
3230                                         if (surrogate != 0) {
3231                                                 if (pos == value_builder.Length)
3232                                                         Array.Resize (ref value_builder, pos * 2);
3233
3234                                                 value_builder[pos++] = (char) c;
3235                                                 c = surrogate;
3236                                         }
3237                                 } else if (c == -1) {
3238                                         Report.Error (1039, Location, "Unterminated string literal");
3239                                         return Token.EOF;
3240                                 } else {
3241                                         ++col;
3242                                 }
3243
3244                                 if (pos == value_builder.Length)
3245                                         Array.Resize (ref value_builder, pos * 2);
3246
3247                                 value_builder[pos++] = (char) c;
3248                         }
3249                 }
3250
3251                 private int consume_identifier (int s)
3252                 {
3253                         int res = consume_identifier (s, false);
3254
3255                         if (doc_state == XmlCommentState.Allowed)
3256                                 doc_state = XmlCommentState.NotAllowed;
3257
3258                         return res;
3259                 }
3260
3261                 int consume_identifier (int c, bool quoted)
3262                 {
3263                         //
3264                         // This method is very performance sensitive. It accounts
3265                         // for approximately 25% of all parser time
3266                         //
3267
3268                         int pos = 0;
3269                         int column = col;
3270                         if (quoted)
3271                                 --column;
3272
3273                         if (c == '\\') {
3274                                 int surrogate;
3275                                 c = escape (c, out surrogate);
3276                                 if (quoted || is_identifier_start_character (c)) {
3277                                         // it's added bellow
3278                                 } else if (surrogate != 0) {
3279                                         id_builder [pos++] = (char)c;
3280                                         c = surrogate;
3281                                 } else {
3282                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3283                                         return Token.ERROR;
3284                                 }
3285                         }
3286
3287                         id_builder [pos++] = (char) c;
3288
3289                         try {
3290                                 while (true) {
3291                                         c = reader.Read ();
3292
3293                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
3294                                                 id_builder [pos++] = (char) c;
3295                                                 continue;
3296                                         }
3297
3298                                         if (c < 0x80) {
3299                                                 if (c == '\\') {
3300                                                         int surrogate;
3301                                                         c = escape (c, out surrogate);
3302                                                         if (is_identifier_part_character ((char) c))
3303                                                                 id_builder[pos++] = (char) c;
3304                                                         else if (surrogate != 0) {
3305                                                                 c = surrogate;
3306                                                         } else {
3307                                                                 switch (c) {
3308                                                                 // TODO: Probably need more whitespace characters
3309                                                                 case 0xFEFF:
3310                                                                         putback_char = c;
3311                                                                         break;
3312                                                                 default:
3313                                                                         Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4"));
3314                                                                         return Token.ERROR;
3315                                                                 }
3316                                                         }
3317
3318                                                         continue;
3319                                                 }
3320                                         } else if (is_identifier_part_character_slow_part ((char) c)) {
3321                                                 id_builder [pos++] = (char) c;
3322                                                 continue;
3323                                         }
3324
3325                                         putback_char = c;
3326                                         break;
3327                                 }
3328                         } catch (IndexOutOfRangeException) {
3329                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
3330                                 --pos;
3331                                 col += pos;
3332                         }
3333
3334                         col += pos - 1;
3335
3336                         //
3337                         // Optimization: avoids doing the keyword lookup
3338                         // on uppercase letters
3339                         //
3340                         if (id_builder [0] >= '_' && !quoted) {
3341                                 int keyword = GetKeyword (id_builder, pos);
3342                                 if (keyword != -1) {
3343                                         val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column);
3344                                         return keyword;
3345                                 }
3346                         }
3347
3348                         string s = InternIdentifier (id_builder, pos);
3349                         val = ltb.Create (s, current_source, ref_line, column);
3350                         if (quoted && parsing_attribute_section)
3351                                 AddEscapedIdentifier (((LocatedToken) val).Location);
3352
3353                         return Token.IDENTIFIER;
3354                 }
3355
3356                 string InternIdentifier (char[] charBuffer, int length)
3357                 {
3358                         //
3359                         // Keep identifiers in an array of hashtables to avoid needless
3360                         // allocations
3361                         //
3362                         var identifiers_group = identifiers[length];
3363                         string s;
3364                         if (identifiers_group != null) {
3365                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
3366                                         return s;
3367                                 }
3368                         } else {
3369                                 // TODO: this should be number of files dependant
3370                                 // corlib compilation peaks at 1000 and System.Core at 150
3371                                 int capacity = length > 20 ? 10 : 100;
3372                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
3373                                 identifiers[length] = identifiers_group;
3374                         }
3375
3376                         char[] chars = new char[length];
3377                         Array.Copy (charBuffer, chars, length);
3378
3379                         s = new string (charBuffer, 0, length);
3380                         identifiers_group.Add (chars, s);
3381                         return s;
3382                 }
3383
3384                 public int xtoken ()
3385                 {
3386                         if (parsing_interpolation_format) {
3387                                 return TokenizeInterpolationFormat ();
3388                         }
3389
3390                         int d, c;
3391
3392                         // Whether we have seen comments on the current line
3393                         bool comments_seen = false;
3394                         while ((c = get_char ()) != -1) {
3395                                 switch (c) {
3396                                 case '\t':
3397                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3398                                         continue;
3399
3400                                 case ' ':
3401                                 case '\f':
3402                                 case '\v':
3403                                 case 0xa0:
3404                                 case 0:
3405                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3406                                         continue;
3407
3408 /*                              This is required for compatibility with .NET
3409                                 case 0xEF:
3410                                         if (peek_char () == 0xBB) {
3411                                                 PushPosition ();
3412                                                 get_char ();
3413                                                 if (get_char () == 0xBF)
3414                                                         continue;
3415                                                 PopPosition ();
3416                                         }
3417                                         break;
3418 */
3419                                 case '\\':
3420                                         tokens_seen = true;
3421                                         return consume_identifier (c);
3422
3423                                 case '{':
3424                                         val = ltb.Create (current_source, ref_line, col);
3425
3426                                         if (parsing_string_interpolation > 0)
3427                                                 ++string_interpolation_section;
3428
3429                                         return Token.OPEN_BRACE;
3430                                 case '}':
3431                                         if (parsing_string_interpolation > 0) {
3432                                                 if (string_interpolation_section == 0) {
3433                                                         --parsing_string_interpolation;
3434                                                         bool quoted;
3435                                                         if (parsing_string_interpolation_quoted != null && parsing_string_interpolation_quoted.Count > 0) {
3436                                                                 quoted = parsing_string_interpolation_quoted.Pop ();
3437                                                         } else {
3438                                                                 quoted = false;
3439                                                         }
3440
3441                                                         return TokenizeInterpolatedString (quoted);
3442                                                 }
3443
3444                                                 --string_interpolation_section;
3445                                         }
3446
3447                                         val = ltb.Create (current_source, ref_line, col);
3448                                         return Token.CLOSE_BRACE;
3449                                 case '[':
3450                                         // To block doccomment inside attribute declaration.
3451                                         if (doc_state == XmlCommentState.Allowed)
3452                                                 doc_state = XmlCommentState.NotAllowed;
3453
3454                                         val = ltb.Create (current_source, ref_line, col);
3455
3456                                         if (parsing_block == 0 || lambda_arguments_parsing)
3457                                                 return Token.OPEN_BRACKET;
3458
3459                                         int next = peek_char ();
3460                                         switch (next) {
3461                                         case ']':
3462                                         case ',':
3463                                                 return Token.OPEN_BRACKET;
3464
3465                                         case ' ':
3466                                         case '\f':
3467                                         case '\v':
3468                                         case '\r':
3469                                         case '\n':
3470                                         case UnicodeLS:
3471                                         case UnicodePS:
3472                                         case '/':
3473                                                 next = peek_token ();
3474                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3475                                                         return Token.OPEN_BRACKET;
3476
3477                                                 return Token.OPEN_BRACKET_EXPR;
3478                                         default:
3479                                                 return Token.OPEN_BRACKET_EXPR;
3480                                         }
3481                                 case ']':
3482                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3483                                         return Token.CLOSE_BRACKET;
3484                                 case '(':
3485                                         val = ltb.Create (current_source, ref_line, col);
3486                                         //
3487                                         // An expression versions of parens can appear in block context only
3488                                         //
3489                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3490
3491                                                 //
3492                                                 // Optmize most common case where we know that parens
3493                                                 // is not special
3494                                                 //
3495                                                 switch (current_token) {
3496                                                 case Token.IDENTIFIER:
3497                                                 case Token.IF:
3498                                                 case Token.FOR:
3499                                                 case Token.FOREACH:
3500                                                 case Token.TYPEOF:
3501                                                 case Token.WHILE:
3502                                                 case Token.SWITCH:
3503                                                 case Token.USING:
3504                                                 case Token.DEFAULT:
3505                                                 case Token.DELEGATE:
3506                                                 case Token.OP_GENERICS_GT:
3507                                                 case Token.REFVALUE:
3508                                                         return Token.OPEN_PARENS;
3509                                                 }
3510
3511                                                 // Optimize using peek
3512                                                 int xx = peek_char ();
3513                                                 switch (xx) {
3514                                                 case '(':
3515                                                 case '\'':
3516                                                 case '"':
3517                                                 case '0':
3518                                                 case '1':
3519                                                         return Token.OPEN_PARENS;
3520                                                 }
3521
3522                                                 lambda_arguments_parsing = true;
3523                                                 PushPosition ();
3524                                                 d = TokenizeOpenParens ();
3525                                                 PopPosition ();
3526                                                 lambda_arguments_parsing = false;
3527                                                 return d;
3528                                         }
3529
3530                                         return Token.OPEN_PARENS;
3531                                 case ')':
3532                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3533                                         return Token.CLOSE_PARENS;
3534                                 case ',':
3535                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3536                                         return Token.COMMA;
3537                                 case ';':
3538                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3539                                         return Token.SEMICOLON;
3540                                 case '~':
3541                                         val = ltb.Create (current_source, ref_line, col);
3542                                         return Token.TILDE;
3543                                 case '?':
3544                                         val = ltb.Create (current_source, ref_line, col);
3545                                         return TokenizePossibleNullableType ();
3546                                 case '<':
3547                                         val = ltb.Create (current_source, ref_line, col);
3548                                         if (parsing_generic_less_than++ > 0)
3549                                                 return Token.OP_GENERICS_LT;
3550
3551                                         return TokenizeLessThan ();
3552
3553                                 case '>':
3554                                         val = ltb.Create (current_source, ref_line, col);
3555                                         d = peek_char ();
3556
3557                                         if (d == '='){
3558                                                 get_char ();
3559                                                 return Token.OP_GE;
3560                                         }
3561
3562                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3563                                                 parsing_generic_less_than--;
3564                                                 return Token.OP_GENERICS_GT;
3565                                         }
3566
3567                                         if (d == '>') {
3568                                                 get_char ();
3569                                                 d = peek_char ();
3570
3571                                                 if (d == '=') {
3572                                                         get_char ();
3573                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3574                                                 }
3575                                                 return Token.OP_SHIFT_RIGHT;
3576                                         }
3577
3578                                         return Token.OP_GT;
3579
3580                                 case '+':
3581                                         val = ltb.Create (current_source, ref_line, col);
3582                                         d = peek_char ();
3583                                         if (d == '+') {
3584                                                 d = Token.OP_INC;
3585                                         } else if (d == '=') {
3586                                                 d = Token.OP_ADD_ASSIGN;
3587                                         } else {
3588                                                 return Token.PLUS;
3589                                         }
3590                                         get_char ();
3591                                         return d;
3592
3593                                 case '-':
3594                                         val = ltb.Create (current_source, ref_line, col);
3595                                         d = peek_char ();
3596                                         if (d == '-') {
3597                                                 d = Token.OP_DEC;
3598                                         } else if (d == '=')
3599                                                 d = Token.OP_SUB_ASSIGN;
3600                                         else if (d == '>')
3601                                                 d = Token.OP_PTR;
3602                                         else {
3603                                                 return Token.MINUS;
3604                                         }
3605                                         get_char ();
3606                                         return d;
3607
3608                                 case '!':
3609                                         val = ltb.Create (current_source, ref_line, col);
3610                                         if (peek_char () == '='){
3611                                                 get_char ();
3612                                                 return Token.OP_NE;
3613                                         }
3614                                         return Token.BANG;
3615
3616                                 case '=':
3617                                         val = ltb.Create (current_source, ref_line, col);
3618                                         d = peek_char ();
3619                                         if (d == '='){
3620                                                 get_char ();
3621                                                 return Token.OP_EQ;
3622                                         }
3623                                         if (d == '>'){
3624                                                 get_char ();
3625                                                 return Token.ARROW;
3626                                         }
3627
3628                                         return Token.ASSIGN;
3629
3630                                 case '&':
3631                                         val = ltb.Create (current_source, ref_line, col);
3632                                         d = peek_char ();
3633                                         if (d == '&'){
3634                                                 get_char ();
3635                                                 return Token.OP_AND;
3636                                         }
3637                                         if (d == '='){
3638                                                 get_char ();
3639                                                 return Token.OP_AND_ASSIGN;
3640                                         }
3641                                         return Token.BITWISE_AND;
3642
3643                                 case '|':
3644                                         val = ltb.Create (current_source, ref_line, col);
3645                                         d = peek_char ();
3646                                         if (d == '|'){
3647                                                 get_char ();
3648                                                 return Token.OP_OR;
3649                                         }
3650                                         if (d == '='){
3651                                                 get_char ();
3652                                                 return Token.OP_OR_ASSIGN;
3653                                         }
3654                                         return Token.BITWISE_OR;
3655
3656                                 case '*':
3657                                         val = ltb.Create (current_source, ref_line, col);
3658                                         if (peek_char () == '='){
3659                                                 get_char ();
3660                                                 return Token.OP_MULT_ASSIGN;
3661                                         }
3662                                         return Token.STAR;
3663
3664                                 case '/':
3665                                         d = peek_char ();
3666                                         if (d == '='){
3667                                                 val = ltb.Create (current_source, ref_line, col);
3668                                                 get_char ();
3669                                                 return Token.OP_DIV_ASSIGN;
3670                                         }
3671
3672                                         // Handle double-slash comments.
3673                                         if (d == '/'){
3674                                                 if (parsing_string_interpolation > 0) {
3675                                                         Report.Error (8077, Location, "A single-line comment may not be used in an interpolated string");
3676                                                         goto case '}';
3677                                                 }
3678
3679                                                 get_char ();
3680                                                 if (doc_processing) {
3681                                                         if (peek_char () == '/') {
3682                                                                 get_char ();
3683                                                                 // Don't allow ////.
3684                                                                 if ((d = peek_char ()) != '/') {
3685                                                                         if (doc_state == XmlCommentState.Allowed)
3686                                                                                 handle_one_line_xml_comment ();
3687                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3688                                                                                 WarningMisplacedComment (Location - 3);
3689                                                                 }
3690                                                         } else {
3691                                                                 if (xml_comment_buffer.Length > 0)
3692                                                                         doc_state = XmlCommentState.NotAllowed;
3693                                                         }
3694                                                 }
3695
3696                                                 ReadToEndOfLine ();
3697
3698                                                 any_token_seen |= tokens_seen;
3699                                                 tokens_seen = false;
3700                                                 comments_seen = false;
3701                                                 continue;
3702                                         } else if (d == '*'){
3703                                                 get_char ();
3704                                                 bool docAppend = false;
3705                                                 if (doc_processing && peek_char () == '*') {
3706                                                         get_char ();
3707                                                         // But when it is /**/, just do nothing.
3708                                                         if (peek_char () == '/') {
3709                                                                 get_char ();
3710                                                                 continue;
3711                                                         }
3712                                                         if (doc_state == XmlCommentState.Allowed)
3713                                                                 docAppend = true;
3714                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3715                                                                 WarningMisplacedComment (Location - 2);
3716                                                         }
3717                                                 }
3718
3719                                                 int current_comment_start = 0;
3720                                                 if (docAppend) {
3721                                                         current_comment_start = xml_comment_buffer.Length;
3722                                                         xml_comment_buffer.Append (Environment.NewLine);
3723                                                 }
3724
3725                                                 while ((d = get_char ()) != -1){
3726                                                         if (d == '*' && peek_char () == '/'){
3727                                                                 get_char ();
3728                                                                 comments_seen = true;
3729                                                                 break;
3730                                                         }
3731                                                         if (docAppend)
3732                                                                 xml_comment_buffer.Append ((char) d);
3733
3734                                                         if (d == '\n' || d == UnicodeLS || d == UnicodePS){
3735                                                                 any_token_seen |= tokens_seen;
3736                                                                 tokens_seen = false;
3737                                                                 //
3738                                                                 // Reset 'comments_seen' just to be consistent.
3739                                                                 // It doesn't matter either way, here.
3740                                                                 //
3741                                                                 comments_seen = false;
3742                                                         }
3743                                                 }
3744                                                 if (!comments_seen)
3745                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3746
3747                                                 if (docAppend)
3748                                                         update_formatted_doc_comment (current_comment_start);
3749                                                 continue;
3750                                         }
3751                                         val = ltb.Create (current_source, ref_line, col);
3752                                         return Token.DIV;
3753
3754                                 case '%':
3755                                         val = ltb.Create (current_source, ref_line, col);
3756                                         if (peek_char () == '='){
3757                                                 get_char ();
3758                                                 return Token.OP_MOD_ASSIGN;
3759                                         }
3760                                         return Token.PERCENT;
3761
3762                                 case '^':
3763                                         val = ltb.Create (current_source, ref_line, col);
3764                                         if (peek_char () == '='){
3765                                                 get_char ();
3766                                                 return Token.OP_XOR_ASSIGN;
3767                                         }
3768                                         return Token.CARRET;
3769
3770                                 case ':':
3771                                         val = ltb.Create (current_source, ref_line, col);
3772                                         if (peek_char () == ':') {
3773                                                 get_char ();
3774                                                 return Token.DOUBLE_COLON;
3775                                         }
3776                                         return Token.COLON;
3777
3778                                 case '0': case '1': case '2': case '3': case '4':
3779                                 case '5': case '6': case '7': case '8': case '9':
3780                                         tokens_seen = true;
3781                                         return is_number (c, false);
3782
3783                                 case '\n': // white space
3784                                 case UnicodeLS:
3785                                 case UnicodePS:
3786                                         any_token_seen |= tokens_seen;
3787                                         tokens_seen = false;
3788                                         comments_seen = false;
3789                                         continue;
3790
3791                                 case '.':
3792                                         tokens_seen = true;
3793                                         d = peek_char ();
3794                                         if (d >= '0' && d <= '9')
3795                                                 return is_number (c, true);
3796
3797                                         ltb.CreateOptional (current_source, ref_line, col, ref val);
3798                                         return Token.DOT;
3799
3800                                 case '#':
3801                                         if (tokens_seen || comments_seen) {
3802                                                 Eror_WrongPreprocessorLocation ();
3803                                                 return Token.ERROR;
3804                                         }
3805
3806                                         if (ParsePreprocessingDirective (true))
3807                                                 continue;
3808
3809                                         bool directive_expected = false;
3810                                         while ((c = get_char ()) != -1) {
3811                                                 if (col == 1) {
3812                                                         directive_expected = true;
3813                                                 } else if (!directive_expected) {
3814                                                         // TODO: Implement comment support for disabled code and uncomment this code
3815 //                                                      if (c == '#') {
3816 //                                                              Eror_WrongPreprocessorLocation ();
3817 //                                                              return Token.ERROR;
3818 //                                                      }
3819                                                         continue;
3820                                                 }
3821
3822                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS)
3823                                                         continue;
3824
3825                                                 if (c == '#') {
3826                                                         if (ParsePreprocessingDirective (false))
3827                                                                 break;
3828                                                 }
3829                                                 directive_expected = false;
3830                                         }
3831
3832                                         if (c != -1) {
3833                                                 tokens_seen = false;
3834                                                 continue;
3835                                         }
3836
3837                                         return Token.EOF;
3838
3839                                 case '"':
3840                                         if (parsing_string_interpolation > 0 && !ScanClosingInterpolationBrace ()) {
3841                                                 parsing_string_interpolation = 0;
3842                                                 Report.Error (8076, Location, "Missing close delimiter `}' for interpolated expression");
3843                                                 val = new StringLiteral (context.BuiltinTypes, "", Location);
3844                                                 return Token.INTERPOLATED_STRING_END;
3845                                         }
3846
3847                                         return consume_string (false);
3848
3849                                 case '\'':
3850                                         return TokenizeBackslash ();
3851
3852                                 case '@':
3853                                         c = get_char ();
3854                                         if (c == '"') {
3855                                                 tokens_seen = true;
3856                                                 return consume_string (true);
3857                                         }
3858
3859                                         if (is_identifier_start_character (c)){
3860                                                 return consume_identifier (c, true);
3861                                         }
3862
3863                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3864                                         return Token.ERROR;
3865
3866                                 case '$':
3867                                         switch (peek_char ()) {
3868                                         case '"':
3869                                                 get_char ();
3870                                                 return TokenizeInterpolatedString (false);
3871                                         case '@':
3872                                                 get_char ();
3873                                                 if (peek_char () == '"') {
3874                                                         get_char ();
3875                                                         return TokenizeInterpolatedString (true);
3876                                                 }
3877
3878                                                 break;
3879                                         }
3880
3881                                         break;
3882                                 case EvalStatementParserCharacter:
3883                                         return Token.EVAL_STATEMENT_PARSER;
3884                                 case EvalCompilationUnitParserCharacter:
3885                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3886                                 case EvalUsingDeclarationsParserCharacter:
3887                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3888                                 case DocumentationXref:
3889                                         return Token.DOC_SEE;
3890                                 }
3891
3892                                 if (is_identifier_start_character (c)) {
3893                                         tokens_seen = true;
3894                                         return consume_identifier (c);
3895                                 }
3896
3897                                 if (char.IsWhiteSpace ((char) c))
3898                                         continue;
3899
3900                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3901                         }
3902
3903                         if (CompleteOnEOF){
3904                                 if (generated)
3905                                         return Token.COMPLETE_COMPLETION;
3906
3907                                 generated = true;
3908                                 return Token.GENERATE_COMPLETION;
3909                         }
3910
3911
3912                         return Token.EOF;
3913                 }
3914
3915                 int TokenizeBackslash ()
3916                 {
3917 #if FULL_AST
3918                         int read_start = reader.Position;
3919 #endif
3920                         Location start_location = Location;
3921                         int c = get_char ();
3922                         tokens_seen = true;
3923                         if (c == '\'') {
3924                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3925                                 Report.Error (1011, start_location, "Empty character literal");
3926                                 return Token.LITERAL;
3927                         }
3928
3929                         if (c == '\n' || c == UnicodeLS || c == UnicodePS) {
3930                                 Report.Error (1010, start_location, "Newline in constant");
3931                                 return Token.ERROR;
3932                         }
3933
3934                         int d;
3935                         c = escape (c, out d);
3936                         if (c == -1)
3937                                 return Token.ERROR;
3938                         if (d != 0)
3939                                 throw new NotImplementedException ();
3940
3941                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3942                         val = res;
3943                         c = get_char ();
3944
3945                         if (c != '\'') {
3946                                 Report.Error (1012, start_location, "Too many characters in character literal");
3947
3948                                 // Try to recover, read until newline or next "'"
3949                                 while ((c = get_char ()) != -1) {
3950                                         if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS)
3951                                                 break;
3952                                 }
3953                         }
3954
3955 #if FULL_AST
3956                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3957 #endif
3958
3959                         return Token.LITERAL;
3960                 }
3961
3962                 int TokenizeLessThan ()
3963                 {
3964                         int d;
3965
3966                         // Save current position and parse next token.
3967                         PushPosition ();
3968                         int generic_dimension = 0;
3969                         if (parse_less_than (ref generic_dimension)) {
3970                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3971                                         d = Token.OP_GENERICS_LT_DECL;
3972                                 } else {
3973                                         if (generic_dimension > 0) {
3974                                                 val = generic_dimension;
3975                                                 DiscardPosition ();
3976                                                 return Token.GENERIC_DIMENSION;
3977                                         }
3978
3979                                         d = Token.OP_GENERICS_LT;
3980                                 }
3981                                 PopPosition ();
3982                                 return d;
3983                         }
3984
3985                         PopPosition ();
3986                         parsing_generic_less_than = 0;
3987
3988                         d = peek_char ();
3989                         if (d == '<') {
3990                                 get_char ();
3991                                 d = peek_char ();
3992
3993                                 if (d == '=') {
3994                                         get_char ();
3995                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3996                                 }
3997                                 return Token.OP_SHIFT_LEFT;
3998                         }
3999
4000                         if (d == '=') {
4001                                 get_char ();
4002                                 return Token.OP_LE;
4003                         }
4004                         return Token.OP_LT;
4005                 }
4006
4007                 int TokenizeInterpolatedString (bool quoted)
4008                 {
4009                         int pos = 0;
4010                         var start_location = Location;
4011
4012                         while (true) {
4013                                 var ch = get_char ();
4014                                 switch (ch) {
4015                                 case '"':
4016                                         if (quoted && peek_char () == '"') {
4017                                                 get_char ();
4018                                                 break;
4019                                         }
4020
4021                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
4022                                         return Token.INTERPOLATED_STRING_END;
4023                                 case '{':
4024                                         if (peek_char () == '{') {
4025                                                 value_builder [pos++] = (char)ch;
4026                                                 get_char ();
4027                                                 break;
4028                                         }
4029
4030                                         ++parsing_string_interpolation;
4031                                         if (quoted) {
4032                                                 if (parsing_string_interpolation_quoted == null)
4033                                                         parsing_string_interpolation_quoted = new Stack<bool> ();
4034                                         }
4035
4036                                         if (parsing_string_interpolation_quoted != null) {
4037                                                 parsing_string_interpolation_quoted.Push (quoted);
4038                                         }
4039
4040                                         val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location);
4041                                         return Token.INTERPOLATED_STRING;
4042                                 case '\\':
4043                                         if (quoted)
4044                                                 break;
4045
4046                                         ++col;
4047                                         int surrogate;
4048                                         ch = escape (ch, out surrogate);
4049                                         if (ch == -1)
4050                                                 return Token.ERROR;
4051
4052                                         if (ch == '{' || ch == '}') {
4053                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
4054                                         }
4055
4056                                         if (surrogate != 0) {
4057                                                 if (pos == value_builder.Length)
4058                                                         Array.Resize (ref value_builder, pos * 2);
4059
4060                                                 if (pos == value_builder.Length)
4061                                                         Array.Resize (ref value_builder, pos * 2);
4062
4063                                                 value_builder [pos++] = (char)ch;
4064                                                 ch = surrogate;
4065                                         }
4066
4067                                         break;
4068                                 case -1:
4069                                         return Token.EOF;
4070                                 }
4071
4072                                 ++col;
4073                                 if (pos == value_builder.Length)
4074                                         Array.Resize (ref value_builder, pos * 2);
4075
4076                                 value_builder[pos++] = (char) ch;
4077                         }
4078                 }
4079
4080                 int TokenizeInterpolationFormat ()
4081                 {
4082                         int pos = 0;
4083                         int braces = 0;
4084                         while (true) {
4085                                 var ch = get_char ();
4086                                 switch (ch) {
4087                                 case '{':
4088                                         ++braces;
4089                                         break;
4090                                 case '}':
4091                                         if (braces == 0) {
4092                                                 putback_char = ch;
4093                                                 if (pos == 0) {
4094                                                         Report.Error (8089, Location, "Empty interpolated expression format specifier");
4095                                                 } else if (Array.IndexOf (simple_whitespaces, value_builder [pos - 1]) >= 0) {
4096                                                         Report.Error (8088, Location, "A interpolated expression format specifier may not contain trailing whitespace");
4097                                                 }
4098
4099                                                 val = CreateStringFromBuilder (pos);
4100                                                 return Token.LITERAL;
4101                                         }
4102
4103                                         --braces;
4104                                         break;
4105                                 case '\\':
4106                                         if (parsing_string_interpolation_quoted != null && parsing_string_interpolation_quoted.Peek ())
4107                                                 break;
4108
4109                                         ++col;
4110                                         int surrogate;
4111                                         ch = escape (ch, out surrogate);
4112                                         if (ch == -1)
4113                                                 return Token.ERROR;
4114
4115                                         if (ch == '{' || ch == '}') {
4116                                                 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ());
4117                                         }
4118
4119                                         if (surrogate != 0) {
4120                                                 if (pos == value_builder.Length)
4121                                                         Array.Resize (ref value_builder, pos * 2);
4122
4123                                                 value_builder [pos++] = (char)ch;
4124                                                 ch = surrogate;
4125                                         }
4126
4127                                         break;
4128                                 case -1:
4129                                         return Token.EOF;
4130                                 }
4131
4132                                 ++col;
4133                                 value_builder[pos++] = (char) ch;
4134                         }
4135                 }
4136
4137                 string CreateStringFromBuilder (int pos)
4138                 {
4139                         if (pos == 0)
4140                                 return string.Empty;
4141                         if (pos <= 4)
4142                                 return InternIdentifier (value_builder, pos);
4143
4144                         return new string (value_builder, 0, pos);
4145                 }
4146
4147                 //
4148                 // Handles one line xml comment
4149                 //
4150                 private void handle_one_line_xml_comment ()
4151                 {
4152                         int c;
4153                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
4154                                 xml_comment_buffer.Append ((char) get_char ());
4155                         }
4156                         if (c == '\r' || c == '\n')
4157                                 xml_comment_buffer.Append (Environment.NewLine);
4158                 }
4159
4160                 //
4161                 // Remove heading "*" in Javadoc-like xml documentation.
4162                 //
4163                 private void update_formatted_doc_comment (int current_comment_start)
4164                 {
4165                         int length = xml_comment_buffer.Length - current_comment_start;
4166                         string [] lines = xml_comment_buffer.ToString (
4167                                 current_comment_start,
4168                                 length).Replace ("\r", "").Split ('\n');
4169
4170                         // The first line starts with /**, thus it is not target
4171                         // for the format check.
4172                         for (int i = 1; i < lines.Length; i++) {
4173                                 string s = lines [i];
4174                                 int idx = s.IndexOf ('*');
4175                                 string head = null;
4176                                 if (idx < 0) {
4177                                         if (i < lines.Length - 1)
4178                                                 return;
4179                                         head = s;
4180                                 } else
4181                                         head = s.Substring (0, idx);
4182                                 foreach (char c in head)
4183                                         if (c != ' ')
4184                                                 return;
4185                                 lines [i] = s.Substring (idx + 1);
4186                         }
4187                         xml_comment_buffer.Remove (current_comment_start, length);
4188                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
4189                 }
4190
4191                 //
4192                 // Checks if there was incorrect doc comments and raise
4193                 // warnings.
4194                 //
4195                 public void check_incorrect_doc_comment ()
4196                 {
4197                         if (xml_comment_buffer.Length > 0)
4198                                 WarningMisplacedComment (Location);
4199                 }
4200
4201                 //
4202                 // Consumes the saved xml comment lines (if any)
4203                 // as for current target member or type.
4204                 //
4205                 public string consume_doc_comment ()
4206                 {
4207                         if (xml_comment_buffer.Length > 0) {
4208                                 string ret = xml_comment_buffer.ToString ();
4209                                 reset_doc_comment ();
4210                                 return ret;
4211                         }
4212                         return null;
4213                 }
4214
4215                 void reset_doc_comment ()
4216                 {
4217                         xml_comment_buffer.Length = 0;
4218                 }
4219
4220                 public void cleanup ()
4221                 {
4222                         if (ifstack != null && ifstack.Count >= 1) {
4223                                 int state = ifstack.Pop ();
4224                                 if ((state & REGION) != 0)
4225                                         Report.Error (1038, Location, "#endregion directive expected");
4226                                 else
4227                                         Report.Error (1027, Location, "Expected `#endif' directive");
4228                         }
4229                 }
4230         }
4231
4232         //
4233         // Indicates whether it accepts XML documentation or not.
4234         //
4235         public enum XmlCommentState {
4236                 // comment is allowed in this state.
4237                 Allowed,
4238                 // comment is not allowed in this state.
4239                 NotAllowed,
4240                 // once comments appeared when it is NotAllowed, then the
4241                 // state is changed to it, until the state is changed to
4242                 // .Allowed.
4243                 Error
4244         }
4245 }
4246