mcs/mcs/cs-tokenizer.cs

   1 //
   2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
   3 //                  This also implements the preprocessor
   4 //
   5 // Author: Miguel de Icaza (miguel@gnu.org)
   6 //         Marek Safar (marek.safar@gmail.com)
   7 //
   8 // Dual licensed under the terms of the MIT X11 or GNU GPL
   9 //
  10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11 // Copyright 2004-2008 Novell, Inc
  12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13 //
  14
  15 using System;
  16 using System.Text;
  17 using System.Collections.Generic;
  18 using System.Globalization;
  19 using System.Diagnostics;
  20
  21 namespace Mono.CSharp
  22 {
  23         /// <summary>
  24         ///    Tokenizer for C# source code.
  25         /// </summary>
  26
  27         public class Tokenizer : yyParser.yyInput
  28         {
  29                 class KeywordEntry<T>
  30                 {
  31                         public readonly T Token;
  32                         public KeywordEntry<T> Next;
  33                         public readonly char[] Value;
  34
  35                         public KeywordEntry (string value, T token)
  36                         {
  37                                 this.Value = value.ToCharArray ();
  38                                 this.Token = token;
  39                         }
  40                 }
  41
  42                 sealed class IdentifiersComparer : IEqualityComparer<char[]>
  43                 {
  44                         readonly int length;
  45
  46                         public IdentifiersComparer (int length)
  47                         {
  48                                 this.length = length;
  49                         }
  50
  51                         public bool Equals (char[] x, char[] y)
  52                         {
  53                                 for (int i = 0; i < length; ++i)
  54                                         if (x [i] != y [i])
  55                                                 return false;
  56
  57                                 return true;
  58                         }
  59
  60                         public int GetHashCode (char[] obj)
  61                         {
  62                                 int h = 0;
  63                                 for (int i = 0; i < length; ++i)
  64                                         h = (h << 5) - h + obj [i];
  65
  66                                 return h;
  67                         }
  68                 }
  69
  70                 //
  71                 // This class has to be used in the parser only, it reuses token
  72                 // details after each parse
  73                 //
  74                 public class LocatedToken
  75                 {
  76                         int row, column;
  77                         string value;
  78
  79                         static LocatedToken[] buffer = new LocatedToken[0];
  80                         static int pos;
  81
  82                         private LocatedToken ()
  83                         {
  84                         }
  85
  86                         public static LocatedToken Create (int row, int column)
  87                         {
  88                                 return Create (null, row, column);
  89                         }
  90
  91                         public static LocatedToken Create (string value, Location loc)
  92                         {
  93                                 return Create (value, loc.Row, loc.Column);
  94                         }
  95
  96                         public static LocatedToken Create (string value, int row, int column)
  97                         {
  98                                 //
  99                                 // TODO: I am not very happy about the logic but it's the best
 100                                 // what I could come up with for now.
 101                                 // Ideally we should be using just tiny buffer (256 elements) which
 102                                 // is enough to hold all details for currect stack and recycle elements
 103                                 // poped from the stack but there is a trick needed to recycle
 104                                 // them properly.
 105                                 //
 106                                 LocatedToken entry;
 107                                 if (pos >= buffer.Length) {
 108                                         entry = new LocatedToken ();
 109                                 } else {
 110                                         entry = buffer [pos];
 111                                         if (entry == null) {
 112                                                 entry = new LocatedToken ();
 113                                                 buffer [pos] = entry;
 114                                         }
 115
 116                                         ++pos;
 117                                 }
 118                                 entry.value = value;
 119                                 entry.row = row;
 120                                 entry.column = column;
 121                                 return entry;
 122                         }
 123
 124                         //
 125                         // Used for token not required by expression evaluator
 126                         //
 127                         [Conditional ("FULL_AST")]
 128                         public static void CreateOptional (int row, int col, ref object token)
 129                         {
 130                                 token = Create (row, col);
 131                         }
 132
 133                         public static void Initialize ()
 134                         {
 135 #if !FULL_AST
 136                                 if (buffer.Length == 0)
 137                                         buffer = new LocatedToken [15000];
 138 #endif
 139                                 pos = 0;
 140                         }
 141
 142                         public Location Location {
 143                                 get { return new Location (row, column); }
 144                         }
 145
 146                         public string Value {
 147                                 get { return value; }
 148                         }
 149                 }
 150
 151                 public enum PreprocessorDirective
 152                 {
 153                         Invalid = 0,
 154
 155                         Region = 1,
 156                         Endregion = 2,
 157                         If = 3 | RequiresArgument,
 158                         Endif = 4,
 159                         Elif = 5 | RequiresArgument,
 160                         Else = 6,
 161                         Define = 7 | RequiresArgument,
 162                         Undef = 8 | RequiresArgument,
 163                         Error = 9,
 164                         Warning = 10,
 165                         Pragma = 11 | CustomArgumentsParsing,
 166                         Line = 12 | CustomArgumentsParsing,
 167
 168                         CustomArgumentsParsing = 1 << 10,
 169                         RequiresArgument = 1 << 11
 170                 }
 171
 172                 readonly SeekableStreamReader reader;
 173                 readonly CompilationSourceFile source_file;
 174                 readonly CompilerContext context;
 175
 176                 SourceFile current_source;
 177                 Location hidden_block_start;
 178                 int ref_line = 1;
 179                 int line = 1;
 180                 int col = 0;
 181                 int previous_col;
 182                 int current_token;
 183                 readonly int tab_size;
 184                 bool handle_get_set = false;
 185                 bool handle_remove_add = false;
 186                 bool handle_where = false;
 187                 bool handle_typeof = false;
 188                 bool lambda_arguments_parsing;
 189                 List<Location> escaped_identifiers;
 190                 int parsing_generic_less_than;
 191                 readonly bool doc_processing;
 192
 193                 //
 194                 // Used mainly for parser optimizations. Some expressions for instance
 195                 // can appear only in block (including initializer, base initializer)
 196                 // scope only
 197                 //
 198                 public int parsing_block;
 199                 internal bool query_parsing;
 200
 201                 //
 202                 // When parsing type only, useful for ambiguous nullable types
 203                 //
 204                 public int parsing_type;
 205
 206                 //
 207                 // Set when parsing generic declaration (type or method header)
 208                 //
 209                 public bool parsing_generic_declaration;
 210                 public bool parsing_generic_declaration_doc;
 211
 212                 //
 213                 // The value indicates that we have not reach any declaration or
 214                 // namespace yet
 215                 //
 216                 public int parsing_declaration;
 217
 218                 public bool parsing_attribute_section;
 219
 220                 public bool parsing_modifiers;
 221
 222                 //
 223                 // The special characters to inject on streams to run the unit parser
 224                 // in the special expression mode. Using private characters from
 225                 // Plane Sixteen (U+100000 to U+10FFFD)
 226                 //
 227                 // This character is only tested just before the tokenizer is about to report
 228                 // an error;   So on the regular operation mode, this addition will have no
 229                 // impact on the tokenizer's performance.
 230                 //
 231
 232                 public const int EvalStatementParserCharacter = 0x100000;
 233                 public const int EvalCompilationUnitParserCharacter = 0x100001;
 234                 public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 235                 public const int DocumentationXref = 0x100003;
 236
 237                 //
 238                 // XML documentation buffer. The save point is used to divide
 239                 // comments on types and comments on members.
 240                 //
 241                 StringBuilder xml_comment_buffer;
 242
 243                 //
 244                 // See comment on XmlCommentState enumeration.
 245                 //
 246                 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 247
 248                 //
 249                 // Whether tokens have been seen on this line
 250                 //
 251                 bool tokens_seen = false;
 252
 253                 //
 254                 // Set to true once the GENERATE_COMPLETION token has bee
 255                 // returned.   This helps produce one GENERATE_COMPLETION,
 256                 // as many COMPLETE_COMPLETION as necessary to complete the
 257                 // AST tree and one final EOF.
 258                 //
 259                 bool generated;
 260
 261                 //
 262                 // Whether a token has been seen on the file
 263                 // This is needed because `define' is not allowed to be used
 264                 // after a token has been seen.
 265                 //
 266                 bool any_token_seen;
 267
 268                 //
 269                 // Class variables
 270                 //
 271                 static readonly KeywordEntry<int>[][] keywords;
 272                 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 273                 static readonly HashSet<string> keyword_strings;
 274                 static readonly NumberStyles styles;
 275                 static readonly NumberFormatInfo csharp_format_info;
 276
 277                 // Pragma arguments
 278                 static readonly char[] pragma_warning = "warning".ToCharArray ();
 279                 static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 280                 static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 281                 static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 282                 static readonly char[] line_hidden = "hidden".ToCharArray ();
 283                 static readonly char[] line_default = "default".ToCharArray ();
 284
 285                 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 286
 287                 public bool PropertyParsing {
 288                         get { return handle_get_set; }
 289                         set { handle_get_set = value; }
 290                 }
 291
 292                 public bool EventParsing {
 293                         get { return handle_remove_add; }
 294                         set { handle_remove_add = value; }
 295                 }
 296
 297                 public bool ConstraintsParsing {
 298                         get { return handle_where; }
 299                         set { handle_where = value; }
 300                 }
 301
 302                 public bool TypeOfParsing {
 303                         get { return handle_typeof; }
 304                         set { handle_typeof = value; }
 305                 }
 306
 307                 public XmlCommentState doc_state {
 308                         get { return xml_doc_state; }
 309                         set {
 310                                 if (value == XmlCommentState.Allowed) {
 311                                         check_incorrect_doc_comment ();
 312                                         reset_doc_comment ();
 313                                 }
 314                                 xml_doc_state = value;
 315                         }
 316                 }
 317
 318                 //
 319                 // This is used to trigger completion generation on the parser
 320                 public bool CompleteOnEOF;
 321
 322                 void AddEscapedIdentifier (Location loc)
 323                 {
 324                         if (escaped_identifiers == null)
 325                                 escaped_identifiers = new List<Location> ();
 326
 327                         escaped_identifiers.Add (loc);
 328                 }
 329
 330                 public bool IsEscapedIdentifier (ATypeNameExpression name)
 331                 {
 332                         return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 333                 }
 334
 335                 //
 336                 // Values for the associated token returned
 337                 //
 338                 internal int putback_char;      // Used by repl only
 339                 object val;
 340
 341                 //
 342                 // Pre-processor
 343                 //
 344                 const int TAKING        = 1;
 345                 const int ELSE_SEEN     = 4;
 346                 const int PARENT_TAKING = 8;
 347                 const int REGION        = 16;
 348
 349                 //
 350                 // pre-processor if stack state:
 351                 //
 352                 Stack<int> ifstack;
 353
 354                 const int max_id_size = 512;
 355                 const int max_number_size = 512;
 356
 357 #if FULL_AST
 358                 readonly char [] id_builder = new char [max_id_size];
 359
 360                 Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 361
 362                 char [] number_builder = new char [max_number_size];
 363                 int number_pos;
 364
 365                 char[] value_builder = new char[256];
 366 #else
 367                 static readonly char [] id_builder = new char [max_id_size];
 368
 369                 static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
 370
 371                 static char [] number_builder = new char [max_number_size];
 372                 static int number_pos;
 373
 374                 static char[] value_builder = new char[256];
 375 #endif
 376
 377                 public int Line {
 378                         get {
 379                                 return ref_line;
 380                         }
 381                 }
 382
 383                 //
 384                 // This is used when the tokenizer needs to save
 385                 // the current position as it needs to do some parsing
 386                 // on its own to deamiguate a token in behalf of the
 387                 // parser.
 388                 //
 389                 Stack<Position> position_stack = new Stack<Position> (2);
 390
 391                 class Position {
 392                         public int position;
 393                         public int line;
 394                         public int ref_line;
 395                         public int col;
 396                         public Location hidden;
 397                         public int putback_char;
 398                         public int previous_col;
 399                         public Stack<int> ifstack;
 400                         public int parsing_generic_less_than;
 401                         public int current_token;
 402                         public object val;
 403
 404                         public Position (Tokenizer t)
 405                         {
 406                                 position = t.reader.Position;
 407                                 line = t.line;
 408                                 ref_line = t.ref_line;
 409                                 col = t.col;
 410                                 hidden = t.hidden_block_start;
 411                                 putback_char = t.putback_char;
 412                                 previous_col = t.previous_col;
 413                                 if (t.ifstack != null && t.ifstack.Count != 0) {
 414                                         // There is no simple way to clone Stack<T> all
 415                                         // methods reverse the order
 416                                         var clone = t.ifstack.ToArray ();
 417                                         Array.Reverse (clone);
 418                                         ifstack = new Stack<int> (clone);
 419                                 }
 420                                 parsing_generic_less_than = t.parsing_generic_less_than;
 421                                 current_token = t.current_token;
 422                                 val = t.val;
 423                         }
 424                 }
 425
 426                 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file)
 427                 {
 428                         this.source_file = file;
 429                         this.context = file.Compiler;
 430                         this.current_source = file.SourceFile;
 431
 432                         reader = input;
 433
 434                         putback_char = -1;
 435
 436                         xml_comment_buffer = new StringBuilder ();
 437                         doc_processing = context.Settings.DocumentationFile != null;
 438
 439                         tab_size = context.Settings.TabSize;
 440
 441                         Mono.CSharp.Location.Push (current_source);
 442                 }
 443
 444                 public void PushPosition ()
 445                 {
 446                         position_stack.Push (new Position (this));
 447                 }
 448
 449                 public void PopPosition ()
 450                 {
 451                         Position p = position_stack.Pop ();
 452
 453                         reader.Position = p.position;
 454                         ref_line = p.ref_line;
 455                         line = p.line;
 456                         col = p.col;
 457                         hidden_block_start = p.hidden;
 458                         putback_char = p.putback_char;
 459                         previous_col = p.previous_col;
 460                         ifstack = p.ifstack;
 461                         parsing_generic_less_than = p.parsing_generic_less_than;
 462                         current_token = p.current_token;
 463                         val = p.val;
 464                 }
 465
 466                 // Do not reset the position, ignore it.
 467                 public void DiscardPosition ()
 468                 {
 469                         position_stack.Pop ();
 470                 }
 471
 472                 static void AddKeyword (string kw, int token)
 473                 {
 474                         keyword_strings.Add (kw);
 475
 476                         AddKeyword (keywords, kw, token);
 477                 }
 478
 479                 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 480                 {
 481                         AddKeyword (keywords_preprocessor, kw, directive);
 482                 }
 483
 484                 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 485                 {
 486                         int length = kw.Length;
 487                         if (keywords[length] == null) {
 488                                 keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 489                         }
 490
 491                         int char_index = kw[0] - '_';
 492                         var kwe = keywords[length][char_index];
 493                         if (kwe == null) {
 494                                 keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 495                                 return;
 496                         }
 497
 498                         while (kwe.Next != null) {
 499                                 kwe = kwe.Next;
 500                         }
 501
 502                         kwe.Next = new KeywordEntry<T> (kw, token);
 503                 }
 504
 505                 //
 506                 // Class initializer
 507                 //
 508                 static Tokenizer ()
 509                 {
 510                         keyword_strings = new HashSet<string> ();
 511
 512                         // 11 is the length of the longest keyword for now
 513                         keywords = new KeywordEntry<int>[11][];
 514
 515                         AddKeyword ("__arglist", Token.ARGLIST);
 516                         AddKeyword ("__makeref", Token.MAKEREF);
 517                         AddKeyword ("__reftype", Token.REFTYPE);
 518                         AddKeyword ("__refvalue", Token.REFVALUE);
 519                         AddKeyword ("abstract", Token.ABSTRACT);
 520                         AddKeyword ("as", Token.AS);
 521                         AddKeyword ("add", Token.ADD);
 522                         AddKeyword ("base", Token.BASE);
 523                         AddKeyword ("bool", Token.BOOL);
 524                         AddKeyword ("break", Token.BREAK);
 525                         AddKeyword ("byte", Token.BYTE);
 526                         AddKeyword ("case", Token.CASE);
 527                         AddKeyword ("catch", Token.CATCH);
 528                         AddKeyword ("char", Token.CHAR);
 529                         AddKeyword ("checked", Token.CHECKED);
 530                         AddKeyword ("class", Token.CLASS);
 531                         AddKeyword ("const", Token.CONST);
 532                         AddKeyword ("continue", Token.CONTINUE);
 533                         AddKeyword ("decimal", Token.DECIMAL);
 534                         AddKeyword ("default", Token.DEFAULT);
 535                         AddKeyword ("delegate", Token.DELEGATE);
 536                         AddKeyword ("do", Token.DO);
 537                         AddKeyword ("double", Token.DOUBLE);
 538                         AddKeyword ("else", Token.ELSE);
 539                         AddKeyword ("enum", Token.ENUM);
 540                         AddKeyword ("event", Token.EVENT);
 541                         AddKeyword ("explicit", Token.EXPLICIT);
 542                         AddKeyword ("extern", Token.EXTERN);
 543                         AddKeyword ("false", Token.FALSE);
 544                         AddKeyword ("finally", Token.FINALLY);
 545                         AddKeyword ("fixed", Token.FIXED);
 546                         AddKeyword ("float", Token.FLOAT);
 547                         AddKeyword ("for", Token.FOR);
 548                         AddKeyword ("foreach", Token.FOREACH);
 549                         AddKeyword ("goto", Token.GOTO);
 550                         AddKeyword ("get", Token.GET);
 551                         AddKeyword ("if", Token.IF);
 552                         AddKeyword ("implicit", Token.IMPLICIT);
 553                         AddKeyword ("in", Token.IN);
 554                         AddKeyword ("int", Token.INT);
 555                         AddKeyword ("interface", Token.INTERFACE);
 556                         AddKeyword ("internal", Token.INTERNAL);
 557                         AddKeyword ("is", Token.IS);
 558                         AddKeyword ("lock", Token.LOCK);
 559                         AddKeyword ("long", Token.LONG);
 560                         AddKeyword ("namespace", Token.NAMESPACE);
 561                         AddKeyword ("new", Token.NEW);
 562                         AddKeyword ("null", Token.NULL);
 563                         AddKeyword ("object", Token.OBJECT);
 564                         AddKeyword ("operator", Token.OPERATOR);
 565                         AddKeyword ("out", Token.OUT);
 566                         AddKeyword ("override", Token.OVERRIDE);
 567                         AddKeyword ("params", Token.PARAMS);
 568                         AddKeyword ("private", Token.PRIVATE);
 569                         AddKeyword ("protected", Token.PROTECTED);
 570                         AddKeyword ("public", Token.PUBLIC);
 571                         AddKeyword ("readonly", Token.READONLY);
 572                         AddKeyword ("ref", Token.REF);
 573                         AddKeyword ("remove", Token.REMOVE);
 574                         AddKeyword ("return", Token.RETURN);
 575                         AddKeyword ("sbyte", Token.SBYTE);
 576                         AddKeyword ("sealed", Token.SEALED);
 577                         AddKeyword ("set", Token.SET);
 578                         AddKeyword ("short", Token.SHORT);
 579                         AddKeyword ("sizeof", Token.SIZEOF);
 580                         AddKeyword ("stackalloc", Token.STACKALLOC);
 581                         AddKeyword ("static", Token.STATIC);
 582                         AddKeyword ("string", Token.STRING);
 583                         AddKeyword ("struct", Token.STRUCT);
 584                         AddKeyword ("switch", Token.SWITCH);
 585                         AddKeyword ("this", Token.THIS);
 586                         AddKeyword ("throw", Token.THROW);
 587                         AddKeyword ("true", Token.TRUE);
 588                         AddKeyword ("try", Token.TRY);
 589                         AddKeyword ("typeof", Token.TYPEOF);
 590                         AddKeyword ("uint", Token.UINT);
 591                         AddKeyword ("ulong", Token.ULONG);
 592                         AddKeyword ("unchecked", Token.UNCHECKED);
 593                         AddKeyword ("unsafe", Token.UNSAFE);
 594                         AddKeyword ("ushort", Token.USHORT);
 595                         AddKeyword ("using", Token.USING);
 596                         AddKeyword ("virtual", Token.VIRTUAL);
 597                         AddKeyword ("void", Token.VOID);
 598                         AddKeyword ("volatile", Token.VOLATILE);
 599                         AddKeyword ("while", Token.WHILE);
 600                         AddKeyword ("partial", Token.PARTIAL);
 601                         AddKeyword ("where", Token.WHERE);
 602
 603                         // LINQ keywords
 604                         AddKeyword ("from", Token.FROM);
 605                         AddKeyword ("join", Token.JOIN);
 606                         AddKeyword ("on", Token.ON);
 607                         AddKeyword ("equals", Token.EQUALS);
 608                         AddKeyword ("select", Token.SELECT);
 609                         AddKeyword ("group", Token.GROUP);
 610                         AddKeyword ("by", Token.BY);
 611                         AddKeyword ("let", Token.LET);
 612                         AddKeyword ("orderby", Token.ORDERBY);
 613                         AddKeyword ("ascending", Token.ASCENDING);
 614                         AddKeyword ("descending", Token.DESCENDING);
 615                         AddKeyword ("into", Token.INTO);
 616
 617                         // Contextual async keywords
 618                         AddKeyword ("async", Token.ASYNC);
 619                         AddKeyword ("await", Token.AWAIT);
 620
 621                         keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 622
 623                         AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 624                         AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 625                         AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 626                         AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 627                         AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 628                         AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 629                         AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 630                         AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 631                         AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 632                         AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 633                         AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 634                         AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 635
 636                         csharp_format_info = NumberFormatInfo.InvariantInfo;
 637                         styles = NumberStyles.Float;
 638                 }
 639
 640                 int GetKeyword (char[] id, int id_len)
 641                 {
 642                         //
 643                         // Keywords are stored in an array of arrays grouped by their
 644                         // length and then by the first character
 645                         //
 646                         if (id_len >= keywords.Length || keywords [id_len] == null)
 647                                 return -1;
 648
 649                         int first_index = id [0] - '_';
 650                         if (first_index > 'z' - '_')
 651                                 return -1;
 652
 653                         var kwe = keywords [id_len] [first_index];
 654                         if (kwe == null)
 655                                 return -1;
 656
 657                         int res;
 658                         do {
 659                                 res = kwe.Token;
 660                                 for (int i = 1; i < id_len; ++i) {
 661                                         if (id [i] != kwe.Value [i]) {
 662                                                 res = 0;
 663                                                 kwe = kwe.Next;
 664                                                 break;
 665                                         }
 666                                 }
 667                         } while (res == 0 && kwe != null);
 668
 669                         if (res == 0)
 670                                 return -1;
 671
 672                         int next_token;
 673                         switch (res) {
 674                         case Token.GET:
 675                         case Token.SET:
 676                                 if (!handle_get_set)
 677                                         res = -1;
 678                                 break;
 679                         case Token.REMOVE:
 680                         case Token.ADD:
 681                                 if (!handle_remove_add)
 682                                         res = -1;
 683                                 break;
 684                         case Token.EXTERN:
 685                                 if (parsing_declaration == 0)
 686                                         res = Token.EXTERN_ALIAS;
 687                                 break;
 688                         case Token.DEFAULT:
 689                                 if (peek_token () == Token.COLON) {
 690                                         token ();
 691                                         res = Token.DEFAULT_COLON;
 692                                 }
 693                                 break;
 694                         case Token.WHERE:
 695                                 if (!handle_where && !query_parsing)
 696                                         res = -1;
 697                                 break;
 698                         case Token.FROM:
 699                                 //
 700                                 // A query expression is any expression that starts with `from identifier'
 701                                 // followed by any token except ; , =
 702                                 //
 703                                 if (!query_parsing) {
 704                                         if (lambda_arguments_parsing) {
 705                                                 res = -1;
 706                                                 break;
 707                                         }
 708
 709                                         PushPosition ();
 710                                         // HACK: to disable generics micro-parser, because PushPosition does not
 711                                         // store identifiers array
 712                                         parsing_generic_less_than = 1;
 713                                         switch (xtoken ()) {
 714                                         case Token.IDENTIFIER:
 715                                         case Token.INT:
 716                                         case Token.BOOL:
 717                                         case Token.BYTE:
 718                                         case Token.CHAR:
 719                                         case Token.DECIMAL:
 720                                         case Token.FLOAT:
 721                                         case Token.LONG:
 722                                         case Token.OBJECT:
 723                                         case Token.STRING:
 724                                         case Token.UINT:
 725                                         case Token.ULONG:
 726                                                 next_token = xtoken ();
 727                                                 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
 728                                                         goto default;
 729
 730                                                 res = Token.FROM_FIRST;
 731                                                 query_parsing = true;
 732                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 733                                                         Report.FeatureIsNotAvailable (context, Location, "query expressions");
 734                                                 break;
 735                                         case Token.VOID:
 736                                                 Expression.Error_VoidInvalidInTheContext (Location, Report);
 737                                                 break;
 738                                         default:
 739                                                 PopPosition ();
 740                                                 // HACK: A token is not a keyword so we need to restore identifiers buffer
 741                                                 // which has been overwritten before we grabbed the identifier
 742                                                 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 743                                                 return -1;
 744                                         }
 745                                         PopPosition ();
 746                                 }
 747                                 break;
 748                         case Token.JOIN:
 749                         case Token.ON:
 750                         case Token.EQUALS:
 751                         case Token.SELECT:
 752                         case Token.GROUP:
 753                         case Token.BY:
 754                         case Token.LET:
 755                         case Token.ORDERBY:
 756                         case Token.ASCENDING:
 757                         case Token.DESCENDING:
 758                         case Token.INTO:
 759                                 if (!query_parsing)
 760                                         res = -1;
 761                                 break;
 762
 763                         case Token.USING:
 764                         case Token.NAMESPACE:
 765                                 // TODO: some explanation needed
 766                                 check_incorrect_doc_comment ();
 767                                 break;
 768
 769                         case Token.PARTIAL:
 770                                 if (parsing_block > 0) {
 771                                         res = -1;
 772                                         break;
 773                                 }
 774
 775                                 // Save current position and parse next token.
 776                                 PushPosition ();
 777
 778                                 next_token = token ();
 779                                 bool ok = (next_token == Token.CLASS) ||
 780                                         (next_token == Token.STRUCT) ||
 781                                         (next_token == Token.INTERFACE) ||
 782                                         (next_token == Token.VOID);
 783
 784                                 PopPosition ();
 785
 786                                 if (ok) {
 787                                         if (next_token == Token.VOID) {
 788                                                 if (context.Settings.Version <= LanguageVersion.ISO_2)
 789                                                         Report.FeatureIsNotAvailable (context, Location, "partial methods");
 790                                         } else if (context.Settings.Version == LanguageVersion.ISO_1)
 791                                                 Report.FeatureIsNotAvailable (context, Location, "partial types");
 792
 793                                         return res;
 794                                 }
 795
 796                                 if (next_token < Token.LAST_KEYWORD) {
 797                                         Report.Error (267, Location,
 798                                                 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 799                                         return token ();
 800                                 }
 801
 802                                 res = -1;
 803                                 break;
 804
 805                         case Token.ASYNC:
 806                                 if (parsing_modifiers) {
 807                                         //
 808                                         // Skip attributes section or constructor called async
 809                                         //
 810                                         if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 811                                                 res = -1;
 812                                         } else {
 813                                                 // async is keyword
 814                                         }
 815                                 } else if (parsing_block > 0) {
 816                                         switch (peek_token ()) {
 817                                         case Token.DELEGATE:
 818                                         case Token.OPEN_PARENS_LAMBDA:
 819                                                 // async is keyword
 820                                                 break;
 821                                         case Token.IDENTIFIER:
 822                                                 PushPosition ();
 823                                                 xtoken ();
 824                                                 if (xtoken () != Token.ARROW)
 825                                                         res = -1;
 826
 827                                                 PopPosition ();
 828                                                 break;
 829                                         default:
 830                                                 res = -1;
 831                                                 break;
 832                                         }
 833                                 } else {
 834                                         res = -1;
 835                                 }
 836
 837                                 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 838                                         Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 839                                 }
 840
 841                                 break;
 842
 843                         case Token.AWAIT:
 844                                 if (parsing_block == 0)
 845                                         res = -1;
 846
 847                                 break;
 848                         }
 849
 850
 851                         return res;
 852                 }
 853
 854                 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 855                 {
 856                         //
 857                         // Keywords are stored in an array of arrays grouped by their
 858                         // length and then by the first character
 859                         //
 860                         if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 861                                 return PreprocessorDirective.Invalid;
 862
 863                         int first_index = id[0] - '_';
 864                         if (first_index > 'z' - '_')
 865                                 return PreprocessorDirective.Invalid;
 866
 867                         var kwe = keywords_preprocessor[id_len][first_index];
 868                         if (kwe == null)
 869                                 return PreprocessorDirective.Invalid;
 870
 871                         PreprocessorDirective res = PreprocessorDirective.Invalid;
 872                         do {
 873                                 res = kwe.Token;
 874                                 for (int i = 1; i < id_len; ++i) {
 875                                         if (id[i] != kwe.Value[i]) {
 876                                                 res = 0;
 877                                                 kwe = kwe.Next;
 878                                                 break;
 879                                         }
 880                                 }
 881                         } while (res == PreprocessorDirective.Invalid && kwe != null);
 882
 883                         return res;
 884                 }
 885
 886                 public Location Location {
 887                         get {
 888                                 return new Location (ref_line, col);
 889                         }
 890                 }
 891
 892                 static bool is_identifier_start_character (int c)
 893                 {
 894                         return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 895                 }
 896
 897                 static bool is_identifier_part_character (char c)
 898                 {
 899                         if (c >= 'a' && c <= 'z')
 900                                 return true;
 901
 902                         if (c >= 'A' && c <= 'Z')
 903                                 return true;
 904
 905                         if (c == '_' || (c >= '0' && c <= '9'))
 906                                 return true;
 907
 908                         if (c < 0x80)
 909                                 return false;
 910
 911                         return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
 912                 }
 913
 914                 public static bool IsKeyword (string s)
 915                 {
 916                         return keyword_strings.Contains (s);
 917                 }
 918
 919                 //
 920                 // Open parens micro parser. Detects both lambda and cast ambiguity.
 921                 //
 922                 int TokenizeOpenParens ()
 923                 {
 924                         int ptoken;
 925                         current_token = -1;
 926
 927                         int bracket_level = 0;
 928                         bool is_type = false;
 929                         bool can_be_type = false;
 930
 931                         while (true) {
 932                                 ptoken = current_token;
 933                                 token ();
 934
 935                                 switch (current_token) {
 936                                 case Token.CLOSE_PARENS:
 937                                         token ();
 938
 939                                         //
 940                                         // Expression inside parens is lambda, (int i) =>
 941                                         //
 942                                         if (current_token == Token.ARROW)
 943                                                 return Token.OPEN_PARENS_LAMBDA;
 944
 945                                         //
 946                                         // Expression inside parens is single type, (int[])
 947                                         //
 948                                         if (is_type)
 949                                                 return Token.OPEN_PARENS_CAST;
 950
 951                                         //
 952                                         // Expression is possible cast, look at next token, (T)null
 953                                         //
 954                                         if (can_be_type) {
 955                                                 switch (current_token) {
 956                                                 case Token.OPEN_PARENS:
 957                                                 case Token.BANG:
 958                                                 case Token.TILDE:
 959                                                 case Token.IDENTIFIER:
 960                                                 case Token.LITERAL:
 961                                                 case Token.BASE:
 962                                                 case Token.CHECKED:
 963                                                 case Token.DELEGATE:
 964                                                 case Token.FALSE:
 965                                                 case Token.FIXED:
 966                                                 case Token.NEW:
 967                                                 case Token.NULL:
 968                                                 case Token.SIZEOF:
 969                                                 case Token.THIS:
 970                                                 case Token.THROW:
 971                                                 case Token.TRUE:
 972                                                 case Token.TYPEOF:
 973                                                 case Token.UNCHECKED:
 974                                                 case Token.UNSAFE:
 975                                                 case Token.DEFAULT:
 976
 977                                                 //
 978                                                 // These can be part of a member access
 979                                                 //
 980                                                 case Token.INT:
 981                                                 case Token.UINT:
 982                                                 case Token.SHORT:
 983                                                 case Token.USHORT:
 984                                                 case Token.LONG:
 985                                                 case Token.ULONG:
 986                                                 case Token.DOUBLE:
 987                                                 case Token.FLOAT:
 988                                                 case Token.CHAR:
 989                                                 case Token.BYTE:
 990                                                 case Token.DECIMAL:
 991                                                 case Token.BOOL:
 992                                                         return Token.OPEN_PARENS_CAST;
 993                                                 }
 994                                         }
 995                                         return Token.OPEN_PARENS;
 996
 997                                 case Token.DOT:
 998                                 case Token.DOUBLE_COLON:
 999                                         if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1000                                                 goto default;
1001
1002                                         continue;
1003
1004                                 case Token.IDENTIFIER:
1005                                         switch (ptoken) {
1006                                         case Token.DOT:
1007                                                 if (bracket_level == 0) {
1008                                                         is_type = false;
1009                                                         can_be_type = true;
1010                                                 }
1011
1012                                                 continue;
1013                                         case Token.OP_GENERICS_LT:
1014                                         case Token.COMMA:
1015                                         case Token.DOUBLE_COLON:
1016                                         case -1:
1017                                                 if (bracket_level == 0)
1018                                                         can_be_type = true;
1019                                                 continue;
1020                                         default:
1021                                                 can_be_type = is_type = false;
1022                                                 continue;
1023                                         }
1024
1025                                 case Token.OBJECT:
1026                                 case Token.STRING:
1027                                 case Token.BOOL:
1028                                 case Token.DECIMAL:
1029                                 case Token.FLOAT:
1030                                 case Token.DOUBLE:
1031                                 case Token.SBYTE:
1032                                 case Token.BYTE:
1033                                 case Token.SHORT:
1034                                 case Token.USHORT:
1035                                 case Token.INT:
1036                                 case Token.UINT:
1037                                 case Token.LONG:
1038                                 case Token.ULONG:
1039                                 case Token.CHAR:
1040                                 case Token.VOID:
1041                                         if (bracket_level == 0)
1042                                                 is_type = true;
1043                                         continue;
1044
1045                                 case Token.COMMA:
1046                                         if (bracket_level == 0) {
1047                                                 bracket_level = 100;
1048                                                 can_be_type = is_type = false;
1049                                         }
1050                                         continue;
1051
1052                                 case Token.OP_GENERICS_LT:
1053                                 case Token.OPEN_BRACKET:
1054                                         if (bracket_level++ == 0)
1055                                                 is_type = true;
1056                                         continue;
1057
1058                                 case Token.OP_GENERICS_GT:
1059                                 case Token.CLOSE_BRACKET:
1060                                         --bracket_level;
1061                                         continue;
1062
1063                                 case Token.INTERR_NULLABLE:
1064                                 case Token.STAR:
1065                                         if (bracket_level == 0)
1066                                                 is_type = true;
1067                                         continue;
1068
1069                                 case Token.REF:
1070                                 case Token.OUT:
1071                                         can_be_type = is_type = false;
1072                                         continue;
1073
1074                                 default:
1075                                         return Token.OPEN_PARENS;
1076                                 }
1077                         }
1078                 }
1079
1080                 public static bool IsValidIdentifier (string s)
1081                 {
1082                         if (s == null || s.Length == 0)
1083                                 return false;
1084
1085                         if (!is_identifier_start_character (s [0]))
1086                                 return false;
1087
1088                         for (int i = 1; i < s.Length; i ++)
1089                                 if (! is_identifier_part_character (s [i]))
1090                                         return false;
1091
1092                         return true;
1093                 }
1094
1095                 bool parse_less_than ()
1096                 {
1097                 start:
1098                         int the_token = token ();
1099                         if (the_token == Token.OPEN_BRACKET) {
1100                                 do {
1101                                         the_token = token ();
1102                                 } while (the_token != Token.CLOSE_BRACKET);
1103                                 the_token = token ();
1104                         } else if (the_token == Token.IN || the_token == Token.OUT) {
1105                                 the_token = token ();
1106                         }
1107                         switch (the_token) {
1108                         case Token.IDENTIFIER:
1109                         case Token.OBJECT:
1110                         case Token.STRING:
1111                         case Token.BOOL:
1112                         case Token.DECIMAL:
1113                         case Token.FLOAT:
1114                         case Token.DOUBLE:
1115                         case Token.SBYTE:
1116                         case Token.BYTE:
1117                         case Token.SHORT:
1118                         case Token.USHORT:
1119                         case Token.INT:
1120                         case Token.UINT:
1121                         case Token.LONG:
1122                         case Token.ULONG:
1123                         case Token.CHAR:
1124                         case Token.VOID:
1125                                 break;
1126                         case Token.OP_GENERICS_GT:
1127                         case Token.IN:
1128                         case Token.OUT:
1129                                 return true;
1130
1131                         default:
1132                                 return false;
1133                         }
1134                 again:
1135                         the_token = token ();
1136
1137                         if (the_token == Token.OP_GENERICS_GT)
1138                                 return true;
1139                         else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1140                                 goto start;
1141                         else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1142                                 goto again;
1143                         else if (the_token == Token.OP_GENERICS_LT) {
1144                                 if (!parse_less_than ())
1145                                         return false;
1146                                 goto again;
1147                         } else if (the_token == Token.OPEN_BRACKET) {
1148                         rank_specifiers:
1149                                 the_token = token ();
1150                                 if (the_token == Token.CLOSE_BRACKET)
1151                                         goto again;
1152                                 else if (the_token == Token.COMMA)
1153                                         goto rank_specifiers;
1154                                 return false;
1155                         }
1156
1157                         return false;
1158                 }
1159
1160                 bool parse_generic_dimension (out int dimension)
1161                 {
1162                         dimension = 1;
1163
1164                 again:
1165                         int the_token = token ();
1166                         if (the_token == Token.OP_GENERICS_GT)
1167                                 return true;
1168                         else if (the_token == Token.COMMA) {
1169                                 dimension++;
1170                                 goto again;
1171                         }
1172
1173                         return false;
1174                 }
1175
1176                 public int peek_token ()
1177                 {
1178                         int the_token;
1179
1180                         PushPosition ();
1181                         the_token = token ();
1182                         PopPosition ();
1183
1184                         return the_token;
1185                 }
1186
1187                 //
1188                 // Tonizes `?' using custom disambiguous rules to return one
1189                 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1190                 //
1191                 // Tricky expression look like:
1192                 //
1193                 // Foo ? a = x ? b : c;
1194                 //
1195                 int TokenizePossibleNullableType ()
1196                 {
1197                         if (parsing_block == 0 || parsing_type > 0)
1198                                 return Token.INTERR_NULLABLE;
1199
1200                         int d = peek_char ();
1201                         if (d == '?') {
1202                                 get_char ();
1203                                 return Token.OP_COALESCING;
1204                         }
1205
1206                         switch (current_token) {
1207                         case Token.CLOSE_PARENS:
1208                         case Token.TRUE:
1209                         case Token.FALSE:
1210                         case Token.NULL:
1211                         case Token.LITERAL:
1212                                 return Token.INTERR;
1213                         }
1214
1215                         if (d != ' ') {
1216                                 if (d == ',' || d == ';' || d == '>')
1217                                         return Token.INTERR_NULLABLE;
1218                                 if (d == '*' || (d >= '0' && d <= '9'))
1219                                         return Token.INTERR;
1220                         }
1221
1222                         PushPosition ();
1223                         current_token = Token.NONE;
1224                         int next_token;
1225                         switch (xtoken ()) {
1226                         case Token.LITERAL:
1227                         case Token.TRUE:
1228                         case Token.FALSE:
1229                         case Token.NULL:
1230                         case Token.THIS:
1231                         case Token.NEW:
1232                                 next_token = Token.INTERR;
1233                                 break;
1234
1235                         case Token.SEMICOLON:
1236                         case Token.COMMA:
1237                         case Token.CLOSE_PARENS:
1238                         case Token.OPEN_BRACKET:
1239                         case Token.OP_GENERICS_GT:
1240                         case Token.INTERR:
1241                                 next_token = Token.INTERR_NULLABLE;
1242                                 break;
1243
1244                         default:
1245                                 next_token = -1;
1246                                 break;
1247                         }
1248
1249                         if (next_token == -1) {
1250                                 switch (xtoken ()) {
1251                                 case Token.COMMA:
1252                                 case Token.SEMICOLON:
1253                                 case Token.OPEN_BRACE:
1254                                 case Token.CLOSE_PARENS:
1255                                 case Token.IN:
1256                                         next_token = Token.INTERR_NULLABLE;
1257                                         break;
1258
1259                                 case Token.COLON:
1260                                         next_token = Token.INTERR;
1261                                         break;
1262
1263                                 default:
1264                                         int ntoken;
1265                                         int interrs = 1;
1266                                         int colons = 0;
1267                                         //
1268                                         // All shorcuts failed, do it hard way
1269                                         //
1270                                         while ((ntoken = xtoken ()) != Token.EOF) {
1271                                                 if (ntoken == Token.SEMICOLON)
1272                                                         break;
1273
1274                                                 if (ntoken == Token.COLON) {
1275                                                         if (++colons == interrs)
1276                                                                 break;
1277                                                         continue;
1278                                                 }
1279
1280                                                 if (ntoken == Token.INTERR) {
1281                                                         ++interrs;
1282                                                         continue;
1283                                                 }
1284                                         }
1285
1286                                         next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1287                                         break;
1288                                 }
1289                         }
1290
1291                         PopPosition ();
1292                         return next_token;
1293                 }
1294
1295                 bool decimal_digits (int c)
1296                 {
1297                         int d;
1298                         bool seen_digits = false;
1299
1300                         if (c != -1){
1301                                 if (number_pos == max_number_size)
1302                                         Error_NumericConstantTooLong ();
1303                                 number_builder [number_pos++] = (char) c;
1304                         }
1305
1306                         //
1307                         // We use peek_char2, because decimal_digits needs to do a
1308                         // 2-character look-ahead (5.ToString for example).
1309                         //
1310                         while ((d = peek_char2 ()) != -1){
1311                                 if (d >= '0' && d <= '9'){
1312                                         if (number_pos == max_number_size)
1313                                                 Error_NumericConstantTooLong ();
1314                                         number_builder [number_pos++] = (char) d;
1315                                         get_char ();
1316                                         seen_digits = true;
1317                                 } else
1318                                         break;
1319                         }
1320
1321                         return seen_digits;
1322                 }
1323
1324                 static bool is_hex (int e)
1325                 {
1326                         return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1327                 }
1328
1329                 static TypeCode real_type_suffix (int c)
1330                 {
1331                         switch (c){
1332                         case 'F': case 'f':
1333                                 return TypeCode.Single;
1334                         case 'D': case 'd':
1335                                 return TypeCode.Double;
1336                         case 'M': case 'm':
1337                                 return TypeCode.Decimal;
1338                         default:
1339                                 return TypeCode.Empty;
1340                         }
1341                 }
1342
1343                 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1344                 {
1345                         bool is_unsigned = false;
1346                         bool is_long = false;
1347
1348                         if (c != -1){
1349                                 bool scanning = true;
1350                                 do {
1351                                         switch (c){
1352                                         case 'U': case 'u':
1353                                                 if (is_unsigned)
1354                                                         scanning = false;
1355                                                 is_unsigned = true;
1356                                                 get_char ();
1357                                                 break;
1358
1359                                         case 'l':
1360                                                 if (!is_unsigned){
1361                                                         //
1362                                                         // if we have not seen anything in between
1363                                                         // report this error
1364                                                         //
1365                                                         Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1366                                                 }
1367
1368                                                 goto case 'L';
1369
1370                                         case 'L':
1371                                                 if (is_long)
1372                                                         scanning = false;
1373                                                 is_long = true;
1374                                                 get_char ();
1375                                                 break;
1376
1377                                         default:
1378                                                 scanning = false;
1379                                                 break;
1380                                         }
1381                                         c = peek_char ();
1382                                 } while (scanning);
1383                         }
1384
1385                         if (is_long && is_unsigned){
1386                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1387                         }
1388
1389                         if (is_unsigned){
1390                                 // uint if possible, or ulong else.
1391
1392                                 if ((ul & 0xffffffff00000000) == 0)
1393                                         return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1394                                 else
1395                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1396                         } else if (is_long){
1397                                 // long if possible, ulong otherwise
1398                                 if ((ul & 0x8000000000000000) != 0)
1399                                         return new ULongLiteral (context.BuiltinTypes, ul, loc);
1400                                 else
1401                                         return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1402                         } else {
1403                                 // int, uint, long or ulong in that order
1404                                 if ((ul & 0xffffffff00000000) == 0){
1405                                         uint ui = (uint) ul;
1406
1407                                         if ((ui & 0x80000000) != 0)
1408                                                 return new UIntLiteral (context.BuiltinTypes, ui, loc);
1409                                         else
1410                                                 return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1411                                 } else {
1412                                         if ((ul & 0x8000000000000000) != 0)
1413                                                 return new ULongLiteral (context.BuiltinTypes, ul, loc);
1414                                         else
1415                                                 return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1416                                 }
1417                         }
1418                 }
1419
1420                 //
1421                 // given `c' as the next char in the input decide whether
1422                 // we need to convert to a special type, and then choose
1423                 // the best representation for the integer
1424                 //
1425                 ILiteralConstant adjust_int (int c, Location loc)
1426                 {
1427                         try {
1428                                 if (number_pos > 9){
1429                                         ulong ul = (uint) (number_builder [0] - '0');
1430
1431                                         for (int i = 1; i < number_pos; i++){
1432                                                 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1433                                         }
1434
1435                                         return integer_type_suffix (ul, c, loc);
1436                                 } else {
1437                                         uint ui = (uint) (number_builder [0] - '0');
1438
1439                                         for (int i = 1; i < number_pos; i++){
1440                                                 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1441                                         }
1442
1443                                         return integer_type_suffix (ui, c, loc);
1444                                 }
1445                         } catch (OverflowException) {
1446                                 Error_NumericConstantTooLong ();
1447                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1448                         }
1449                         catch (FormatException) {
1450                                 Report.Error (1013, Location, "Invalid number");
1451                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1452                         }
1453                 }
1454
1455                 ILiteralConstant adjust_real (TypeCode t, Location loc)
1456                 {
1457                         string s = new string (number_builder, 0, number_pos);
1458                         const string error_details = "Floating-point constant is outside the range of type `{0}'";
1459
1460                         switch (t){
1461                         case TypeCode.Decimal:
1462                                 try {
1463                                         return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1464                                 } catch (OverflowException) {
1465                                         Report.Error (594, Location, error_details, "decimal");
1466                                         return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1467                                 }
1468                         case TypeCode.Single:
1469                                 try {
1470                                         return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1471                                 } catch (OverflowException) {
1472                                         Report.Error (594, Location, error_details, "float");
1473                                         return new FloatLiteral (context.BuiltinTypes, 0, loc);
1474                                 }
1475                         default:
1476                                 try {
1477                                         return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1478                                 } catch (OverflowException) {
1479                                         Report.Error (594, loc, error_details, "double");
1480                                         return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1481                                 }
1482                         }
1483                 }
1484
1485                 ILiteralConstant handle_hex (Location loc)
1486                 {
1487                         int d;
1488                         ulong ul;
1489
1490                         get_char ();
1491                         while ((d = peek_char ()) != -1){
1492                                 if (is_hex (d)){
1493                                         number_builder [number_pos++] = (char) d;
1494                                         get_char ();
1495                                 } else
1496                                         break;
1497                         }
1498
1499                         string s = new String (number_builder, 0, number_pos);
1500
1501                         try {
1502                                 if (number_pos <= 8)
1503                                         ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1504                                 else
1505                                         ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1506
1507                                 return integer_type_suffix (ul, peek_char (), loc);
1508                         } catch (OverflowException){
1509                                 Error_NumericConstantTooLong ();
1510                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1511                         }
1512                         catch (FormatException) {
1513                                 Report.Error (1013, Location, "Invalid number");
1514                                 return new IntLiteral (context.BuiltinTypes, 0, loc);
1515                         }
1516                 }
1517
1518                 //
1519                 // Invoked if we know we have .digits or digits
1520                 //
1521                 int is_number (int c)
1522                 {
1523                         ILiteralConstant res;
1524
1525 #if FULL_AST
1526                         int read_start = reader.Position - 1;
1527                         if (c == '.') {
1528                                 //
1529                                 // Caller did peek_char
1530                                 //
1531                                 --read_start;
1532                         }
1533 #endif
1534                         number_pos = 0;
1535                         var loc = Location;
1536
1537                         if (c >= '0' && c <= '9'){
1538                                 if (c == '0'){
1539                                         int peek = peek_char ();
1540
1541                                         if (peek == 'x' || peek == 'X') {
1542                                                 val = res = handle_hex (loc);
1543 #if FULL_AST
1544                                                 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1545 #endif
1546
1547                                                 return Token.LITERAL;
1548                                         }
1549                                 }
1550                                 decimal_digits (c);
1551                                 c = get_char ();
1552                         }
1553
1554                         //
1555                         // We need to handle the case of
1556                         // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1557                         //
1558                         bool is_real = false;
1559                         if (c == '.'){
1560                                 if (decimal_digits ('.')){
1561                                         is_real = true;
1562                                         c = get_char ();
1563                                 } else {
1564                                         putback ('.');
1565                                         number_pos--;
1566                                         val = res = adjust_int (-1, loc);
1567
1568 #if FULL_AST
1569                                         res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1570 #endif
1571                                         return Token.LITERAL;
1572                                 }
1573                         }
1574
1575                         if (c == 'e' || c == 'E'){
1576                                 is_real = true;
1577                                 if (number_pos == max_number_size)
1578                                         Error_NumericConstantTooLong ();
1579                                 number_builder [number_pos++] = (char) c;
1580                                 c = get_char ();
1581
1582                                 if (c == '+'){
1583                                         if (number_pos == max_number_size)
1584                                                 Error_NumericConstantTooLong ();
1585                                         number_builder [number_pos++] = '+';
1586                                         c = -1;
1587                                 } else if (c == '-') {
1588                                         if (number_pos == max_number_size)
1589                                                 Error_NumericConstantTooLong ();
1590                                         number_builder [number_pos++] = '-';
1591                                         c = -1;
1592                                 } else {
1593                                         if (number_pos == max_number_size)
1594                                                 Error_NumericConstantTooLong ();
1595                                         number_builder [number_pos++] = '+';
1596                                 }
1597
1598                                 decimal_digits (c);
1599                                 c = get_char ();
1600                         }
1601
1602                         var type = real_type_suffix (c);
1603                         if (type == TypeCode.Empty && !is_real) {
1604                                 putback (c);
1605                                 res = adjust_int (c, loc);
1606                         } else {
1607                                 is_real = true;
1608
1609                                 if (type == TypeCode.Empty) {
1610                                         putback (c);
1611                                 }
1612
1613                                 res = adjust_real (type, loc);
1614                         }
1615
1616                         val = res;
1617
1618 #if FULL_AST
1619                         var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1620                         if (chars[chars.Length - 1] == '\r')
1621                                 Array.Resize (ref chars, chars.Length - 1);
1622                         res.ParsedValue = chars;
1623 #endif
1624
1625                         return Token.LITERAL;
1626                 }
1627
1628                 //
1629                 // Accepts exactly count (4 or 8) hex, no more no less
1630                 //
1631                 int getHex (int count, out int surrogate, out bool error)
1632                 {
1633                         int i;
1634                         int total = 0;
1635                         int c;
1636                         int top = count != -1 ? count : 4;
1637
1638                         get_char ();
1639                         error = false;
1640                         surrogate = 0;
1641                         for (i = 0; i < top; i++){
1642                                 c = get_char ();
1643
1644                                 if (c >= '0' && c <= '9')
1645                                         c = (int) c - (int) '0';
1646                                 else if (c >= 'A' && c <= 'F')
1647                                         c = (int) c - (int) 'A' + 10;
1648                                 else if (c >= 'a' && c <= 'f')
1649                                         c = (int) c - (int) 'a' + 10;
1650                                 else {
1651                                         error = true;
1652                                         return 0;
1653                                 }
1654
1655                                 total = (total * 16) + c;
1656                                 if (count == -1){
1657                                         int p = peek_char ();
1658                                         if (p == -1)
1659                                                 break;
1660                                         if (!is_hex ((char)p))
1661                                                 break;
1662                                 }
1663                         }
1664
1665                         if (top == 8) {
1666                                 if (total > 0x0010FFFF) {
1667                                         error = true;
1668                                         return 0;
1669                                 }
1670
1671                                 if (total >= 0x00010000) {
1672                                         surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1673                                         total = ((total - 0x00010000) / 0x0400 + 0xD800);
1674                                 }
1675                         }
1676
1677                         return total;
1678                 }
1679
1680                 int escape (int c, out int surrogate)
1681                 {
1682                         bool error;
1683                         int d;
1684                         int v;
1685
1686                         d = peek_char ();
1687                         if (c != '\\') {
1688                                 surrogate = 0;
1689                                 return c;
1690                         }
1691
1692                         switch (d){
1693                         case 'a':
1694                                 v = '\a'; break;
1695                         case 'b':
1696                                 v = '\b'; break;
1697                         case 'n':
1698                                 v = '\n'; break;
1699                         case 't':
1700                                 v = '\t'; break;
1701                         case 'v':
1702                                 v = '\v'; break;
1703                         case 'r':
1704                                 v = '\r'; break;
1705                         case '\\':
1706                                 v = '\\'; break;
1707                         case 'f':
1708                                 v = '\f'; break;
1709                         case '0':
1710                                 v = 0; break;
1711                         case '"':
1712                                 v = '"'; break;
1713                         case '\'':
1714                                 v = '\''; break;
1715                         case 'x':
1716                                 v = getHex (-1, out surrogate, out error);
1717                                 if (error)
1718                                         goto default;
1719                                 return v;
1720                         case 'u':
1721                         case 'U':
1722                                 return EscapeUnicode (d, out surrogate);
1723                         default:
1724                                 surrogate = 0;
1725                                 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1726                                 return d;
1727                         }
1728
1729                         get_char ();
1730                         surrogate = 0;
1731                         return v;
1732                 }
1733
1734                 int EscapeUnicode (int ch, out int surrogate)
1735                 {
1736                         bool error;
1737                         if (ch == 'U') {
1738                                 ch = getHex (8, out surrogate, out error);
1739                         } else {
1740                                 ch = getHex (4, out surrogate, out error);
1741                         }
1742
1743                         if (error)
1744                                 Report.Error (1009, Location, "Unrecognized escape sequence");
1745
1746                         return ch;
1747                 }
1748
1749                 int get_char ()
1750                 {
1751                         int x;
1752                         if (putback_char != -1) {
1753                                 x = putback_char;
1754                                 putback_char = -1;
1755                         } else {
1756                                 x = reader.Read ();
1757                         }
1758
1759                         if (x == '\r') {
1760                                 if (peek_char () == '\n') {
1761                                         putback_char = -1;
1762                                 }
1763
1764                                 x = '\n';
1765                                 advance_line ();
1766                         } else if (x == '\n') {
1767                                 advance_line ();
1768                         } else {
1769                                 col++;
1770                         }
1771                         return x;
1772                 }
1773
1774                 void advance_line ()
1775                 {
1776                         line++;
1777                         ref_line++;
1778                         previous_col = col;
1779                         col = 0;
1780                 }
1781
1782                 int peek_char ()
1783                 {
1784                         if (putback_char == -1)
1785                                 putback_char = reader.Read ();
1786                         return putback_char;
1787                 }
1788
1789                 int peek_char2 ()
1790                 {
1791                         if (putback_char != -1)
1792                                 return putback_char;
1793                         return reader.Peek ();
1794                 }
1795
1796                 public void putback (int c)
1797                 {
1798                         if (putback_char != -1){
1799                                 Console.WriteLine ("Col: " + col);
1800                                 Console.WriteLine ("Row: " + line);
1801                                 Console.WriteLine ("Name: " + current_source.Name);
1802                                 Console.WriteLine ("Current [{0}] putting back [{1}]  ", putback_char, c);
1803                                 throw new Exception ("This should not happen putback on putback");
1804                         }
1805                         if (c == '\n' || col == 0) {
1806                                 // It won't happen though.
1807                                 line--;
1808                                 ref_line--;
1809                                 col = previous_col;
1810                         }
1811                         else
1812                                 col--;
1813                         putback_char = c;
1814                 }
1815
1816                 public bool advance ()
1817                 {
1818                         return peek_char () != -1 || CompleteOnEOF;
1819                 }
1820
1821                 public Object Value {
1822                         get {
1823                                 return val;
1824                         }
1825                 }
1826
1827                 public Object value ()
1828                 {
1829                         return val;
1830                 }
1831
1832                 public int token ()
1833                 {
1834                         current_token = xtoken ();
1835                         return current_token;
1836                 }
1837
1838                 int TokenizePreprocessorIdentifier (out int c)
1839                 {
1840                         // skip over white space
1841                         do {
1842                                 c = get_char ();
1843                         } while (c == ' ' || c == '\t');
1844
1845
1846                         int pos = 0;
1847                         while (c != -1 && c >= 'a' && c <= 'z') {
1848                                 id_builder[pos++] = (char) c;
1849                                 c = get_char ();
1850                                 if (c == '\\') {
1851                                         int peek = peek_char ();
1852                                         if (peek == 'U' || peek == 'u') {
1853                                                 int surrogate;
1854                                                 c = EscapeUnicode (c, out surrogate);
1855                                                 if (surrogate != 0) {
1856                                                         if (is_identifier_part_character ((char) c)) {
1857                                                                 id_builder[pos++] = (char) c;
1858                                                         }
1859                                                         c = surrogate;
1860                                                 }
1861                                         }
1862                                 }
1863                         }
1864
1865                         return pos;
1866                 }
1867
1868                 PreprocessorDirective get_cmd_arg (out string arg)
1869                 {
1870                         int c;
1871
1872                         tokens_seen = false;
1873                         arg = "";
1874
1875                         var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorIdentifier (out c));
1876
1877                         if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0)
1878                                 return cmd;
1879
1880                         // skip over white space
1881                         while (c == ' ' || c == '\t')
1882                                 c = get_char ();
1883
1884                         int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument);
1885                         int pos = 0;
1886
1887                         while (c != -1 && c != '\n') {
1888                                 if (c == '\\' && has_identifier_argument >= 0) {
1889                                         if (has_identifier_argument != 0) {
1890                                                 has_identifier_argument = 1;
1891
1892                                                 int peek = peek_char ();
1893                                                 if (peek == 'U' || peek == 'u') {
1894                                                         int surrogate;
1895                                                         c = EscapeUnicode (c, out surrogate);
1896                                                         if (surrogate != 0) {
1897                                                                 if (is_identifier_part_character ((char) c)) {
1898                                                                         if (pos == value_builder.Length)
1899                                                                                 Array.Resize (ref value_builder, pos * 2);
1900
1901                                                                         value_builder[pos++] = (char) c;
1902                                                                 }
1903                                                                 c = surrogate;
1904                                                         }
1905                                                 }
1906                                         } else {
1907                                                 has_identifier_argument = -1;
1908                                         }
1909                                 } else if (c == '/' && peek_char () == '/') {
1910                                         //
1911                                         // Eat single-line comments
1912                                         //
1913                                         get_char ();
1914                                         do {
1915                                                 c = get_char ();
1916                                         } while (c != -1 && c != '\n');
1917
1918                                         break;
1919                                 }
1920
1921                                 if (pos == value_builder.Length)
1922                                         Array.Resize (ref value_builder, pos * 2);
1923
1924                                 value_builder[pos++] = (char) c;
1925                                 c = get_char ();
1926                         }
1927
1928                         if (pos != 0) {
1929                                 if (pos > max_id_size)
1930                                         arg = new string (value_builder, 0, pos);
1931                                 else
1932                                         arg = InternIdentifier (value_builder, pos);
1933
1934                                 // Eat any trailing whitespaces
1935                                 arg = arg.Trim (simple_whitespaces);
1936                         }
1937
1938                         return cmd;
1939                 }
1940
1941                 //
1942                 // Handles the #line directive
1943                 //
1944                 bool PreProcessLine ()
1945                 {
1946                         Location loc = Location;
1947
1948                         int c;
1949
1950                         int length = TokenizePreprocessorIdentifier (out c);
1951                         if (length == line_default.Length) {
1952                                 if (!IsTokenIdentifierEqual (line_default))
1953                                         return false;
1954
1955                                 current_source = source_file.SourceFile;
1956                                 if (!hidden_block_start.IsNull) {
1957                                         current_source.RegisterHiddenScope (hidden_block_start, loc);
1958                                         hidden_block_start = Location.Null;
1959                                 }
1960
1961                                 ref_line = line;
1962                                 Location.Push (current_source);
1963                                 return true;
1964                         }
1965
1966                         if (length == line_hidden.Length) {
1967                                 if (!IsTokenIdentifierEqual (line_hidden))
1968                                         return false;
1969
1970                                 if (hidden_block_start.IsNull)
1971                                         hidden_block_start = loc;
1972
1973                                 return true;
1974                         }
1975
1976                         if (length != 0 || c < '0' || c > '9') {
1977                                 //
1978                                 // Eat any remaining characters to continue parsing on next line
1979                                 //
1980                                 while (c != -1 && c != '\n') {
1981                                         c = get_char ();
1982                                 }
1983
1984                                 return false;
1985                         }
1986
1987                         int new_line = TokenizeNumber (c);
1988                         if (new_line < 1) {
1989                                 //
1990                                 // Eat any remaining characters to continue parsing on next line
1991                                 //
1992                                 while (c != -1 && c != '\n') {
1993                                         c = get_char ();
1994                                 }
1995
1996                                 return new_line != 0;
1997                         }
1998
1999                         c = get_char ();
2000                         if (c == ' ') {
2001                                 // skip over white space
2002                                 do {
2003                                         c = get_char ();
2004                                 } while (c == ' ' || c == '\t');
2005                         } else if (c == '"') {
2006                                 c = 0;
2007                         }
2008
2009                         if (c != '\n' && c != '/' && c != '"') {
2010                                 //
2011                                 // Eat any remaining characters to continue parsing on next line
2012                                 //
2013                                 while (c != -1 && c != '\n') {
2014                                         c = get_char ();
2015                                 }
2016
2017                                 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected");
2018                                 return true;
2019                         }
2020
2021                         string new_file_name = null;
2022                         if (c == '"') {
2023                                 new_file_name = TokenizeFileName (ref c);
2024
2025                                 // skip over white space
2026                                 while (c == ' ' || c == '\t') {
2027                                         c = get_char ();
2028                                 }
2029                         }
2030
2031                         if (c == '\n') {
2032                         } else if (c == '/') {
2033                                 ReadSingleLineComment ();
2034                         } else {
2035                                 //
2036                                 // Eat any remaining characters to continue parsing on next line
2037                                 //
2038                                 while (c != -1 && c != '\n') {
2039                                         c = get_char ();
2040                                 }
2041
2042                                 Error_EndLineExpected ();
2043                                 return true;
2044                         }
2045
2046                         if (new_file_name != null) {
2047                                 current_source = context.LookupFile (source_file, new_file_name);
2048                                 source_file.AddIncludeFile (current_source);
2049                                 Location.Push (current_source);
2050                         }
2051
2052                         if (!hidden_block_start.IsNull) {
2053                                 current_source.RegisterHiddenScope (hidden_block_start, loc);
2054                                 hidden_block_start = Location.Null;
2055                         }
2056
2057                         ref_line = new_line;
2058                         return true;
2059                 }
2060
2061                 //
2062                 // Handles #define and #undef
2063                 //
2064                 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
2065                 {
2066                         if (ident.Length == 0 || ident == "true" || ident == "false"){
2067                                 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
2068                                 return;
2069                         }
2070
2071                         if (ident.IndexOfAny (simple_whitespaces) != -1){
2072                                 Error_EndLineExpected ();
2073                                 return;
2074                         }
2075
2076                         if (!is_identifier_start_character (ident [0]))
2077                                 Report.Error (1001, Location, "Identifier expected: {0}", ident);
2078
2079                         foreach (char c in ident.Substring (1)){
2080                                 if (!is_identifier_part_character (c)){
2081                                         Report.Error (1001, Location, "Identifier expected: {0}",  ident);
2082                                         return;
2083                                 }
2084                         }
2085
2086                         if (!caller_is_taking)
2087                                 return;
2088
2089                         if (is_define) {
2090                                 //
2091                                 // #define ident
2092                                 //
2093                                 if (context.Settings.IsConditionalSymbolDefined (ident))
2094                                         return;
2095
2096                                 source_file.AddDefine (ident);
2097                         } else {
2098                                 //
2099                                 // #undef ident
2100                                 //
2101                                 source_file.AddUndefine (ident);
2102                         }
2103                 }
2104
2105                 byte read_hex (out bool error)
2106                 {
2107                         int total;
2108                         int c = get_char ();
2109
2110                         if ((c >= '0') && (c <= '9'))
2111                                 total = (int) c - (int) '0';
2112                         else if ((c >= 'A') && (c <= 'F'))
2113                                 total = (int) c - (int) 'A' + 10;
2114                         else if ((c >= 'a') && (c <= 'f'))
2115                                 total = (int) c - (int) 'a' + 10;
2116                         else {
2117                                 error = true;
2118                                 return 0;
2119                         }
2120
2121                         total *= 16;
2122                         c = get_char ();
2123
2124                         if ((c >= '0') && (c <= '9'))
2125                                 total += (int) c - (int) '0';
2126                         else if ((c >= 'A') && (c <= 'F'))
2127                                 total += (int) c - (int) 'A' + 10;
2128                         else if ((c >= 'a') && (c <= 'f'))
2129                                 total += (int) c - (int) 'a' + 10;
2130                         else {
2131                                 error = true;
2132                                 return 0;
2133                         }
2134
2135                         error = false;
2136                         return (byte) total;
2137                 }
2138
2139                 //
2140                 // Parses #pragma checksum
2141                 //
2142                 bool ParsePragmaChecksum ()
2143                 {
2144                         //
2145                         // The syntax is ` "foo.txt" "{guid}" "hash"'
2146                         //
2147                         int c = get_char ();
2148
2149                         if (c != '"')
2150                                 return false;
2151
2152                         string file_name = TokenizeFileName (ref c);
2153
2154                         // TODO: Any white-spaces count
2155                         if (c != ' ')
2156                                 return false;
2157
2158                         SourceFile file = context.LookupFile (source_file, file_name);
2159
2160                         if (get_char () != '"' || get_char () != '{')
2161                                 return false;
2162
2163                         bool error;
2164                         byte[] guid_bytes = new byte [16];
2165                         int i = 0;
2166
2167                         for (; i < 4; i++) {
2168                                 guid_bytes [i] = read_hex (out error);
2169                                 if (error)
2170                                         return false;
2171                         }
2172
2173                         if (get_char () != '-')
2174                                 return false;
2175
2176                         for (; i < 10; i++) {
2177                                 guid_bytes [i] = read_hex (out error);
2178                                 if (error)
2179                                         return false;
2180
2181                                 guid_bytes [i++] = read_hex (out error);
2182                                 if (error)
2183                                         return false;
2184
2185                                 if (get_char () != '-')
2186                                         return false;
2187                         }
2188
2189                         for (; i < 16; i++) {
2190                                 guid_bytes [i] = read_hex (out error);
2191                                 if (error)
2192                                         return false;
2193                         }
2194
2195                         if (get_char () != '}' || get_char () != '"')
2196                                 return false;
2197
2198                         // TODO: Any white-spaces count
2199                         c = get_char ();
2200                         if (c != ' ')
2201                                 return false;
2202
2203                         if (get_char () != '"')
2204                                 return false;
2205
2206                         // Any length of checksum
2207                         List<byte> checksum_bytes = new List<byte> (16);
2208
2209                         c = peek_char ();
2210                         while (c != '"' && c != -1) {
2211                                 checksum_bytes.Add (read_hex (out error));
2212                                 if (error)
2213                                         return false;
2214
2215                                 c = peek_char ();
2216                         }
2217
2218                         if (c == '/') {
2219                                 ReadSingleLineComment ();
2220                         } else if (get_char () != '"') {
2221                                 return false;
2222                         }
2223
2224                         file.SetChecksum (guid_bytes, checksum_bytes.ToArray ());
2225                         current_source.AutoGenerated = true;
2226                         return true;
2227                 }
2228
2229                 bool IsTokenIdentifierEqual (char[] identifier)
2230                 {
2231                         for (int i = 0; i < identifier.Length; ++i) {
2232                                 if (identifier[i] != id_builder[i])
2233                                         return false;
2234                         }
2235
2236                         return true;
2237                 }
2238
2239                 int TokenizeNumber (int value)
2240                 {
2241                         number_pos = 0;
2242
2243                         decimal_digits (value);
2244                         uint ui = (uint) (number_builder[0] - '0');
2245
2246                         try {
2247                                 for (int i = 1; i < number_pos; i++) {
2248                                         ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0')));
2249                                 }
2250
2251                                 return (int) ui;
2252                         } catch (OverflowException) {
2253                                 Error_NumericConstantTooLong ();
2254                                 return -1;
2255                         }
2256                 }
2257
2258                 string TokenizeFileName (ref int c)
2259                 {
2260                         var string_builder = new StringBuilder ();
2261                         while (c != -1 && c != '\n') {
2262                                 c = get_char ();
2263                                 if (c == '"') {
2264                                         c = get_char ();
2265                                         break;
2266                                 }
2267
2268                                 string_builder.Append ((char) c);
2269                         }
2270
2271                         if (string_builder.Length == 0) {
2272                                 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty");
2273                         }
2274
2275                         return string_builder.ToString ();
2276                 }
2277
2278                 int TokenizePragmaNumber (ref int c)
2279                 {
2280                         number_pos = 0;
2281
2282                         int number;
2283
2284                         if (c >= '0' && c <= '9') {
2285                                 number = TokenizeNumber (c);
2286
2287                                 c = get_char ();
2288
2289                                 // skip over white space
2290                                 while (c == ' ' || c == '\t')
2291                                         c = get_char ();
2292
2293                                 if (c == ',') {
2294                                         c = get_char ();
2295                                 }
2296
2297                                 // skip over white space
2298                                 while (c == ' ' || c == '\t')
2299                                         c = get_char ();
2300                         } else {
2301                                 number = -1;
2302                                 if (c == '/') {
2303                                         ReadSingleLineComment ();
2304                                 } else {
2305                                         Report.Warning (1692, 1, Location, "Invalid number");
2306
2307                                         // Read everything till the end of the line or file
2308                                         do {
2309                                                 c = get_char ();
2310                                         } while (c != -1 && c != '\n');
2311                                 }
2312                         }
2313
2314                         return number;
2315                 }
2316
2317                 void ReadSingleLineComment ()
2318                 {
2319                         if (peek_char () != '/')
2320                                 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected");
2321
2322                         // Read everything till the end of the line or file
2323                         int c;
2324                         do {
2325                                 c = get_char ();
2326                         } while (c != -1 && c != '\n');
2327                 }
2328
2329                 /// <summary>
2330                 /// Handles #pragma directive
2331                 /// </summary>
2332                 void ParsePragmaDirective (string arg)
2333                 {
2334                         int c;
2335                         int length = TokenizePreprocessorIdentifier (out c);
2336                         if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) {
2337                                 length = TokenizePreprocessorIdentifier (out c);
2338
2339                                 //
2340                                 // #pragma warning disable
2341                                 // #pragma warning restore
2342                                 //
2343                                 if (length == pragma_warning_disable.Length) {
2344                                         bool disable = IsTokenIdentifierEqual (pragma_warning_disable);
2345                                         if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) {
2346                                                 // skip over white space
2347                                                 while (c == ' ' || c == '\t')
2348                                                         c = get_char ();
2349
2350                                                 var loc = Location;
2351
2352                                                 if (c == '\n' || c == '/') {
2353                                                         if (c == '/')
2354                                                                 ReadSingleLineComment ();
2355
2356                                                         //
2357                                                         // Disable/Restore all warnings
2358                                                         //
2359                                                         if (disable) {
2360                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row);
2361                                                         } else {
2362                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row);
2363                                                         }
2364                                                 } else {
2365                                                         //
2366                                                         // Disable/Restore a warning or group of warnings
2367                                                         //
2368                                                         int code;
2369                                                         do {
2370                                                                 code = TokenizePragmaNumber (ref c);
2371                                                                 if (code > 0) {
2372                                                                         if (disable) {
2373                                                                                 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report);
2374                                                                         } else {
2375                                                                                 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context);
2376                                                                         }
2377                                                                 }
2378                                                         } while (code >= 0 && c != '\n' && c != -1);
2379                                                 }
2380
2381                                                 return;
2382                                         }
2383                                 }
2384
2385                                 Report.Warning (1634, 1, Location, "Expected disable or restore");
2386                                 return;
2387                         }
2388
2389                         //
2390                         // #pragma checksum
2391                         //
2392                         if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) {
2393                                 if (c != ' ' || !ParsePragmaChecksum ()) {
2394                                         Report.Warning (1695, 1, Location,
2395                                                 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2396                                 }
2397
2398                                 return;
2399                         }
2400
2401                         Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
2402                 }
2403
2404                 bool eval_val (string s)
2405                 {
2406                         if (s == "true")
2407                                 return true;
2408                         if (s == "false")
2409                                 return false;
2410
2411                         return source_file.IsConditionalDefined (s);
2412                 }
2413
2414                 bool pp_primary (ref string s)
2415                 {
2416                         s = s.Trim ();
2417                         int len = s.Length;
2418
2419                         if (len > 0){
2420                                 char c = s [0];
2421
2422                                 if (c == '('){
2423                                         s = s.Substring (1);
2424                                         bool val = pp_expr (ref s, false);
2425                                         if (s.Length > 0 && s [0] == ')'){
2426                                                 s = s.Substring (1);
2427                                                 return val;
2428                                         }
2429                                         Error_InvalidDirective ();
2430                                         return false;
2431                                 }
2432
2433                                 if (is_identifier_start_character (c)){
2434                                         int j = 1;
2435
2436                                         while (j < len){
2437                                                 c = s [j];
2438
2439                                                 if (is_identifier_part_character (c)){
2440                                                         j++;
2441                                                         continue;
2442                                                 }
2443                                                 bool v = eval_val (s.Substring (0, j));
2444                                                 s = s.Substring (j);
2445                                                 return v;
2446                                         }
2447                                         bool vv = eval_val (s);
2448                                         s = "";
2449                                         return vv;
2450                                 }
2451                         }
2452                         Error_InvalidDirective ();
2453                         return false;
2454                 }
2455
2456                 bool pp_unary (ref string s)
2457                 {
2458                         s = s.Trim ();
2459                         int len = s.Length;
2460
2461                         if (len > 0){
2462                                 if (s [0] == '!'){
2463                                         if (len > 1 && s [1] == '='){
2464                                                 Error_InvalidDirective ();
2465                                                 return false;
2466                                         }
2467                                         s = s.Substring (1);
2468                                         return ! pp_primary (ref s);
2469                                 } else
2470                                         return pp_primary (ref s);
2471                         } else {
2472                                 Error_InvalidDirective ();
2473                                 return false;
2474                         }
2475                 }
2476
2477                 bool pp_eq (ref string s)
2478                 {
2479                         bool va = pp_unary (ref s);
2480
2481                         s = s.Trim ();
2482                         int len = s.Length;
2483                         if (len > 0){
2484                                 if (s [0] == '='){
2485                                         if (len > 2 && s [1] == '='){
2486                                                 s = s.Substring (2);
2487                                                 return va == pp_unary (ref s);
2488                                         } else {
2489                                                 Error_InvalidDirective ();
2490                                                 return false;
2491                                         }
2492                                 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2493                                         s = s.Substring (2);
2494
2495                                         return va != pp_unary (ref s);
2496
2497                                 }
2498                         }
2499
2500                         return va;
2501
2502                 }
2503
2504                 bool pp_and (ref string s)
2505                 {
2506                         bool va = pp_eq (ref s);
2507
2508                         s = s.Trim ();
2509                         int len = s.Length;
2510                         if (len > 0){
2511                                 if (s [0] == '&'){
2512                                         if (len > 2 && s [1] == '&'){
2513                                                 s = s.Substring (2);
2514                                                 return (va & pp_and (ref s));
2515                                         } else {
2516                                                 Error_InvalidDirective ();
2517                                                 return false;
2518                                         }
2519                                 }
2520                         }
2521                         return va;
2522                 }
2523
2524                 //
2525                 // Evaluates an expression for `#if' or `#elif'
2526                 //
2527                 bool pp_expr (ref string s, bool isTerm)
2528                 {
2529                         bool va = pp_and (ref s);
2530                         s = s.Trim ();
2531                         int len = s.Length;
2532                         if (len > 0){
2533                                 char c = s [0];
2534
2535                                 if (c == '|'){
2536                                         if (len > 2 && s [1] == '|'){
2537                                                 s = s.Substring (2);
2538                                                 return va | pp_expr (ref s, isTerm);
2539                                         } else {
2540                                                 Error_InvalidDirective ();
2541                                                 return false;
2542                                         }
2543                                 }
2544                                 if (isTerm) {
2545                                         Error_EndLineExpected ();
2546                                         return false;
2547                                 }
2548                         }
2549
2550                         return va;
2551                 }
2552
2553                 bool eval (string s)
2554                 {
2555                         bool v = pp_expr (ref s, true);
2556                         s = s.Trim ();
2557                         if (s.Length != 0){
2558                                 return false;
2559                         }
2560
2561                         return v;
2562                 }
2563
2564                 void Error_NumericConstantTooLong ()
2565                 {
2566                         Report.Error (1021, Location, "Integral constant is too large");
2567                 }
2568
2569                 void Error_InvalidDirective ()
2570                 {
2571                         Report.Error (1517, Location, "Invalid preprocessor directive");
2572                 }
2573
2574                 void Error_UnexpectedDirective (string extra)
2575                 {
2576                         Report.Error (
2577                                 1028, Location,
2578                                 "Unexpected processor directive ({0})", extra);
2579                 }
2580
2581                 void Error_TokensSeen ()
2582                 {
2583                         Report.Error (1032, Location,
2584                                 "Cannot define or undefine preprocessor symbols after first token in file");
2585                 }
2586
2587                 void Eror_WrongPreprocessorLocation ()
2588                 {
2589                         Report.Error (1040, Location,
2590                                 "Preprocessor directives must appear as the first non-whitespace character on a line");
2591                 }
2592
2593                 void Error_EndLineExpected ()
2594                 {
2595                         Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2596                 }
2597
2598                 //
2599                 // Raises a warning when tokenizer found documentation comment
2600                 // on unexpected place
2601                 //
2602                 void WarningMisplacedComment (Location loc)
2603                 {
2604                         if (doc_state != XmlCommentState.Error) {
2605                                 doc_state = XmlCommentState.Error;
2606                                 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element");
2607                         }
2608                 }
2609
2610                 //
2611                 // if true, then the code continues processing the code
2612                 // if false, the code stays in a loop until another directive is
2613                 // reached.
2614                 // When caller_is_taking is false we ignore all directives except the ones
2615                 // which can help us to identify where the #if block ends
2616                 bool ParsePreprocessingDirective (bool caller_is_taking)
2617                 {
2618                         string arg;
2619                         bool region_directive = false;
2620
2621                         var directive = get_cmd_arg (out arg);
2622
2623                         //
2624                         // The first group of pre-processing instructions is always processed
2625                         //
2626                         switch (directive) {
2627                         case PreprocessorDirective.Region:
2628                                 region_directive = true;
2629                                 arg = "true";
2630                                 goto case PreprocessorDirective.If;
2631
2632                         case PreprocessorDirective.Endregion:
2633                                 if (ifstack == null || ifstack.Count == 0){
2634                                         Error_UnexpectedDirective ("no #region for this #endregion");
2635                                         return true;
2636                                 }
2637                                 int pop = ifstack.Pop ();
2638
2639                                 if ((pop & REGION) == 0)
2640                                         Report.Error (1027, Location, "Expected `#endif' directive");
2641
2642                                 return caller_is_taking;
2643
2644                         case PreprocessorDirective.If:
2645                                 if (ifstack == null)
2646                                         ifstack = new Stack<int> (2);
2647
2648                                 int flags = region_directive ? REGION : 0;
2649                                 if (ifstack.Count == 0){
2650                                         flags |= PARENT_TAKING;
2651                                 } else {
2652                                         int state = ifstack.Peek ();
2653                                         if ((state & TAKING) != 0) {
2654                                                 flags |= PARENT_TAKING;
2655                                         }
2656                                 }
2657
2658                                 if (eval (arg) && caller_is_taking) {
2659                                         ifstack.Push (flags | TAKING);
2660                                         return true;
2661                                 }
2662                                 ifstack.Push (flags);
2663                                 return false;
2664
2665                         case PreprocessorDirective.Endif:
2666                                 if (ifstack == null || ifstack.Count == 0){
2667                                         Error_UnexpectedDirective ("no #if for this #endif");
2668                                         return true;
2669                                 } else {
2670                                         pop = ifstack.Pop ();
2671
2672                                         if ((pop & REGION) != 0)
2673                                                 Report.Error (1038, Location, "#endregion directive expected");
2674
2675                                         if (arg.Length != 0) {
2676                                                 Error_EndLineExpected ();
2677                                         }
2678
2679                                         if (ifstack.Count == 0)
2680                                                 return true;
2681
2682                                         int state = ifstack.Peek ();
2683                                         return (state & TAKING) != 0;
2684                                 }
2685
2686                         case PreprocessorDirective.Elif:
2687                                 if (ifstack == null || ifstack.Count == 0){
2688                                         Error_UnexpectedDirective ("no #if for this #elif");
2689                                         return true;
2690                                 } else {
2691                                         int state = ifstack.Pop ();
2692
2693                                         if ((state & REGION) != 0) {
2694                                                 Report.Error (1038, Location, "#endregion directive expected");
2695                                                 return true;
2696                                         }
2697
2698                                         if ((state & ELSE_SEEN) != 0){
2699                                                 Error_UnexpectedDirective ("#elif not valid after #else");
2700                                                 return true;
2701                                         }
2702
2703                                         if ((state & TAKING) != 0) {
2704                                                 ifstack.Push (0);
2705                                                 return false;
2706                                         }
2707
2708                                         if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2709                                                 ifstack.Push (state | TAKING);
2710                                                 return true;
2711                                         }
2712
2713                                         ifstack.Push (state);
2714                                         return false;
2715                                 }
2716
2717                         case PreprocessorDirective.Else:
2718                                 if (ifstack == null || ifstack.Count == 0){
2719                                         Error_UnexpectedDirective ("no #if for this #else");
2720                                         return true;
2721                                 } else {
2722                                         int state = ifstack.Peek ();
2723
2724                                         if ((state & REGION) != 0) {
2725                                                 Report.Error (1038, Location, "#endregion directive expected");
2726                                                 return true;
2727                                         }
2728
2729                                         if ((state & ELSE_SEEN) != 0){
2730                                                 Error_UnexpectedDirective ("#else within #else");
2731                                                 return true;
2732                                         }
2733
2734                                         ifstack.Pop ();
2735
2736                                         if (arg.Length != 0) {
2737                                                 Error_EndLineExpected ();
2738                                                 return true;
2739                                         }
2740
2741                                         bool ret = false;
2742                                         if ((state & PARENT_TAKING) != 0) {
2743                                                 ret = (state & TAKING) == 0;
2744
2745                                                 if (ret)
2746                                                         state |= TAKING;
2747                                                 else
2748                                                         state &= ~TAKING;
2749                                         }
2750
2751                                         ifstack.Push (state | ELSE_SEEN);
2752
2753                                         return ret;
2754                                 }
2755                         case PreprocessorDirective.Define:
2756                                 if (any_token_seen){
2757                                         Error_TokensSeen ();
2758                                         return caller_is_taking;
2759                                 }
2760                                 PreProcessDefinition (true, arg, caller_is_taking);
2761                                 return caller_is_taking;
2762
2763                         case PreprocessorDirective.Undef:
2764                                 if (any_token_seen){
2765                                         Error_TokensSeen ();
2766                                         return caller_is_taking;
2767                                 }
2768                                 PreProcessDefinition (false, arg, caller_is_taking);
2769                                 return caller_is_taking;
2770
2771                         case PreprocessorDirective.Invalid:
2772                                 Report.Error (1024, Location, "Wrong preprocessor directive");
2773                                 return true;
2774                         }
2775
2776                         //
2777                         // These are only processed if we are in a `taking' block
2778                         //
2779                         if (!caller_is_taking)
2780                                 return false;
2781
2782                         switch (directive){
2783                         case PreprocessorDirective.Error:
2784                                 Report.Error (1029, Location, "#error: '{0}'", arg);
2785                                 return true;
2786
2787                         case PreprocessorDirective.Warning:
2788                                 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2789                                 return true;
2790
2791                         case PreprocessorDirective.Pragma:
2792                                 if (context.Settings.Version == LanguageVersion.ISO_1) {
2793                                         Report.FeatureIsNotAvailable (context, Location, "#pragma");
2794                                 }
2795
2796                                 ParsePragmaDirective (arg);
2797                                 return true;
2798
2799                         case PreprocessorDirective.Line:
2800                                 Location loc = Location;
2801                                 if (!PreProcessLine ())
2802                                         Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid");
2803
2804                                 return caller_is_taking;
2805                         }
2806
2807                         throw new NotImplementedException (directive.ToString ());
2808                 }
2809
2810                 private int consume_string (bool quoted)
2811                 {
2812                         int c;
2813                         int pos = 0;
2814                         Location start_location = Location;
2815                         if (quoted)
2816                                 start_location = start_location - 1;
2817
2818 #if FULL_AST
2819                         int reader_pos = reader.Position;
2820 #endif
2821
2822                         while (true){
2823                                 c = get_char ();
2824                                 if (c == '"') {
2825                                         if (quoted && peek_char () == '"') {
2826                                                 if (pos == value_builder.Length)
2827                                                         Array.Resize (ref value_builder, pos * 2);
2828
2829                                                 value_builder[pos++] = (char) c;
2830                                                 get_char ();
2831                                                 continue;
2832                                         }
2833
2834                                         string s;
2835                                         if (pos == 0)
2836                                                 s = string.Empty;
2837                                         else if (pos <= 4)
2838                                                 s = InternIdentifier (value_builder, pos);
2839                                         else
2840                                                 s = new string (value_builder, 0, pos);
2841
2842                                         ILiteralConstant res = new StringLiteral (context.BuiltinTypes, s, start_location);
2843                                         val = res;
2844 #if FULL_AST
2845                                         res.ParsedValue = quoted ?
2846                                                 reader.ReadChars (reader_pos - 2, reader.Position - 1) :
2847                                                 reader.ReadChars (reader_pos - 1, reader.Position);
2848 #endif
2849
2850                                         return Token.LITERAL;
2851                                 }
2852
2853                                 if (c == '\n') {
2854                                         if (!quoted) {
2855                                                 Report.Error (1010, Location, "Newline in constant");
2856                                                 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
2857                                                 return Token.LITERAL;
2858                                         }
2859                                 } else if (c == '\\' && !quoted) {
2860                                         int surrogate;
2861                                         c = escape (c, out surrogate);
2862                                         if (c == -1)
2863                                                 return Token.ERROR;
2864                                         if (surrogate != 0) {
2865                                                 if (pos == value_builder.Length)
2866                                                         Array.Resize (ref value_builder, pos * 2);
2867
2868                                                 value_builder[pos++] = (char) c;
2869                                                 c = surrogate;
2870                                         }
2871                                 } else if (c == -1) {
2872                                         Report.Error (1039, Location, "Unterminated string literal");
2873                                         return Token.EOF;
2874                                 }
2875
2876                                 if (pos == value_builder.Length)
2877                                         Array.Resize (ref value_builder, pos * 2);
2878
2879                                 value_builder[pos++] = (char) c;
2880                         }
2881                 }
2882
2883                 private int consume_identifier (int s)
2884                 {
2885                         int res = consume_identifier (s, false);
2886
2887                         if (doc_state == XmlCommentState.Allowed)
2888                                 doc_state = XmlCommentState.NotAllowed;
2889
2890                         return res;
2891                 }
2892
2893                 int consume_identifier (int c, bool quoted)
2894                 {
2895                         //
2896                         // This method is very performance sensitive. It accounts
2897                         // for approximately 25% of all parser time
2898                         //
2899
2900                         int pos = 0;
2901                         int column = col;
2902                         if (quoted)
2903                                 --column;
2904
2905                         if (c == '\\') {
2906                                 int surrogate;
2907                                 c = escape (c, out surrogate);
2908                                 if (surrogate != 0) {
2909                                         id_builder [pos++] = (char) c;
2910                                         c = surrogate;
2911                                 }
2912                         }
2913
2914                         id_builder [pos++] = (char) c;
2915
2916                         try {
2917                                 while (true) {
2918                                         c = reader.Read ();
2919
2920                                         if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2921                                                 id_builder [pos++] = (char) c;
2922                                                 continue;
2923                                         }
2924
2925                                         if (c < 0x80) {
2926                                                 if (c == '\\') {
2927                                                         int surrogate;
2928                                                         c = escape (c, out surrogate);
2929                                                         if (is_identifier_part_character ((char) c))
2930                                                                 id_builder[pos++] = (char) c;
2931
2932                                                         if (surrogate != 0) {
2933                                                                 c = surrogate;
2934                                                         }
2935
2936                                                         continue;
2937                                                 }
2938                                         } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2939                                                 id_builder [pos++] = (char) c;
2940                                                 continue;
2941                                         }
2942
2943                                         putback_char = c;
2944                                         break;
2945                                 }
2946                         } catch (IndexOutOfRangeException) {
2947                                 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2948                                 --pos;
2949                                 col += pos;
2950                         }
2951
2952                         col += pos - 1;
2953
2954                         //
2955                         // Optimization: avoids doing the keyword lookup
2956                         // on uppercase letters
2957                         //
2958                         if (id_builder [0] >= '_' && !quoted) {
2959                                 int keyword = GetKeyword (id_builder, pos);
2960                                 if (keyword != -1) {
2961                                         val = LocatedToken.Create (keyword == Token.AWAIT ? "await" : null, ref_line, column);
2962                                         return keyword;
2963                                 }
2964                         }
2965
2966                         string s = InternIdentifier (id_builder, pos);
2967                         val = LocatedToken.Create (s, ref_line, column);
2968                         if (quoted && parsing_attribute_section)
2969                                 AddEscapedIdentifier (((LocatedToken) val).Location);
2970
2971                         return Token.IDENTIFIER;
2972                 }
2973
2974                 string InternIdentifier (char[] charBuffer, int length)
2975                 {
2976                         //
2977                         // Keep identifiers in an array of hashtables to avoid needless
2978                         // allocations
2979                         //
2980                         var identifiers_group = identifiers[length];
2981                         string s;
2982                         if (identifiers_group != null) {
2983                                 if (identifiers_group.TryGetValue (charBuffer, out s)) {
2984                                         return s;
2985                                 }
2986                         } else {
2987                                 // TODO: this should be number of files dependant
2988                                 // corlib compilation peaks at 1000 and System.Core at 150
2989                                 int capacity = length > 20 ? 10 : 100;
2990                                 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length));
2991                                 identifiers[length] = identifiers_group;
2992                         }
2993
2994                         char[] chars = new char[length];
2995                         Array.Copy (charBuffer, chars, length);
2996
2997                         s = new string (charBuffer, 0, length);
2998                         identifiers_group.Add (chars, s);
2999                         return s;
3000                 }
3001
3002                 public int xtoken ()
3003                 {
3004                         int d, c;
3005
3006                         // Whether we have seen comments on the current line
3007                         bool comments_seen = false;
3008                         while ((c = get_char ()) != -1) {
3009                                 switch (c) {
3010                                 case '\t':
3011                                         col = ((col - 1 + tab_size) / tab_size) * tab_size;
3012                                         continue;
3013
3014                                 case ' ':
3015                                 case '\f':
3016                                 case '\v':
3017                                 case 0xa0:
3018                                 case 0:
3019                                 case 0xFEFF:    // Ignore BOM anywhere in the file
3020                                         continue;
3021
3022 /*                              This is required for compatibility with .NET
3023                                 case 0xEF:
3024                                         if (peek_char () == 0xBB) {
3025                                                 PushPosition ();
3026                                                 get_char ();
3027                                                 if (get_char () == 0xBF)
3028                                                         continue;
3029                                                 PopPosition ();
3030                                         }
3031                                         break;
3032 */
3033                                 case '\\':
3034                                         tokens_seen = true;
3035                                         return consume_identifier (c);
3036
3037                                 case '{':
3038                                         val = LocatedToken.Create (ref_line, col);
3039                                         return Token.OPEN_BRACE;
3040                                 case '}':
3041                                         val = LocatedToken.Create (ref_line, col);
3042                                         return Token.CLOSE_BRACE;
3043                                 case '[':
3044                                         // To block doccomment inside attribute declaration.
3045                                         if (doc_state == XmlCommentState.Allowed)
3046                                                 doc_state = XmlCommentState.NotAllowed;
3047
3048                                         val = LocatedToken.Create (ref_line, col);
3049
3050                                         if (parsing_block == 0 || lambda_arguments_parsing)
3051                                                 return Token.OPEN_BRACKET;
3052
3053                                         int next = peek_char ();
3054                                         switch (next) {
3055                                         case ']':
3056                                         case ',':
3057                                                 return Token.OPEN_BRACKET;
3058
3059                                         case ' ':
3060                                         case '\f':
3061                                         case '\v':
3062                                         case '\r':
3063                                         case '\n':
3064                                         case '/':
3065                                                 next = peek_token ();
3066                                                 if (next == Token.COMMA || next == Token.CLOSE_BRACKET)
3067                                                         return Token.OPEN_BRACKET;
3068
3069                                                 return Token.OPEN_BRACKET_EXPR;
3070                                         default:
3071                                                 return Token.OPEN_BRACKET_EXPR;
3072                                         }
3073                                 case ']':
3074                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3075                                         return Token.CLOSE_BRACKET;
3076                                 case '(':
3077                                         val = LocatedToken.Create (ref_line, col);
3078                                         //
3079                                         // An expression versions of parens can appear in block context only
3080                                         //
3081                                         if (parsing_block != 0 && !lambda_arguments_parsing) {
3082
3083                                                 //
3084                                                 // Optmize most common case where we know that parens
3085                                                 // is not special
3086                                                 //
3087                                                 switch (current_token) {
3088                                                 case Token.IDENTIFIER:
3089                                                 case Token.IF:
3090                                                 case Token.FOR:
3091                                                 case Token.FOREACH:
3092                                                 case Token.TYPEOF:
3093                                                 case Token.WHILE:
3094                                                 case Token.USING:
3095                                                 case Token.DEFAULT:
3096                                                 case Token.DELEGATE:
3097                                                 case Token.OP_GENERICS_GT:
3098                                                         return Token.OPEN_PARENS;
3099                                                 }
3100
3101                                                 // Optimize using peek
3102                                                 int xx = peek_char ();
3103                                                 switch (xx) {
3104                                                 case '(':
3105                                                 case '\'':
3106                                                 case '"':
3107                                                 case '0':
3108                                                 case '1':
3109                                                         return Token.OPEN_PARENS;
3110                                                 }
3111
3112                                                 lambda_arguments_parsing = true;
3113                                                 PushPosition ();
3114                                                 d = TokenizeOpenParens ();
3115                                                 PopPosition ();
3116                                                 lambda_arguments_parsing = false;
3117                                                 return d;
3118                                         }
3119
3120                                         return Token.OPEN_PARENS;
3121                                 case ')':
3122                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3123                                         return Token.CLOSE_PARENS;
3124                                 case ',':
3125                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3126                                         return Token.COMMA;
3127                                 case ';':
3128                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3129                                         return Token.SEMICOLON;
3130                                 case '~':
3131                                         val = LocatedToken.Create (ref_line, col);
3132                                         return Token.TILDE;
3133                                 case '?':
3134                                         val = LocatedToken.Create (ref_line, col);
3135                                         return TokenizePossibleNullableType ();
3136                                 case '<':
3137                                         val = LocatedToken.Create (ref_line, col);
3138                                         if (parsing_generic_less_than++ > 0)
3139                                                 return Token.OP_GENERICS_LT;
3140
3141                                         return TokenizeLessThan ();
3142
3143                                 case '>':
3144                                         val = LocatedToken.Create (ref_line, col);
3145                                         d = peek_char ();
3146
3147                                         if (d == '='){
3148                                                 get_char ();
3149                                                 return Token.OP_GE;
3150                                         }
3151
3152                                         if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
3153                                                 parsing_generic_less_than--;
3154                                                 return Token.OP_GENERICS_GT;
3155                                         }
3156
3157                                         if (d == '>') {
3158                                                 get_char ();
3159                                                 d = peek_char ();
3160
3161                                                 if (d == '=') {
3162                                                         get_char ();
3163                                                         return Token.OP_SHIFT_RIGHT_ASSIGN;
3164                                                 }
3165                                                 return Token.OP_SHIFT_RIGHT;
3166                                         }
3167
3168                                         return Token.OP_GT;
3169
3170                                 case '+':
3171                                         val = LocatedToken.Create (ref_line, col);
3172                                         d = peek_char ();
3173                                         if (d == '+') {
3174                                                 d = Token.OP_INC;
3175                                         } else if (d == '=') {
3176                                                 d = Token.OP_ADD_ASSIGN;
3177                                         } else {
3178                                                 return Token.PLUS;
3179                                         }
3180                                         get_char ();
3181                                         return d;
3182
3183                                 case '-':
3184                                         val = LocatedToken.Create (ref_line, col);
3185                                         d = peek_char ();
3186                                         if (d == '-') {
3187                                                 d = Token.OP_DEC;
3188                                         } else if (d == '=')
3189                                                 d = Token.OP_SUB_ASSIGN;
3190                                         else if (d == '>')
3191                                                 d = Token.OP_PTR;
3192                                         else {
3193                                                 return Token.MINUS;
3194                                         }
3195                                         get_char ();
3196                                         return d;
3197
3198                                 case '!':
3199                                         val = LocatedToken.Create (ref_line, col);
3200                                         if (peek_char () == '='){
3201                                                 get_char ();
3202                                                 return Token.OP_NE;
3203                                         }
3204                                         return Token.BANG;
3205
3206                                 case '=':
3207                                         val = LocatedToken.Create (ref_line, col);
3208                                         d = peek_char ();
3209                                         if (d == '='){
3210                                                 get_char ();
3211                                                 return Token.OP_EQ;
3212                                         }
3213                                         if (d == '>'){
3214                                                 get_char ();
3215                                                 return Token.ARROW;
3216                                         }
3217
3218                                         return Token.ASSIGN;
3219
3220                                 case '&':
3221                                         val = LocatedToken.Create (ref_line, col);
3222                                         d = peek_char ();
3223                                         if (d == '&'){
3224                                                 get_char ();
3225                                                 return Token.OP_AND;
3226                                         }
3227                                         if (d == '='){
3228                                                 get_char ();
3229                                                 return Token.OP_AND_ASSIGN;
3230                                         }
3231                                         return Token.BITWISE_AND;
3232
3233                                 case '|':
3234                                         val = LocatedToken.Create (ref_line, col);
3235                                         d = peek_char ();
3236                                         if (d == '|'){
3237                                                 get_char ();
3238                                                 return Token.OP_OR;
3239                                         }
3240                                         if (d == '='){
3241                                                 get_char ();
3242                                                 return Token.OP_OR_ASSIGN;
3243                                         }
3244                                         return Token.BITWISE_OR;
3245
3246                                 case '*':
3247                                         val = LocatedToken.Create (ref_line, col);
3248                                         if (peek_char () == '='){
3249                                                 get_char ();
3250                                                 return Token.OP_MULT_ASSIGN;
3251                                         }
3252                                         return Token.STAR;
3253
3254                                 case '/':
3255                                         d = peek_char ();
3256                                         if (d == '='){
3257                                                 val = LocatedToken.Create (ref_line, col);
3258                                                 get_char ();
3259                                                 return Token.OP_DIV_ASSIGN;
3260                                         }
3261
3262                                         // Handle double-slash comments.
3263                                         if (d == '/'){
3264                                                 get_char ();
3265                                                 if (doc_processing) {
3266                                                         if (peek_char () == '/') {
3267                                                                 get_char ();
3268                                                                 // Don't allow ////.
3269                                                                 if ((d = peek_char ()) != '/') {
3270                                                                         if (doc_state == XmlCommentState.Allowed)
3271                                                                                 handle_one_line_xml_comment ();
3272                                                                         else if (doc_state == XmlCommentState.NotAllowed)
3273                                                                                 WarningMisplacedComment (Location - 3);
3274                                                                 }
3275                                                         } else {
3276                                                                 if (xml_comment_buffer.Length > 0)
3277                                                                         doc_state = XmlCommentState.NotAllowed;
3278                                                         }
3279                                                 }
3280
3281                                                 while ((d = get_char ()) != -1 && d != '\n');
3282
3283                                                 any_token_seen |= tokens_seen;
3284                                                 tokens_seen = false;
3285                                                 comments_seen = false;
3286                                                 continue;
3287                                         } else if (d == '*'){
3288                                                 get_char ();
3289                                                 bool docAppend = false;
3290                                                 if (doc_processing && peek_char () == '*') {
3291                                                         get_char ();
3292                                                         // But when it is /**/, just do nothing.
3293                                                         if (peek_char () == '/') {
3294                                                                 get_char ();
3295                                                                 continue;
3296                                                         }
3297                                                         if (doc_state == XmlCommentState.Allowed)
3298                                                                 docAppend = true;
3299                                                         else if (doc_state == XmlCommentState.NotAllowed) {
3300                                                                 WarningMisplacedComment (Location - 2);
3301                                                         }
3302                                                 }
3303
3304                                                 int current_comment_start = 0;
3305                                                 if (docAppend) {
3306                                                         current_comment_start = xml_comment_buffer.Length;
3307                                                         xml_comment_buffer.Append (Environment.NewLine);
3308                                                 }
3309
3310                                                 while ((d = get_char ()) != -1){
3311                                                         if (d == '*' && peek_char () == '/'){
3312                                                                 get_char ();
3313                                                                 comments_seen = true;
3314                                                                 break;
3315                                                         }
3316                                                         if (docAppend)
3317                                                                 xml_comment_buffer.Append ((char) d);
3318
3319                                                         if (d == '\n'){
3320                                                                 any_token_seen |= tokens_seen;
3321                                                                 tokens_seen = false;
3322                                                                 //
3323                                                                 // Reset 'comments_seen' just to be consistent.
3324                                                                 // It doesn't matter either way, here.
3325                                                                 //
3326                                                                 comments_seen = false;
3327                                                         }
3328                                                 }
3329                                                 if (!comments_seen)
3330                                                         Report.Error (1035, Location, "End-of-file found, '*/' expected");
3331
3332                                                 if (docAppend)
3333                                                         update_formatted_doc_comment (current_comment_start);
3334                                                 continue;
3335                                         }
3336                                         val = LocatedToken.Create (ref_line, col);
3337                                         return Token.DIV;
3338
3339                                 case '%':
3340                                         val = LocatedToken.Create (ref_line, col);
3341                                         if (peek_char () == '='){
3342                                                 get_char ();
3343                                                 return Token.OP_MOD_ASSIGN;
3344                                         }
3345                                         return Token.PERCENT;
3346
3347                                 case '^':
3348                                         val = LocatedToken.Create (ref_line, col);
3349                                         if (peek_char () == '='){
3350                                                 get_char ();
3351                                                 return Token.OP_XOR_ASSIGN;
3352                                         }
3353                                         return Token.CARRET;
3354
3355                                 case ':':
3356                                         val = LocatedToken.Create (ref_line, col);
3357                                         if (peek_char () == ':') {
3358                                                 get_char ();
3359                                                 return Token.DOUBLE_COLON;
3360                                         }
3361                                         return Token.COLON;
3362
3363                                 case '0': case '1': case '2': case '3': case '4':
3364                                 case '5': case '6': case '7': case '8': case '9':
3365                                         tokens_seen = true;
3366                                         return is_number (c);
3367
3368                                 case '\n': // white space
3369                                         any_token_seen |= tokens_seen;
3370                                         tokens_seen = false;
3371                                         comments_seen = false;
3372                                         continue;
3373
3374                                 case '.':
3375                                         tokens_seen = true;
3376                                         d = peek_char ();
3377                                         if (d >= '0' && d <= '9')
3378                                                 return is_number (c);
3379
3380                                         LocatedToken.CreateOptional (ref_line, col, ref val);
3381                                         return Token.DOT;
3382
3383                                 case '#':
3384                                         if (tokens_seen || comments_seen) {
3385                                                 Eror_WrongPreprocessorLocation ();
3386                                                 return Token.ERROR;
3387                                         }
3388
3389                                         if (ParsePreprocessingDirective (true))
3390                                                 continue;
3391
3392                                         bool directive_expected = false;
3393                                         while ((c = get_char ()) != -1) {
3394                                                 if (col == 1) {
3395                                                         directive_expected = true;
3396                                                 } else if (!directive_expected) {
3397                                                         // TODO: Implement comment support for disabled code and uncomment this code
3398 //                                                      if (c == '#') {
3399 //                                                              Eror_WrongPreprocessorLocation ();
3400 //                                                              return Token.ERROR;
3401 //                                                      }
3402                                                         continue;
3403                                                 }
3404
3405                                                 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' )
3406                                                         continue;
3407
3408                                                 if (c == '#') {
3409                                                         if (ParsePreprocessingDirective (false))
3410                                                                 break;
3411                                                 }
3412                                                 directive_expected = false;
3413                                         }
3414
3415                                         if (c != -1) {
3416                                                 tokens_seen = false;
3417                                                 continue;
3418                                         }
3419
3420                                         return Token.EOF;
3421
3422                                 case '"':
3423                                         return consume_string (false);
3424
3425                                 case '\'':
3426                                         return TokenizeBackslash ();
3427
3428                                 case '@':
3429                                         c = get_char ();
3430                                         if (c == '"') {
3431                                                 tokens_seen = true;
3432                                                 return consume_string (true);
3433                                         }
3434
3435                                         if (is_identifier_start_character (c)){
3436                                                 return consume_identifier (c, true);
3437                                         }
3438
3439                                         Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
3440                                         return Token.ERROR;
3441
3442                                 case EvalStatementParserCharacter:
3443                                         return Token.EVAL_STATEMENT_PARSER;
3444                                 case EvalCompilationUnitParserCharacter:
3445                                         return Token.EVAL_COMPILATION_UNIT_PARSER;
3446                                 case EvalUsingDeclarationsParserCharacter:
3447                                         return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
3448                                 case DocumentationXref:
3449                                         return Token.DOC_SEE;
3450                                 }
3451
3452                                 if (is_identifier_start_character (c)) {
3453                                         tokens_seen = true;
3454                                         return consume_identifier (c);
3455                                 }
3456
3457                                 if (char.IsWhiteSpace ((char) c))
3458                                         continue;
3459
3460                                 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ());
3461                         }
3462
3463                         if (CompleteOnEOF){
3464                                 if (generated)
3465                                         return Token.COMPLETE_COMPLETION;
3466
3467                                 generated = true;
3468                                 return Token.GENERATE_COMPLETION;
3469                         }
3470
3471
3472                         return Token.EOF;
3473                 }
3474
3475                 int TokenizeBackslash ()
3476                 {
3477 #if FULL_AST
3478                         int read_start = reader.Position;
3479 #endif
3480                         Location start_location = Location;
3481                         int c = get_char ();
3482                         tokens_seen = true;
3483                         if (c == '\'') {
3484                                 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3485                                 Report.Error (1011, start_location, "Empty character literal");
3486                                 return Token.LITERAL;
3487                         }
3488
3489                         if (c == '\n') {
3490                                 Report.Error (1010, start_location, "Newline in constant");
3491                                 return Token.ERROR;
3492                         }
3493
3494                         int d;
3495                         c = escape (c, out d);
3496                         if (c == -1)
3497                                 return Token.ERROR;
3498                         if (d != 0)
3499                                 throw new NotImplementedException ();
3500
3501                         ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location);
3502                         val = res;
3503                         c = get_char ();
3504
3505                         if (c != '\'') {
3506                                 Report.Error (1012, start_location, "Too many characters in character literal");
3507
3508                                 // Try to recover, read until newline or next "'"
3509                                 while ((c = get_char ()) != -1) {
3510                                         if (c == '\n' || c == '\'')
3511                                                 break;
3512                                 }
3513                         }
3514
3515 #if FULL_AST
3516                         res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position);
3517 #endif
3518
3519                         return Token.LITERAL;
3520                 }
3521
3522                 int TokenizeLessThan ()
3523                 {
3524                         int d;
3525                         if (handle_typeof) {
3526                                 PushPosition ();
3527                                 if (parse_generic_dimension (out d)) {
3528                                         val = d;
3529                                         DiscardPosition ();
3530                                         return Token.GENERIC_DIMENSION;
3531                                 }
3532                                 PopPosition ();
3533                         }
3534
3535                         // Save current position and parse next token.
3536                         PushPosition ();
3537                         if (parse_less_than ()) {
3538                                 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) {
3539                                         d = Token.OP_GENERICS_LT_DECL;
3540                                 } else {
3541                                         d = Token.OP_GENERICS_LT;
3542                                 }
3543                                 PopPosition ();
3544                                 return d;
3545                         }
3546
3547                         PopPosition ();
3548                         parsing_generic_less_than = 0;
3549
3550                         d = peek_char ();
3551                         if (d == '<') {
3552                                 get_char ();
3553                                 d = peek_char ();
3554
3555                                 if (d == '=') {
3556                                         get_char ();
3557                                         return Token.OP_SHIFT_LEFT_ASSIGN;
3558                                 }
3559                                 return Token.OP_SHIFT_LEFT;
3560                         }
3561
3562                         if (d == '=') {
3563                                 get_char ();
3564                                 return Token.OP_LE;
3565                         }
3566                         return Token.OP_LT;
3567                 }
3568
3569                 //
3570                 // Handles one line xml comment
3571                 //
3572                 private void handle_one_line_xml_comment ()
3573                 {
3574                         int c;
3575                         while ((c = peek_char ()) == ' ')
3576                                 get_char (); // skip heading whitespaces.
3577                         while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3578                                 xml_comment_buffer.Append ((char) get_char ());
3579                         }
3580                         if (c == '\r' || c == '\n')
3581                                 xml_comment_buffer.Append (Environment.NewLine);
3582                 }
3583
3584                 //
3585                 // Remove heading "*" in Javadoc-like xml documentation.
3586                 //
3587                 private void update_formatted_doc_comment (int current_comment_start)
3588                 {
3589                         int length = xml_comment_buffer.Length - current_comment_start;
3590                         string [] lines = xml_comment_buffer.ToString (
3591                                 current_comment_start,
3592                                 length).Replace ("\r", "").Split ('\n');
3593
3594                         // The first line starts with /**, thus it is not target
3595                         // for the format check.
3596                         for (int i = 1; i < lines.Length; i++) {
3597                                 string s = lines [i];
3598                                 int idx = s.IndexOf ('*');
3599                                 string head = null;
3600                                 if (idx < 0) {
3601                                         if (i < lines.Length - 1)
3602                                                 return;
3603                                         head = s;
3604                                 } else
3605                                         head = s.Substring (0, idx);
3606                                 foreach (char c in head)
3607                                         if (c != ' ')
3608                                                 return;
3609                                 lines [i] = s.Substring (idx + 1);
3610                         }
3611                         xml_comment_buffer.Remove (current_comment_start, length);
3612                         xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3613                 }
3614
3615                 //
3616                 // Checks if there was incorrect doc comments and raise
3617                 // warnings.
3618                 //
3619                 public void check_incorrect_doc_comment ()
3620                 {
3621                         if (xml_comment_buffer.Length > 0)
3622                                 WarningMisplacedComment (Location);
3623                 }
3624
3625                 //
3626                 // Consumes the saved xml comment lines (if any)
3627                 // as for current target member or type.
3628                 //
3629                 public string consume_doc_comment ()
3630                 {
3631                         if (xml_comment_buffer.Length > 0) {
3632                                 string ret = xml_comment_buffer.ToString ();
3633                                 reset_doc_comment ();
3634                                 return ret;
3635                         }
3636                         return null;
3637                 }
3638
3639                 Report Report {
3640                         get { return context.Report; }
3641                 }
3642
3643                 void reset_doc_comment ()
3644                 {
3645                         xml_comment_buffer.Length = 0;
3646                 }
3647
3648                 public void cleanup ()
3649                 {
3650                         if (ifstack != null && ifstack.Count >= 1) {
3651                                 int state = ifstack.Pop ();
3652                                 if ((state & REGION) != 0)
3653                                         Report.Error (1038, Location, "#endregion directive expected");
3654                                 else
3655                                         Report.Error (1027, Location, "Expected `#endif' directive");
3656                         }
3657                 }
3658         }
3659
3660         //
3661         // Indicates whether it accepts XML documentation or not.
3662         //
3663         public enum XmlCommentState {
3664                 // comment is allowed in this state.
3665                 Allowed,
3666                 // comment is not allowed in this state.
3667                 NotAllowed,
3668                 // once comments appeared when it is NotAllowed, then the
3669                 // state is changed to it, until the state is changed to
3670                 // .Allowed.
3671                 Error
3672         }
3673 }
3674